Merge remote-tracking branch 'upstream/develop' into addTests

CMakeLists.txt

@@ -155,7 +155,7 @@ set (ETH_HAVE_WEBENGINE 1)
 # Backwards compatibility
 if (HEADLESS)
 	message("*** WARNING: -DHEADLESS=1 option is DEPRECATED! Use -DBUNDLE=minimal or -DGUI=0")
-	set(BUNDLE "minimal")
+	set(GUI OFF)
 endif ()
 
 # TODO: Abstract into something sensible and move into a function.
@@ -295,6 +295,7 @@ message("-- EVMJIT           Build LLVM-based JIT EVM (experimental!) ${EVMJIT}"
 message("------------------------------------------------------------------------")
 message("")
 
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/bin")
 set(CMAKE_THREAD_LIBS_INIT pthread)
 
 include(EthCompilerSettings)
@@ -396,8 +397,22 @@ if (GUI)
 
 endif()
 
+if (APPLE AND GUI)
+
+	add_custom_target(appdmg
+		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+		COMMAND ${CMAKE_COMMAND}
+		-DAPP_DMG_EXE=${ETH_APP_DMG}
+		-DAPP_DMG_FILE=appdmg.json.in
+		-DAPP_DMG_ICON="alethzero/alethzero.icns"
+		-DAPP_DMG_BACKGROUND="bg.png"
+		-DETH_BUILD_DIR="${CMAKE_BINARY_DIR}"
+		-DETH_MIX_APP="$<TARGET_FILE_DIR:mix>"
+		-DETH_ALETHZERO_APP="$<TARGET_FILE_DIR:AlethZero>"
+		-P "${ETH_SCRIPTS_DIR}/appdmg.cmake"
+	)
 
-#unset(TARGET_PLATFORM CACHE)
+endif ()
 
 if (WIN32)
 	# packaging stuff

alethzero/Context.h

@@ -29,7 +29,7 @@
 
 class QComboBox;
 
-namespace dev { namespace eth { struct StateDiff; } }
+namespace dev { namespace eth { struct StateDiff; class KeyManager; } }
 
 #define Small "font-size: small; "
 #define Mono "font-family: Ubuntu Mono, Monospace, Lucida Console, Courier New; font-weight: bold; "
@@ -64,5 +64,8 @@ public:
 	virtual std::pair<dev::Address, dev::bytes> fromString(std::string const& _a) const = 0;
 	virtual std::string renderDiff(dev::eth::StateDiff const& _d) const = 0;
 	virtual std::string render(dev::Address const& _a) const = 0;
+	virtual dev::Secret retrieveSecret(dev::Address const& _a) const = 0;
+	virtual dev::eth::KeyManager& keyManager() = 0;
+
 };
 

alethzero/Main.ui

@@ -151,6 +151,7 @@
     </property>
     <addaction name="mine"/>
     <addaction name="turboMining"/>
+    <addaction name="prepNextDAG"/>
     <addaction name="separator"/>
     <addaction name="newTransaction"/>
     <addaction name="newAccount"/>
@@ -548,12 +549,15 @@
        <property name="frameShape">
         <enum>QFrame::NoFrame</enum>
        </property>
-       <property name="dragDropMode">
-        <enum>QAbstractItemView::InternalMove</enum>
+       <property name="showDropIndicator" stdset="0">
+        <bool>false</bool>
        </property>
        <property name="alternatingRowColors">
         <bool>true</bool>
        </property>
+       <property name="sortingEnabled">
+        <bool>true</bool>
+       </property>
       </widget>
      </item>
     </layout>
@@ -1724,6 +1728,11 @@ font-size: 14pt</string>
     <string>In&amp;ject Block</string>
    </property>
   </action>
+  <action name="prepNextDAG">
+   <property name="text">
+    <string>Prepare Next &amp;DAG</string>
+   </property>
+  </action>
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <customwidgets>

alethzero/MainWin.cpp

@@ -31,6 +31,7 @@
 #include <QtWidgets/QDialog>
 #include <QtWidgets/QMessageBox>
 #include <QtWidgets/QInputDialog>
+#include <QtWidgets/QListWidgetItem>
 #include <QtWebEngine/QtWebEngine>
 #include <QtWebEngineWidgets/QWebEngineView>
 #include <QtWebEngineWidgets/QWebEngineCallback>
@@ -143,6 +144,38 @@ Main::Main(QWidget *parent) :
 //		ui->log->addItem(QString::fromStdString(s));
 	};
 
+	// Open Key Store
+	bool opened = false;
+	if (m_keyManager.exists())
+		while (!opened)
+		{
+			QString s = QInputDialog::getText(nullptr, "Master password", "Enter your MASTER account password.", QLineEdit::Password, QString());
+			if (m_keyManager.load(s.toStdString()))
+				opened = true;
+			else if (QMessageBox::question(
+					nullptr,
+					"Invalid password entered",
+					"The password you entered is incorrect. If you have forgotten your password, and you wish to start afresh, manually remove the file: " + QString::fromStdString(getDataDir("ethereum")) + "/keys.info",
+					QMessageBox::Retry,
+					QMessageBox::Abort)
+				== QMessageBox::Abort)
+				exit(0);
+		}
+	if (!opened)
+	{
+		QString password;
+		while (true)
+		{
+			password = QInputDialog::getText(nullptr, "Master password", "Enter a MASTER password for your key store. Make it strong. You probably want to write it down somewhere and keep it safe and secure; your identity will rely on this - you never want to lose it.", QLineEdit::Password, QString());
+			QString confirm = QInputDialog::getText(nullptr, "Master password", "Confirm this password by typing it again", QLineEdit::Password, QString());
+			if (password == confirm)
+				break;
+			QMessageBox::warning(nullptr, "Try again", "You entered two different passwords - please enter the same password twice.", QMessageBox::Ok);
+		}
+		m_keyManager.create(password.toStdString());
+		m_keyManager.import(Secret::random(), "Default identity");
+	}
+
 #if ETH_DEBUG
 	m_servers.append("127.0.0.1:30300");
 #endif
@@ -168,15 +201,13 @@ Main::Main(QWidget *parent) :
 
 	ui->blockCount->setText(QString("PV%1.%2 D%3 %4-%5 v%6").arg(eth::c_protocolVersion).arg(eth::c_minorProtocolVersion).arg(c_databaseVersion).arg(QString::fromStdString(ProofOfWork::name())).arg(ProofOfWork::revision()).arg(dev::Version));
 
-	connect(ui->ourAccounts->model(), SIGNAL(rowsMoved(const QModelIndex &, int, int, const QModelIndex &, int)), SLOT(ourAccountsRowsMoved()));
-
 	QSettings s("ethereum", "alethzero");
 	m_networkConfig = s.value("peers").toByteArray();
 	bytesConstRef network((byte*)m_networkConfig.data(), m_networkConfig.size());
-	m_webThree.reset(new WebThreeDirect(string("AlethZero/v") + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), getDataDir(), WithExisting::Trust, {"eth", "shh"}, p2p::NetworkPreferences(), network));
+	m_webThree.reset(new WebThreeDirect(string("AlethZero/v") + dev::Version + "/" DEV_QUOTED(ETH_BUILD_TYPE) "/" DEV_QUOTED(ETH_BUILD_PLATFORM), getDataDir(), WithExisting::Trust, {"eth"/*, "shh"*/}, p2p::NetworkPreferences(), network));
 
 	m_httpConnector.reset(new jsonrpc::HttpServer(SensibleHttpPort, "", "", dev::SensibleHttpThreads));
-	m_server.reset(new OurWebThreeStubServer(*m_httpConnector, *web3(), keysAsVector(m_myKeys), this));
+	m_server.reset(new OurWebThreeStubServer(*m_httpConnector, *web3(), this));
 	connect(&*m_server, SIGNAL(onNewId(QString)), SLOT(addNewId(QString)));
 	m_server->setIdentities(keysAsVector(owned()));
 	m_server->StartListening();
@@ -198,6 +229,7 @@ Main::Main(QWidget *parent) :
 //	ui->webView->page()->settings()->setAttribute(QWebEngineSettings::DeveloperExtrasEnabled, true);
 //	QWebEngineInspector* inspector = new QWebEngineInspector();
 //	inspector->setPage(page);
+	setBeneficiary(*m_keyManager.accounts().begin());
 	readSettings();
 #if !ETH_FATDB
 	removeDockWidget(ui->dockWidget_accounts);
@@ -358,9 +390,9 @@ void Main::installBalancesWatch()
 	// TODO: Update for new currencies reg.
 	for (unsigned i = 0; i < ethereum()->stateAt(coinsAddr, PendingBlock); ++i)
 		altCoins.push_back(right160(ethereum()->stateAt(coinsAddr, i + 1)));
-	for (auto i: m_myKeys)
+	for (auto const& i: m_keyManager.accounts())
 		for (auto c: altCoins)
-			tf.address(c).topic(0, h256(i.address(), h256::AlignRight));
+			tf.address(c).topic(0, h256(i, h256::AlignRight));
 
 	uninstallWatch(m_balancesFilter);
 	m_balancesFilter = installWatch(tf, [=](LocalisedLogEntries const&){ onBalancesChange(); });
@@ -429,7 +461,7 @@ void Main::load(QString _s)
 
 void Main::on_newTransaction_triggered()
 {
-	m_transact.setEnvironment(m_myKeys, ethereum(), &m_natSpecDB);
+	m_transact.setEnvironment(m_keyManager.accounts(), ethereum(), &m_natSpecDB);
 	m_transact.exec();
 }
 
@@ -616,17 +648,7 @@ void Main::on_paranoia_triggered()
 void Main::writeSettings()
 {
 	QSettings s("ethereum", "alethzero");
-	{
-		QByteArray b;
-		b.resize(sizeof(Secret) * m_myKeys.size());
-		auto p = b.data();
-		for (auto i: m_myKeys)
-		{
-			memcpy(p, &(i.secret()), sizeof(Secret));
-			p += sizeof(Secret);
-		}
-		s.setValue("address", b);
-	}
+	s.remove("address");
 	{
 		QByteArray b;
 		b.resize(sizeof(Secret) * m_myIdentities.size());
@@ -666,6 +688,20 @@ void Main::writeSettings()
 	s.setValue("windowState", saveState());
 }
 
+Secret Main::retrieveSecret(Address const& _a) const
+{
+	auto info = m_keyManager.accountDetails()[_a];
+	while (true)
+	{
+		if (Secret s = m_keyManager.secret(_a, [&](){
+				return QInputDialog::getText(const_cast<Main*>(this), "Import Account Key", QString("Enter the password for the account %2 (%1). The hint is:\n%3").arg(QString::fromStdString(_a.abridged())).arg(QString::fromStdString(info.first)).arg(QString::fromStdString(info.second)), QLineEdit::Password).toStdString();
+			}))
+			return s;
+		else if (QMessageBox::warning(const_cast<Main*>(this), "Incorrect Password", "The password you gave is incorrect for this key.", QMessageBox::Retry, QMessageBox::Cancel) == QMessageBox::Cancel)
+			return Secret();
+	}
+}
+
 void Main::readSettings(bool _skipGeometry)
 {
 	QSettings s("ethereum", "alethzero");
@@ -675,22 +711,17 @@ void Main::readSettings(bool _skipGeometry)
 	restoreState(s.value("windowState").toByteArray());
 
 	{
-		m_myKeys.clear();
 		QByteArray b = s.value("address").toByteArray();
-		if (b.isEmpty())
-			m_myKeys.append(KeyPair::create());
-		else
+		if (!b.isEmpty())
 		{
 			h256 k;
 			for (unsigned i = 0; i < b.size() / sizeof(Secret); ++i)
 			{
 				memcpy(&k, b.data() + i * sizeof(Secret), sizeof(Secret));
-				if (!count(m_myKeys.begin(), m_myKeys.end(), KeyPair(k)))
-					m_myKeys.append(KeyPair(k));
+				if (!m_keyManager.accounts().count(KeyPair(k).address()))
+					m_keyManager.import(k, "Imported (UNSAFE) key.");
 			}
 		}
-		ethereum()->setAddress(m_myKeys.back().address());
-		m_server->setAccounts(keysAsVector(m_myKeys));
 	}
 
 	{
@@ -735,16 +766,38 @@ void Main::readSettings(bool _skipGeometry)
 	on_urlEdit_returnPressed();
 }
 
+std::string Main::getPassword(std::string const& _title, std::string const& _for)
+{
+	QString password;
+	while (true)
+	{
+		password = QInputDialog::getText(nullptr, QString::fromStdString(_title), QString::fromStdString(_for), QLineEdit::Password, QString());
+		QString confirm = QInputDialog::getText(nullptr, QString::fromStdString(_title), "Confirm this password by typing it again", QLineEdit::Password, QString());
+		if (password == confirm)
+			break;
+		QMessageBox::warning(nullptr, QString::fromStdString(_title), "You entered two different passwords - please enter the same password twice.", QMessageBox::Ok);
+	}
+	return password.toStdString();
+}
+
 void Main::on_importKey_triggered()
 {
-	QString s = QInputDialog::getText(this, "Import Account Key", "Enter account's secret key");
+	QString s = QInputDialog::getText(this, "Import Account Key", "Enter account's secret key", QLineEdit::Password);
 	bytes b = fromHex(s.toStdString());
 	if (b.size() == 32)
 	{
 		auto k = KeyPair(h256(b));
-		if (std::find(m_myKeys.begin(), m_myKeys.end(), k) == m_myKeys.end())
+		if (!m_keyManager.accounts().count(k.address()))
 		{
-			m_myKeys.append(k);
+			QString s = QInputDialog::getText(this, "Import Account Key", "Enter this account's name");
+			if (QMessageBox::question(this, "Additional Security?", "Would you like to use additional security for this key? This lets you protect it with a different password to other keys, but also means you must re-enter the key's password every time you wish to use the account.", QMessageBox::Yes, QMessageBox::No) == QMessageBox::Yes)
+			{
+				std::string password = getPassword("Import Account Key", "Enter the password you would like to use for this key. Don't forget it!");
+				std::string hint = QInputDialog::getText(this, "Import Account Key", "Enter a hint to help you remember this password.").toStdString();
+				m_keyManager.import(k.secret(), s.toStdString(), password, hint);
+			}
+			else
+				m_keyManager.import(k.secret(), s.toStdString());
 			keysChanged();
 		}
 		else
@@ -785,15 +838,8 @@ void Main::on_importKeyFile_triggered()
 			}
 
 			cnote << k.address();
-			if (std::find(m_myKeys.begin(), m_myKeys.end(), k) == m_myKeys.end())
-			{
-				if (m_myKeys.empty())
-				{
-					m_myKeys.push_back(KeyPair::create());
-					keysChanged();
-				}
-				ethereum()->submitTransaction(k.sec(), ethereum()->balanceAt(k.address()) - gasPrice() * c_txGas, m_myKeys.back().address(), {}, c_txGas, gasPrice());
-			}
+			if (!m_keyManager.accounts().count(k.address()))
+				ethereum()->submitTransaction(k.sec(), ethereum()->balanceAt(k.address()) - gasPrice() * c_txGas, m_beneficiary, {}, c_txGas, gasPrice());
 			else
 				QMessageBox::warning(this, "Already Have Key", "Could not import the secret key: we already own this account.");
 		}
@@ -812,10 +858,12 @@ void Main::on_importKeyFile_triggered()
 
 void Main::on_exportKey_triggered()
 {
-	if (ui->ourAccounts->currentRow() >= 0 && ui->ourAccounts->currentRow() < m_myKeys.size())
+	if (ui->ourAccounts->currentRow() >= 0)
 	{
-		auto k = m_myKeys[ui->ourAccounts->currentRow()];
-		QMessageBox::information(this, "Export Account Key", "Secret key to account " + QString::fromStdString(render(k.address()) + " is:\n" + toHex(k.sec().ref())));
+		auto hba = ui->ourAccounts->currentItem()->data(Qt::UserRole).toByteArray();
+		Address h((byte const*)hba.data(), Address::ConstructFromPointer);
+		Secret s = retrieveSecret(h);
+		QMessageBox::information(this, "Export Account Key", "Secret key to account " + QString::fromStdString(render(h) + " is:\n" + s.hex()));
 	}
 }
 
@@ -895,22 +943,48 @@ void Main::on_preview_triggered()
 	refreshAll();
 }
 
+void Main::on_prepNextDAG_triggered()
+{
+	EthashAux::computeFull(ethereum()->blockChain().number() + ETHASH_EPOCH_LENGTH);
+}
+
 void Main::refreshMining()
 {
+	pair<uint64_t, unsigned> gp = EthashAux::fullGeneratingProgress();
+	QString t;
+	if (gp.first != EthashAux::NotGenerating)
+		t = QString("DAG for #%1-#%2: %3% complete; ").arg(gp.first).arg(gp.first + ETHASH_EPOCH_LENGTH - 1).arg(gp.second);
 	MiningProgress p = ethereum()->miningProgress();
-	ui->mineStatus->setText(ethereum()->isMining() ? QString("%1s @ %2kH/s").arg(p.ms / 1000).arg(p.ms ? p.hashes / p.ms : 0) : "Not mining");
-	if (!ui->miningView->isVisible())
-		return;
-	list<MineInfo> l = ethereum()->miningHistory();
-	static unsigned lh = 0;
-	if (p.hashes < lh)
-		ui->miningView->resetStats();
-	lh = p.hashes;
-	ui->miningView->appendStats(l, p);
-/*	if (p.ms)
-		for (MineInfo const& i: l)
-			cnote << i.hashes * 10 << "h/sec, need:" << i.requirement << " best:" << i.best << " best-so-far:" << p.best << " avg-speed:" << (p.hashes * 1000 / p.ms) << "h/sec";
-*/
+	ui->mineStatus->setText(t + (ethereum()->isMining() ? p.hashes > 0 ? QString("%1s @ %2kH/s").arg(p.ms / 1000).arg(p.ms ? p.hashes / p.ms : 0) : "Awaiting DAG" : "Not mining"));
+	if (ethereum()->isMining() && p.hashes > 0)
+	{
+		if (!ui->miningView->isVisible())
+			return;
+		list<MineInfo> l = ethereum()->miningHistory();
+		static unsigned lh = 0;
+		if (p.hashes < lh)
+			ui->miningView->resetStats();
+		lh = p.hashes;
+		ui->miningView->appendStats(l, p);
+	}
+}
+
+void Main::setBeneficiary(Address const& _b)
+{
+	for (int i = 0; i < ui->ourAccounts->count(); ++i)
+	{
+		auto hba = ui->ourAccounts->item(i)->data(Qt::UserRole).toByteArray();
+		auto h = Address((byte const*)hba.data(), Address::ConstructFromPointer);
+		ui->ourAccounts->item(i)->setCheckState(h == _b ? Qt::Checked : Qt::Unchecked);
+	}
+	m_beneficiary = _b;
+	ethereum()->setAddress(_b);
+}
+
+void Main::on_ourAccounts_itemClicked(QListWidgetItem* _i)
+{
+	auto hba = _i->data(Qt::UserRole).toByteArray();
+	setBeneficiary(Address((byte const*)hba.data(), Address::ConstructFromPointer));
 }
 
 void Main::refreshBalances()
@@ -931,11 +1005,13 @@ void Main::refreshBalances()
 //		cdebug << n << addr << denom << sha3(h256(n).asBytes());
 		altCoins[addr] = make_tuple(fromRaw(n), 0, denom);
 	}*/
-	for (auto i: m_myKeys)
+	for (pair<Address, std::pair<std::string, std::string>> const& i: m_keyManager.accountDetails())
 	{
-		u256 b = ethereum()->balanceAt(i.address());
-		(new QListWidgetItem(QString("%2: %1 [%3]").arg(formatBalance(b).c_str()).arg(QString::fromStdString(render(i.address()))).arg((unsigned)ethereum()->countAt(i.address())), ui->ourAccounts))
-			->setData(Qt::UserRole, QByteArray((char const*)i.address().data(), Address::size));
+		u256 b = ethereum()->balanceAt(i.first);
+		QListWidgetItem* li = new QListWidgetItem(QString("%4 %2: %1 [%3]").arg(formatBalance(b).c_str()).arg(QString::fromStdString(render(i.first))).arg((unsigned)ethereum()->countAt(i.first)).arg(QString::fromStdString(i.second.first)), ui->ourAccounts);
+		li->setData(Qt::UserRole, QByteArray((char const*)i.first.data(), Address::size));
+		li->setFlags(Qt::ItemIsUserCheckable | Qt::ItemIsEnabled);
+		li->setCheckState(m_beneficiary == i.first ? Qt::Checked : Qt::Unchecked);
 		totalBalance += b;
 
 //		for (auto& c: altCoins)
@@ -1385,23 +1461,6 @@ void Main::on_transactionQueue_currentItemChanged()
 	ui->pendingInfo->moveCursor(QTextCursor::Start);
 }
 
-void Main::ourAccountsRowsMoved()
-{
-	QList<KeyPair> myKeys;
-	for (int i = 0; i < ui->ourAccounts->count(); ++i)
-	{
-		auto hba = ui->ourAccounts->item(i)->data(Qt::UserRole).toByteArray();
-		auto h = Address((byte const*)hba.data(), Address::ConstructFromPointer);
-		for (auto i: m_myKeys)
-			if (i.address() == h)
-				myKeys.push_back(i);
-	}
-	m_myKeys = myKeys;
-
-	if (m_server.get())
-		m_server->setAccounts(keysAsVector(m_myKeys));
-}
-
 void Main::on_inject_triggered()
 {
 	QString s = QInputDialog::getText(this, "Inject Transaction", "Enter transaction dump in hex");
@@ -1827,7 +1886,8 @@ void Main::on_mine_triggered()
 {
 	if (ui->mine->isChecked())
 	{
-		ethereum()->setAddress(m_myKeys.last().address());
+//		EthashAux::computeFull(ethereum()->blockChain().number());
+		ethereum()->setAddress(m_beneficiary);
 		ethereum()->startMining();
 	}
 	else
@@ -1837,7 +1897,6 @@ void Main::on_mine_triggered()
 void Main::keysChanged()
 {
 	onBalancesChange();
-	m_server->setAccounts(keysAsVector(m_myKeys));
 }
 
 bool beginsWith(Address _a, bytes const& _b)
@@ -1901,24 +1960,39 @@ void Main::on_newAccount_triggered()
 		t->join();
 		delete t;
 	}
-	m_myKeys.append(p);
+
+	QString s = QInputDialog::getText(this, "Create Account", "Enter this account's name");
+	if (QMessageBox::question(this, "Create Account", "Would you like to use additional security for this key? This lets you protect it with a different password to other keys, but also means you must re-enter the key's password every time you wish to use the account.", QMessageBox::Yes, QMessageBox::No) == QMessageBox::Yes)
+	{
+		std::string password = getPassword("Create Account", "Enter the password you would like to use for this key. Don't forget it!");
+		std::string hint = QInputDialog::getText(this, "Create Account", "Enter a hint to help you remember this password.").toStdString();
+		m_keyManager.import(p.secret(), s.toStdString(), password, hint);
+	}
+	else
+		m_keyManager.import(p.secret(), s.toStdString());
 	keysChanged();
 }
 
 void Main::on_killAccount_triggered()
 {
-	if (ui->ourAccounts->currentRow() >= 0 && ui->ourAccounts->currentRow() < m_myKeys.size())
+	if (ui->ourAccounts->currentRow() >= 0)
 	{
-		auto k = m_myKeys[ui->ourAccounts->currentRow()];
+		auto hba = ui->accounts->currentItem()->data(Qt::UserRole).toByteArray();
+		Address h((byte const*)hba.data(), Address::ConstructFromPointer);
+		auto k = m_keyManager.accountDetails()[h];
 		if (
-			ethereum()->balanceAt(k.address()) != 0 &&
-			QMessageBox::critical(this, "Kill Account?!",
-				QString::fromStdString("Account " + render(k.address()) + " has " + formatBalance(ethereum()->balanceAt(k.address())) + " in it. It, and any contract that this account can access, will be lost forever if you continue. Do NOT continue unless you know what you are doing.\n"
+			ethereum()->balanceAt(h) != 0 &&
+			QMessageBox::critical(this, QString::fromStdString("Kill Account " + k.first + "?!"),
+				QString::fromStdString("Account " + k.first + " (" + render(h) + ") has " + formatBalance(ethereum()->balanceAt(h)) + " in it. It, and any contract that this account can access, will be lost forever if you continue. Do NOT continue unless you know what you are doing.\n"
 				"Are you sure you want to continue?"),
 				QMessageBox::Yes, QMessageBox::No) == QMessageBox::No)
 			return;
-		m_myKeys.erase(m_myKeys.begin() + ui->ourAccounts->currentRow());
+		m_keyManager.kill(h);
+		if (m_keyManager.accounts().empty())
+			m_keyManager.import(Secret::random(), "Default account");
 		keysChanged();
+		if (m_beneficiary == h)
+			setBeneficiary(*m_keyManager.accounts().begin());
 	}
 }
 
@@ -1929,7 +2003,8 @@ void Main::on_go_triggered()
 		ui->net->setChecked(true);
 		on_net_triggered();
 	}
-	web3()->addNode(p2p::NodeId(), Host::pocHost());
+	for (auto const& i: Host::pocHosts())
+		web3()->requirePeer(i.first, i.second);
 }
 
 std::string Main::prettyU256(dev::u256 const& _n) const
@@ -1961,6 +2036,7 @@ std::string Main::prettyU256(dev::u256 const& _n) const
 
 void Main::on_post_clicked()
 {
+	return;
 	shh::Message m;
 	m.setTo(stringToPublic(ui->shhTo->currentText()));
 	m.setPayload(parseData(ui->shhData->toPlainText().toStdString()));
@@ -1985,6 +2061,7 @@ int Main::authenticate(QString _title, QString _text)
 
 void Main::refreshWhispers()
 {
+	return;
 	ui->whispers->clear();
 	for (auto const& w: whisper()->all())
 	{

alethzero/MainWin.h

@@ -35,6 +35,7 @@
 #include <libethcore/Common.h>
 #include <libethereum/State.h>
 #include <libethereum/Executive.h>
+#include <libethereum/KeyManager.h>
 #include <libwebthree/WebThree.h>
 #include <libsolidity/CompilerStack.h>
 #include "Context.h"
@@ -42,6 +43,8 @@
 #include "NatspecHandler.h"
 #include "Connect.h"
 
+class QListWidgetItem;
+
 namespace Ui {
 class Main;
 }
@@ -86,10 +89,14 @@ public:
 	std::pair<dev::Address, dev::bytes> fromString(std::string const& _a) const override;
 	std::string renderDiff(dev::eth::StateDiff const& _d) const override;
 
-	QList<dev::KeyPair> owned() const { return m_myIdentities + m_myKeys; }
+	QList<dev::KeyPair> owned() const { return m_myIdentities; }
 
 	dev::u256 gasPrice() const { return 10 * dev::eth::szabo; }
 
+	dev::eth::KeyManager& keyManager() override { return m_keyManager; }
+
+	dev::Secret retrieveSecret(dev::Address const& _a) const override;
+
 public slots:
 	void load(QString _file);
 	void note(QString _entry);
@@ -117,6 +124,7 @@ private slots:
 
 	// Mining
 	void on_mine_triggered();
+	void on_prepNextDAG_triggered();
 
 	// View
 	void on_refresh_triggered();
@@ -144,7 +152,7 @@ private slots:
 	void on_exportState_triggered();
 
 	// Stuff concerning the blocks/transactions/accounts panels
-	void ourAccountsRowsMoved();
+	void on_ourAccounts_itemClicked(QListWidgetItem* _i);
 	void on_ourAccounts_doubleClicked();
 	void on_accounts_doubleClicked();
 	void on_accounts_currentItemChanged();
@@ -236,6 +244,9 @@ private:
 	void refreshBlockCount();
 	void refreshBalances();
 
+	void setBeneficiary(dev::Address const& _b);
+	std::string getPassword(std::string const& _title, std::string const& _for);
+
 	std::unique_ptr<Ui::Main> ui;
 
 	std::unique_ptr<dev::WebThreeDirect> m_webThree;
@@ -247,10 +258,11 @@ private:
 
 	QByteArray m_networkConfig;
 	QStringList m_servers;
-	QList<dev::KeyPair> m_myKeys;
 	QList<dev::KeyPair> m_myIdentities;
+	dev::eth::KeyManager m_keyManager;
 	QString m_privateChain;
 	dev::Address m_nameReg;
+	dev::Address m_beneficiary;
 
 	QList<QPair<QString, QString>> m_consoleHistory;
 	QMutex m_logLock;

alethzero/OurWebThreeStubServer.cpp

@@ -20,23 +20,23 @@
  */
 
 #include "OurWebThreeStubServer.h"
-
 #include <QMessageBox>
 #include <QAbstractButton>
 #include <libwebthree/WebThree.h>
 #include <libnatspec/NatspecExpressionEvaluator.h>
-
 #include "MainWin.h"
-
 using namespace std;
 using namespace dev;
 using namespace dev::eth;
 
-OurWebThreeStubServer::OurWebThreeStubServer(jsonrpc::AbstractServerConnector& _conn, WebThreeDirect& _web3,
-											 vector<KeyPair> const& _accounts, Main* _main):
-	WebThreeStubServer(_conn, _web3, _accounts), m_web3(&_web3), m_main(_main)
+OurWebThreeStubServer::OurWebThreeStubServer(
+	jsonrpc::AbstractServerConnector& _conn,
+	WebThreeDirect& _web3,
+	Main* _main
+):
+	WebThreeStubServer(_conn, _web3, make_shared<OurAccountHolder>(_web3, _main), _main->owned().toVector().toStdVector()),
+	m_main(_main)
 {
-	connect(_main, SIGNAL(poll()), this, SLOT(doValidations()));
 }
 
 string OurWebThreeStubServer::shh_newIdentity()
@@ -46,7 +46,18 @@ string OurWebThreeStubServer::shh_newIdentity()
 	return toJS(kp.pub());
 }
 
-bool OurWebThreeStubServer::showAuthenticationPopup(string const& _title, string const& _text)
+OurAccountHolder::OurAccountHolder(
+	WebThreeDirect& _web3,
+	Main* _main
+):
+	AccountHolder([=](){ return m_web3->ethereum(); }),
+	m_web3(&_web3),
+	m_main(_main)
+{
+	connect(_main, SIGNAL(poll()), this, SLOT(doValidations()));
+}
+
+bool OurAccountHolder::showAuthenticationPopup(string const& _title, string const& _text)
 {
 	if (!m_main->confirm())
 	{
@@ -66,18 +77,18 @@ bool OurWebThreeStubServer::showAuthenticationPopup(string const& _title, string
 	//return button == QMessageBox::Ok;
 }
 
-bool OurWebThreeStubServer::showCreationNotice(TransactionSkeleton const& _t, bool _toProxy)
+bool OurAccountHolder::showCreationNotice(TransactionSkeleton const& _t, bool _toProxy)
 {
 	return showAuthenticationPopup("Contract Creation Transaction", string("ÐApp is attemping to create a contract; ") + (_toProxy ? "(this transaction is not executed directly, but forwarded to another ÐApp) " : "") + "to be endowed with " + formatBalance(_t.value) + ", with additional network fees of up to " + formatBalance(_t.gas * _t.gasPrice) + ".\n\nMaximum total cost is " + formatBalance(_t.value + _t.gas * _t.gasPrice) + ".");
 }
 
-bool OurWebThreeStubServer::showSendNotice(TransactionSkeleton const& _t, bool _toProxy)
+bool OurAccountHolder::showSendNotice(TransactionSkeleton const& _t, bool _toProxy)
 {
 	return showAuthenticationPopup("Fund Transfer Transaction", "ÐApp is attempting to send " + formatBalance(_t.value) + " to a recipient " + m_main->pretty(_t.to) + (_toProxy ? " (this transaction is not executed directly, but forwarded to another ÐApp)" : "") +
 ", with additional network fees of up to " + formatBalance(_t.gas * _t.gasPrice) + ".\n\nMaximum total cost is " + formatBalance(_t.value + _t.gas * _t.gasPrice) + ".");
 }
 
-bool OurWebThreeStubServer::showUnknownCallNotice(TransactionSkeleton const& _t, bool _toProxy)
+bool OurAccountHolder::showUnknownCallNotice(TransactionSkeleton const& _t, bool _toProxy)
 {
 	return showAuthenticationPopup("DANGEROUS! Unknown Contract Transaction!",
 		"ÐApp is attempting to call into an unknown contract at address " +
@@ -93,25 +104,47 @@ bool OurWebThreeStubServer::showUnknownCallNotice(TransactionSkeleton const& _t,
 		"REJECT UNLESS YOU REALLY KNOW WHAT YOU ARE DOING!");
 }
 
-void OurWebThreeStubServer::authenticate(TransactionSkeleton const& _t, bool _toProxy)
+void OurAccountHolder::authenticate(TransactionSkeleton const& _t)
 {
 	Guard l(x_queued);
-	m_queued.push(make_pair(_t, _toProxy));
+	m_queued.push(_t);
 }
 
-void OurWebThreeStubServer::doValidations()
+void OurAccountHolder::doValidations()
 {
 	Guard l(x_queued);
 	while (!m_queued.empty())
 	{
-		auto q = m_queued.front();
+		auto t = m_queued.front();
 		m_queued.pop();
-		if (validateTransaction(q.first, q.second))
-			WebThreeStubServerBase::authenticate(q.first, q.second);
+
+		bool proxy = isProxyAccount(t.from);
+		if (!proxy && !isRealAccount(t.from))
+		{
+			cwarn << "Trying to send from non-existant account" << t.from;
+			return;
+		}
+
+		// TODO: determine gas price.
+
+		if (!validateTransaction(t, proxy))
+			return;
+
+		if (proxy)
+			queueTransaction(t);
+		else
+			// sign and submit.
+			if (Secret s = m_main->retrieveSecret(t.from))
+				m_web3->ethereum()->submitTransaction(s, t);
 	}
 }
 
-bool OurWebThreeStubServer::validateTransaction(TransactionSkeleton const& _t, bool _toProxy)
+AddressHash OurAccountHolder::realAccounts() const
+{
+	return m_main->keyManager().accounts();
+}
+
+bool OurAccountHolder::validateTransaction(TransactionSkeleton const& _t, bool _toProxy)
 {
 	if (_t.creation)
 	{

alethzero/OurWebThreeStubServer.h

@@ -25,26 +25,29 @@
 #include <libethcore/CommonJS.h>
 #include <libdevcrypto/Common.h>
 #include <libweb3jsonrpc/WebThreeStubServer.h>
+#include <libweb3jsonrpc/AccountHolder.h>
 
 class Main;
 
-class OurWebThreeStubServer: public QObject, public WebThreeStubServer
+class OurAccountHolder: public QObject, public dev::eth::AccountHolder
 {
 	Q_OBJECT
 
 public:
-	OurWebThreeStubServer(jsonrpc::AbstractServerConnector& _conn, dev::WebThreeDirect& _web3,
-						  std::vector<dev::KeyPair> const& _accounts, Main* main);
-
-	virtual std::string shh_newIdentity() override;
-	virtual void authenticate(dev::eth::TransactionSkeleton const& _t, bool _toProxy);
-
-signals:
-	void onNewId(QString _s);
+	OurAccountHolder(
+		dev::WebThreeDirect& _web3,
+		Main* _main
+	);
 
 public slots:
 	void doValidations();
 
+protected:
+	// easiest to return keyManager.addresses();
+	virtual dev::AddressHash realAccounts() const override;
+	// use web3 to submit a signed transaction to accept
+	virtual void authenticate(dev::eth::TransactionSkeleton const& _t) override;
+
 private:
 	bool showAuthenticationPopup(std::string const& _title, std::string const& _text);
 	bool showCreationNotice(dev::eth::TransactionSkeleton const& _t, bool _toProxy);
@@ -53,9 +56,29 @@ private:
 
 	bool validateTransaction(dev::eth::TransactionSkeleton const& _t, bool _toProxy);
 
-	std::queue<std::pair<dev::eth::TransactionSkeleton, bool>> m_queued;
+	std::queue<dev::eth::TransactionSkeleton> m_queued;
 	dev::Mutex x_queued;
 
 	dev::WebThreeDirect* m_web3;
 	Main* m_main;
 };
+
+class OurWebThreeStubServer: public QObject, public WebThreeStubServer
+{
+	Q_OBJECT
+
+public:
+	OurWebThreeStubServer(
+		jsonrpc::AbstractServerConnector& _conn,
+		dev::WebThreeDirect& _web3,
+		Main* main
+	);
+
+	virtual std::string shh_newIdentity() override;
+
+signals:
+	void onNewId(QString _s);
+
+private:
+	Main* m_main;
+};

alethzero/Transact.cpp

@@ -39,6 +39,7 @@
 #include <libnatspec/NatspecExpressionEvaluator.h>
 #include <libethereum/Client.h>
 #include <libethereum/Utility.h>
+#include <libethereum/KeyManager.h>
 #if ETH_SERPENT
 #include <libserpent/funcs.h>
 #include <libserpent/util.h>
@@ -69,11 +70,20 @@ Transact::~Transact()
 	delete ui;
 }
 
-void Transact::setEnvironment(QList<dev::KeyPair> _myKeys, dev::eth::Client* _eth, NatSpecFace* _natSpecDB)
+void Transact::setEnvironment(AddressHash const& _accounts, dev::eth::Client* _eth, NatSpecFace* _natSpecDB)
 {
-	m_myKeys = _myKeys;
+	m_accounts = _accounts;
 	m_ethereum = _eth;
 	m_natSpecDB = _natSpecDB;
+
+	ui->from->clear();
+	for (auto const& i: m_accounts)
+	{
+		auto d = m_context->keyManager().accountDetails()[i];
+		u256 b = ethereum()->balanceAt(i, PendingBlock);
+		QString s = QString("%4 %2: %1").arg(formatBalance(b).c_str()).arg(QString::fromStdString(m_context->render(i))).arg(QString::fromStdString(d.first));
+		ui->from->addItem(s);
+	}
 }
 
 bool Transact::isCreation() const
@@ -126,8 +136,8 @@ void Transact::updateFee()
 	ui->total->setText(QString("Total: %1").arg(formatBalance(totalReq).c_str()));
 
 	bool ok = false;
-	for (auto i: m_myKeys)
-		if (ethereum()->balanceAt(i.address()) >= totalReq)
+	for (auto const& i: m_accounts)
+		if (ethereum()->balanceAt(i) >= totalReq)
 		{
 			ok = true;
 			break;
@@ -289,8 +299,9 @@ void Transact::rejigData()
 		return;
 
 	// Determine how much balance we have to play with...
-	auto s = findSecret(value() + ethereum()->gasLimitRemaining() * gasPrice());
-	auto b = ethereum()->balanceAt(KeyPair(s).address(), PendingBlock);
+	//findSecret(value() + ethereum()->gasLimitRemaining() * gasPrice());
+	auto s = fromAccount();
+	auto b = ethereum()->balanceAt(s, PendingBlock);
 
 	m_allGood = true;
 	QString htmlInfo;
@@ -388,22 +399,33 @@ Secret Transact::findSecret(u256 _totalReq) const
 	if (!ethereum())
 		return Secret();
 
-	Secret best;
+	Address best;
 	u256 bestBalance = 0;
-	for (auto const& i: m_myKeys)
+	for (auto const& i: m_accounts)
 	{
-		auto b = ethereum()->balanceAt(i.address(), PendingBlock);
+		auto b = ethereum()->balanceAt(i, PendingBlock);
 		if (b >= _totalReq)
-			return i.secret();
+		{
+			best = i;
+			break;
+		}
 		if (b > bestBalance)
-			bestBalance = b, best = i.secret();
+			bestBalance = b, best = i;
 	}
-	return best;
+	return m_context->retrieveSecret(best);
+}
+
+Address Transact::fromAccount()
+{
+	auto it = m_accounts.begin();
+	std::advance(it, ui->from->currentIndex());
+	return *it;
 }
 
 void Transact::on_send_clicked()
 {
-	Secret s = findSecret(value() + fee());
+//	Secret s = findSecret(value() + fee());
+	Secret s = m_context->retrieveSecret(fromAccount());
 	auto b = ethereum()->balanceAt(KeyPair(s).address(), PendingBlock);
 	if (!s || b < value() + fee())
 	{
@@ -440,9 +462,10 @@ void Transact::on_send_clicked()
 
 void Transact::on_debug_clicked()
 {
-	Secret s = findSecret(value() + fee());
-	auto b = ethereum()->balanceAt(KeyPair(s).address(), PendingBlock);
-	if (!s || b < value() + fee())
+//	Secret s = findSecret(value() + fee());
+	Address from = fromAccount();
+	auto b = ethereum()->balanceAt(from, PendingBlock);
+	if (!from || b < value() + fee())
 	{
 		QMessageBox::critical(this, "Transaction Failed", "Couldn't make transaction: no single account contains at least the required amount.");
 		return;
@@ -452,8 +475,9 @@ void Transact::on_debug_clicked()
 	{
 		State st(ethereum()->postState());
 		Transaction t = isCreation() ?
-			Transaction(value(), gasPrice(), ui->gas->value(), m_data, st.transactionsFrom(dev::toAddress(s)), s) :
-			Transaction(value(), gasPrice(), ui->gas->value(), m_context->fromString(ui->destination->currentText().toStdString()).first, m_data, st.transactionsFrom(dev::toAddress(s)), s);
+			Transaction(value(), gasPrice(), ui->gas->value(), m_data, st.transactionsFrom(from)) :
+			Transaction(value(), gasPrice(), ui->gas->value(), m_context->fromString(ui->destination->currentText().toStdString()).first, m_data, st.transactionsFrom(from));
+		t.forceSender(from);
 		Debugger dw(m_context, this);
 		Executive e(st, ethereum()->blockChain(), 0);
 		dw.populate(e, t);

alethzero/Transact.h

@@ -41,7 +41,7 @@ public:
 	explicit Transact(Context* _context, QWidget* _parent = 0);
 	~Transact();
 
-	void setEnvironment(QList<dev::KeyPair> _myKeys, dev::eth::Client* _eth, NatSpecFace* _natSpecDB);
+	void setEnvironment(dev::AddressHash const& _accounts, dev::eth::Client* _eth, NatSpecFace* _natSpecDB);
 
 private slots:
 	void on_destination_currentTextChanged(QString);
@@ -60,6 +60,7 @@ private:
 	dev::eth::Client* ethereum() const { return m_ethereum; }
 	void rejigData();
 
+	dev::Address fromAccount();
 	void updateDestination();
 	void updateFee();
 	bool isCreation() const;
@@ -76,7 +77,7 @@ private:
 	unsigned m_backupGas = 0;
 	dev::bytes m_data;
 
-	QList<dev::KeyPair> m_myKeys;
+	dev::AddressHash m_accounts;
 	dev::eth::Client* m_ethereum = nullptr;
 	Context* m_context = nullptr;
 	NatSpecFace* m_natSpecDB = nullptr;

alethzero/Transact.ui

@@ -14,91 +14,69 @@
    <string>Transact</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="2" column="1" colspan="2">
-    <widget class="QSpinBox" name="value">
-     <property name="suffix">
-      <string/>
+   <item row="5" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+       <horstretch>0</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
      </property>
-     <property name="maximum">
-      <number>430000000</number>
+     <property name="text">
+      <string>D&amp;ata</string>
      </property>
-     <property name="value">
-      <number>0</number>
+     <property name="alignment">
+      <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+     </property>
+     <property name="buddy">
+      <cstring>data</cstring>
      </property>
     </widget>
    </item>
-   <item row="2" column="0">
-    <widget class="QLabel" name="label5_2">
+   <item row="5" column="3">
+    <widget class="QCheckBox" name="optimize">
      <property name="text">
-      <string>&amp;Amount</string>
+      <string>&amp;Optimise</string>
      </property>
-     <property name="buddy">
-      <cstring>value</cstring>
+     <property name="checked">
+      <bool>true</bool>
      </property>
     </widget>
    </item>
-   <item row="1" column="1" colspan="3">
-    <widget class="QLineEdit" name="calculatedName">
-     <property name="enabled">
-      <bool>false</bool>
+   <item row="4" column="1">
+    <widget class="QSpinBox" name="gas">
+     <property name="suffix">
+      <string> gas</string>
      </property>
-     <property name="readOnly">
-      <bool>true</bool>
+     <property name="minimum">
+      <number>1</number>
      </property>
-     <property name="placeholderText">
-      <string/>
+     <property name="maximum">
+      <number>430000000</number>
+     </property>
+     <property name="value">
+      <number>10000</number>
      </property>
     </widget>
    </item>
-   <item row="5" column="0" colspan="4">
-    <widget class="QSplitter" name="splitter_5">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
+   <item row="8" column="0">
+    <widget class="QPushButton" name="cancel">
+     <property name="text">
+      <string>&amp;Cancel</string>
+     </property>
+     <property name="shortcut">
+      <string>Esc</string>
      </property>
-     <widget class="QPlainTextEdit" name="data">
-      <property name="frameShape">
-       <enum>QFrame::NoFrame</enum>
-      </property>
-      <property name="lineWidth">
-       <number>0</number>
-      </property>
-     </widget>
-     <widget class="QTextEdit" name="code">
-      <property name="focusPolicy">
-       <enum>Qt::ClickFocus</enum>
-      </property>
-      <property name="frameShape">
-       <enum>QFrame::NoFrame</enum>
-      </property>
-      <property name="lineWidth">
-       <number>0</number>
-      </property>
-      <property name="readOnly">
-       <bool>true</bool>
-      </property>
-     </widget>
     </widget>
    </item>
-   <item row="3" column="3">
-    <widget class="QComboBox" name="gasPriceUnits"/>
-   </item>
-   <item row="4" column="1" colspan="2">
-    <widget class="QLabel" name="fee">
-     <property name="sizePolicy">
-      <sizepolicy hsizetype="MinimumExpanding" vsizetype="Preferred">
-       <horstretch>0</horstretch>
-       <verstretch>0</verstretch>
-      </sizepolicy>
-     </property>
+   <item row="8" column="2">
+    <widget class="QPushButton" name="debug">
      <property name="text">
-      <string/>
-     </property>
-     <property name="alignment">
-      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+      <string>&amp;Debug</string>
      </property>
     </widget>
    </item>
-   <item row="0" column="0">
+   <item row="1" column="0">
     <widget class="QLabel" name="label5">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
@@ -114,53 +92,68 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="3">
-    <widget class="QComboBox" name="valueUnits"/>
-   </item>
-   <item row="7" column="2">
-    <widget class="QPushButton" name="debug">
+   <item row="7" column="0" colspan="4">
+    <widget class="QLabel" name="total">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="MinimumExpanding" vsizetype="Preferred">
+       <horstretch>0</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
      <property name="text">
-      <string>&amp;Debug</string>
+      <string/>
      </property>
     </widget>
    </item>
-   <item row="7" column="3">
-    <widget class="QPushButton" name="send">
-     <property name="text">
-      <string>&amp;Execute</string>
+   <item row="3" column="1" colspan="2">
+    <widget class="QSpinBox" name="value">
+     <property name="suffix">
+      <string/>
      </property>
-     <property name="default">
-      <bool>false</bool>
+     <property name="maximum">
+      <number>430000000</number>
+     </property>
+     <property name="value">
+      <number>0</number>
      </property>
     </widget>
    </item>
+   <item row="4" column="3">
+    <widget class="QComboBox" name="gasPriceUnits"/>
+   </item>
    <item row="3" column="0">
-    <widget class="QLabel" name="label_6">
+    <widget class="QLabel" name="label5_2">
      <property name="text">
-      <string>&amp;Gas</string>
+      <string>&amp;Amount</string>
      </property>
      <property name="buddy">
-      <cstring>gas</cstring>
+      <cstring>value</cstring>
      </property>
     </widget>
    </item>
-   <item row="3" column="1">
-    <widget class="QSpinBox" name="gas">
-     <property name="suffix">
-      <string> gas</string>
-     </property>
-     <property name="minimum">
-      <number>1</number>
+   <item row="8" column="3">
+    <widget class="QPushButton" name="send">
+     <property name="text">
+      <string>&amp;Execute</string>
      </property>
-     <property name="maximum">
-      <number>430000000</number>
+     <property name="default">
+      <bool>false</bool>
      </property>
-     <property name="value">
-      <number>10000</number>
+    </widget>
+   </item>
+   <item row="1" column="1" colspan="3">
+    <widget class="QComboBox" name="destination">
+     <property name="editable">
+      <bool>true</bool>
      </property>
+     <item>
+      <property name="text">
+       <string>(Create Contract)</string>
+      </property>
+     </item>
     </widget>
    </item>
-   <item row="3" column="2">
+   <item row="4" column="2">
     <widget class="QSpinBox" name="gasPrice">
      <property name="prefix">
       <string>@ </string>
@@ -173,49 +166,63 @@
      </property>
     </widget>
    </item>
-   <item row="4" column="3">
-    <widget class="QCheckBox" name="optimize">
-     <property name="text">
-      <string>&amp;Optimise</string>
-     </property>
-     <property name="checked">
-      <bool>true</bool>
+   <item row="6" column="0" colspan="4">
+    <widget class="QSplitter" name="splitter_5">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
      </property>
+     <widget class="QPlainTextEdit" name="data">
+      <property name="frameShape">
+       <enum>QFrame::NoFrame</enum>
+      </property>
+      <property name="lineWidth">
+       <number>0</number>
+      </property>
+     </widget>
+     <widget class="QTextEdit" name="code">
+      <property name="focusPolicy">
+       <enum>Qt::ClickFocus</enum>
+      </property>
+      <property name="frameShape">
+       <enum>QFrame::NoFrame</enum>
+      </property>
+      <property name="lineWidth">
+       <number>0</number>
+      </property>
+      <property name="readOnly">
+       <bool>true</bool>
+      </property>
+     </widget>
     </widget>
    </item>
    <item row="4" column="0">
-    <widget class="QLabel" name="label_2">
-     <property name="sizePolicy">
-      <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-       <horstretch>0</horstretch>
-       <verstretch>0</verstretch>
-      </sizepolicy>
-     </property>
+    <widget class="QLabel" name="label_6">
      <property name="text">
-      <string>D&amp;ata</string>
-     </property>
-     <property name="alignment">
-      <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+      <string>&amp;Gas</string>
      </property>
      <property name="buddy">
-      <cstring>data</cstring>
+      <cstring>gas</cstring>
      </property>
     </widget>
    </item>
-   <item row="0" column="1" colspan="3">
-    <widget class="QComboBox" name="destination">
-     <property name="editable">
+   <item row="2" column="1" colspan="3">
+    <widget class="QLineEdit" name="calculatedName">
+     <property name="enabled">
+      <bool>false</bool>
+     </property>
+     <property name="readOnly">
       <bool>true</bool>
      </property>
-     <item>
-      <property name="text">
-       <string>(Create Contract)</string>
-      </property>
-     </item>
+     <property name="placeholderText">
+      <string/>
+     </property>
     </widget>
    </item>
-   <item row="6" column="0" colspan="4">
-    <widget class="QLabel" name="total">
+   <item row="3" column="3">
+    <widget class="QComboBox" name="valueUnits"/>
+   </item>
+   <item row="5" column="1" colspan="2">
+    <widget class="QLabel" name="fee">
      <property name="sizePolicy">
       <sizepolicy hsizetype="MinimumExpanding" vsizetype="Preferred">
        <horstretch>0</horstretch>
@@ -225,18 +232,24 @@
      <property name="text">
       <string/>
      </property>
+     <property name="alignment">
+      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+     </property>
     </widget>
    </item>
-   <item row="7" column="0">
-    <widget class="QPushButton" name="cancel">
+   <item row="0" column="0">
+    <widget class="QLabel" name="label">
      <property name="text">
-      <string>&amp;Cancel</string>
+      <string>&amp;From</string>
      </property>
-     <property name="shortcut">
-      <string>Esc</string>
+     <property name="buddy">
+      <cstring>from</cstring>
      </property>
     </widget>
    </item>
+   <item row="0" column="1" colspan="3">
+    <widget class="QComboBox" name="from"/>
+   </item>
   </layout>
  </widget>
  <resources/>

appdmg.json.in

@@ -0,0 +1,13 @@
+{
+    "title": "Ethereum",
+    "icon": "appdmg_icon.icns",
+    "background": "appdmg_background.png",
+    "icon-size": 80,
+    "contents": [
+        { "x": 600, "y": 170, "type": "link", "path": "/Applications" },
+        { "x": 150, "y": 90,  "type": "file", "path": "${ETH_ALETHZERO_APP}" },
+        { "x": 150, "y": 260, "type": "file", "path": "${ETH_MIX_APP}" }
+    ]
+}
+
+

cmake/EthDependencies.cmake

@@ -159,6 +159,11 @@ if (GUI)
 		message(" - windeployqt path: ${WINDEPLOYQT_APP}")
 	endif()
 
+	if (APPLE)
+		find_program(ETH_APP_DMG appdmg)
+		message(" - appdmg location : ${ETH_APP_DMG}")
+	endif()
+
 	if (USENPM)
 
 		# TODO check node && npm version

cmake/EthExecutableHelper.cmake

@@ -45,25 +45,26 @@ endmacro()
 
 macro(eth_copy_dlls EXECUTABLE DLLS)
 	# dlls must be unsubstitud list variable (without ${}) in format
-	# optimized;path_to_dll.dll;debug;path_to_dlld.dll 
+	# optimized;path_to_dll.dll;debug;path_to_dlld.dll
 	list(GET ${DLLS} 1 DLL_RELEASE)
 	list(GET ${DLLS} 3 DLL_DEBUG)
+	get_target_property(TARGET_RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE} RUNTIME_OUTPUT_DIRECTORY)
 	add_custom_command(TARGET ${EXECUTABLE}
-		POST_BUILD 
-		COMMAND ${CMAKE_COMMAND} ARGS 
-		-DDLL_RELEASE="${DLL_RELEASE}" 
-		-DDLL_DEBUG="${DLL_DEBUG}" 
+		POST_BUILD
+		COMMAND ${CMAKE_COMMAND} ARGS
+		-DDLL_RELEASE="${DLL_RELEASE}"
+		-DDLL_DEBUG="${DLL_DEBUG}"
 		-DCONF="$<CONFIGURATION>"
-		-DDESTINATION="${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}" 
+		-DDESTINATION="${TARGET_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}"
 		-P "${ETH_SCRIPTS_DIR}/copydlls.cmake"
 	)
 endmacro()
 
-# 
+#
 # this function requires the following variables to be specified:
 # ETH_DEPENDENCY_INSTALL_DIR
 #
-# params: 
+# params:
 # QMLDIR
 #
 
@@ -74,7 +75,7 @@ macro(eth_install_executable EXECUTABLE)
 	set (one_value_args QMLDIR)
 	set (multi_value_args DLLS)
 	cmake_parse_arguments (ETH_INSTALL_EXECUTABLE "${options}" "${one_value_args}" "${multi_value_args}" "${extra_macro_args}")
-	
+
 	if (ETH_INSTALL_EXECUTABLE_QMLDIR)
 		if (APPLE)
 			set(eth_qml_dir "-qmldir=${ETH_INSTALL_EXECUTABLE_QMLDIR}")
@@ -87,19 +88,15 @@ macro(eth_install_executable EXECUTABLE)
 	if (APPLE)
 		# First have qt5 install plugins and frameworks
 		add_custom_command(TARGET ${EXECUTABLE} POST_BUILD
-			COMMAND ${MACDEPLOYQT_APP} ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${EXECUTABLE}.app -executable=${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${EXECUTABLE}.app/Contents/MacOS/${EXECUTABLE} ${eth_qml_dir}
-			WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
-			COMMAND sh ${CMAKE_SOURCE_DIR}/macdeployfix.sh ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${EXECUTABLE}.app/Contents
+			COMMAND ${MACDEPLOYQT_APP} ${EXECUTABLE}.app -executable=${EXECUTABLE}.app/Contents/MacOS/${EXECUTABLE} ${eth_qml_dir}
+			WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}
+			COMMAND sh ${CMAKE_SOURCE_DIR}/macdeployfix.sh ${EXECUTABLE}.app/Contents
 		)
-			
-		# This tool and next will inspect linked libraries in order to determine which dependencies are required
-		if (${CMAKE_CFG_INTDIR} STREQUAL ".")
-			# TODO: This should only happen for GUI application
-			set(APP_BUNDLE_PATH "${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE}.app")
-		else ()
-			set(APP_BUNDLE_PATH "${CMAKE_CURRENT_BINARY_DIR}/\$ENV{CONFIGURATION}/${EXECUTABLE}.app")
-		endif ()
 
+		# TODO: This should only happen for GUI application
+		set(APP_BUNDLE_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INDIR}/${EXECUTABLE}.app")
+
+		# This tool and next will inspect linked libraries in order to determine which dependencies are required
 		install(CODE "
 			include(BundleUtilities)
 			set(BU_CHMOD_BUNDLE_ITEMS 1)
@@ -111,14 +108,15 @@ macro(eth_install_executable EXECUTABLE)
 		get_target_property(TARGET_LIBS ${EXECUTABLE} INTERFACE_LINK_LIBRARIES)
 		string(REGEX MATCH "Qt5::Core" HAVE_QT ${TARGET_LIBS})
 		if ("${HAVE_QT}" STREQUAL "Qt5::Core")
+			get_target_property(TARGET_RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE} RUNTIME_OUTPUT_DIRECTORY)
 			add_custom_command(TARGET ${EXECUTABLE} POST_BUILD
-				COMMAND cmd /C "set PATH=${Qt5Core_DIR}/../../../bin;%PATH% && ${WINDEPLOYQT_APP} ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${EXECUTABLE}.exe ${eth_qml_dir}"
+				COMMAND cmd /C "set PATH=${Qt5Core_DIR}/../../../bin;%PATH% && ${WINDEPLOYQT_APP} ${TARGET_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}/${EXECUTABLE}.exe ${eth_qml_dir}"
 				WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
 			)
 			#workaround for https://bugreports.qt.io/browse/QTBUG-42083
 			add_custom_command(TARGET ${EXECUTABLE} POST_BUILD
 				COMMAND cmd /C "(echo [Paths] & echo.Prefix=.)" > "qt.conf"
-				WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR} VERBATIM
+				WORKING_DIRECTORY ${TARGET_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR} VERBATIM
 			)
 		endif()
 
@@ -144,5 +142,3 @@ macro(eth_install_executable EXECUTABLE)
 	endif ()
 
 endmacro()
-
-

cmake/scripts/appdmg.cmake

@@ -0,0 +1,17 @@
+
+if (NOT APP_DMG_EXE)
+	message(FATAL_ERROR "Please install appdmg! https://github.com/LinusU/node-appdmg")
+endif()
+
+string(REPLACE "/Contents/MacOS" "" ETH_MIX_APP "${ETH_MIX_APP}")
+string(REPLACE "/Contents/MacOS" "" ETH_ALETHZERO_APP "${ETH_ALETHZERO_APP}")
+
+set(OUTFILE "${ETH_BUILD_DIR}/appdmg.json")
+
+configure_file(${APP_DMG_FILE} ${OUTFILE})
+
+execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${APP_DMG_ICON}" "${ETH_BUILD_DIR}/appdmg_icon.icns")
+execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${APP_DMG_BACKGROUND}" "${ETH_BUILD_DIR}/appdmg_background.png")
+execute_process(COMMAND ${CMAKE_COMMAND} -E remove "${ETH_BUILD_DIR}/Ethereum.dmg") 
+execute_process(COMMAND ${APP_DMG_EXE} ${OUTFILE} "${ETH_BUILD_DIR}/Ethereum.dmg")
+

cmake/scripts/copydlls.cmake

@@ -14,5 +14,4 @@ else () # Debug
     set(DLL ${DLL_DEBUG})
 endif()
 
-execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${DLL}" "${DESTINATION}")
-
+execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different  "${DLL}" "${DESTINATION}")

eth/main.cpp

@@ -37,6 +37,7 @@
 #include <libevm/VM.h>
 #include <libevm/VMFactory.h>
 #include <libethereum/All.h>
+#include <libethereum/KeyManager.h>
 #include <libwebthree/WebThree.h>
 #if ETH_JSCONSOLE || !ETH_TRUE
 #include <libjsconsole/JSConsole.h>
@@ -46,6 +47,7 @@
 #include <readline/history.h>
 #endif
 #if ETH_JSONRPC || !ETH_TRUE
+#include <libweb3jsonrpc/AccountHolder.h>
 #include <libweb3jsonrpc/WebThreeStubServer.h>
 #include <jsonrpccpp/server/connectors/httpserver.h>
 #include <jsonrpccpp/client/connectors/httpclient.h>
@@ -89,10 +91,8 @@ void interactiveHelp()
 		<< "    minestart  Starts mining." << endl
 		<< "    minestop  Stops mining." << endl
 		<< "    mineforce <enable>  Forces mining, even when there are no transactions." << endl
-		<< "    address  Gives the current address." << endl
-		<< "    secret  Gives the current secret" << endl
 		<< "    block  Gives the current block height." << endl
-		<< "    balance  Gives the current balance." << endl
+		<< "    accounts  Gives information on all owned accounts (balances, mining beneficiary and default signer)." << endl
 		<< "    transact  Execute a given transaction." << endl
 		<< "    send  Execute a given transaction with current secret." << endl
 		<< "    contract  Create a new contract with current secret." << endl
@@ -101,7 +101,7 @@ void interactiveHelp()
 		<< "    listaccounts  List the accounts on the network." << endl
 		<< "    listcontracts  List the contracts on the network." << endl
 #endif
-		<< "    setsecret <secret>  Set the secret to the hex secret key." << endl
+		<< "    setsigningkey <addr>  Set the address with which to sign transactions." << endl
 		<< "    setaddress <addr>  Set the coinbase (mining payout) address." << endl
 		<< "    exportconfig <path>  Export the config (.RLP) to the path provided." << endl
 		<< "    importconfig <path>  Import the config (.RLP) from the path provided." << endl
@@ -125,13 +125,18 @@ void help()
 #endif
 		<< "    -K,--kill  First kill the blockchain." << endl
 		<< "    -R,--rebuild  Rebuild the blockchain from the existing database." << endl
-		<< "    -s,--secret <secretkeyhex>  Set the secret key for use with send command (default: auto)." << endl
-		<< "    -S,--session-secret <secretkeyhex>  Set the secret key for use with send command, for this session only." << endl
+		<< "    -s,--import-secret <secret>  Import a secret key into the key store and use as the default." << endl
+		<< "    -S,--import-session-secret <secret>  Import a secret key into the key store and use as the default for this session only." << endl
+		<< "    --sign-key <address>  Sign all transactions with the key of the given address." << endl
+		<< "    --session-sign-key <address>  Sign all transactions with the key of the given address for this session only." << endl
 		<< "    --master <password>  Give the master password for the key store." << endl
+		<< "    --password <password>  Give a password for a private key." << endl
+		<< endl
 		<< "Client transacting:" << endl
 		<< "    -B,--block-fees <n>  Set the block fee profit in the reference unit e.g. ¢ (default: 15)." << endl
 		<< "    -e,--ether-price <n>  Set the ether price in the reference unit e.g. ¢ (default: 30.679)." << endl
 		<< "    -P,--priority <0 - 100>  Default % priority of a transaction (default: 50)." << endl
+		<< endl
 		<< "Client mining:" << endl
 		<< "    -a,--address <addr>  Set the coinbase (mining payout) address to addr (default: auto)." << endl
 		<< "    -m,--mining <on/off/number>  Enable mining, optionally for a specified number of blocks (default: off)" << endl
@@ -141,6 +146,7 @@ void help()
 		<< "    --opencl-platform <n>  When mining using -G/--opencl use OpenCL platform n (default: 0)." << endl
 		<< "    --opencl-device <n>  When mining using -G/--opencl use OpenCL device n (default: 0)." << endl
 		<< "    -t, --mining-threads <n> Limit number of CPU/GPU miners to n (default: use everything available on selected platform)" << endl
+		<< endl
 		<< "Client networking:" << endl
 		<< "    --client-name <name>  Add a name to your client's version string (default: blank)." << endl
 		<< "    -b,--bootstrap  Connect to the default Ethereum peerserver." << endl
@@ -152,13 +158,21 @@ void help()
 		<< "    --port <port>  Connect to remote port (default: 30303)." << endl
 		<< "    --network-id <n> Only connect to other hosts with this network id (default:0)." << endl
 		<< "    --upnp <on/off>  Use UPnP for NAT (default: on)." << endl
+		<< endl
+		<< "Client structured logging:" << endl
+		<< "    --structured-logging  Enable structured logging (default output to stdout)." << endl
+		<< "    --structured-logging-format <format>  Set the structured logging time format." << endl
+		<< "    --structured-logging-url <URL>  Set the structured logging destination (currently only file:// supported)." << endl
 #if ETH_JSONRPC || !ETH_TRUE
+		<< endl
 		<< "Work farming mode:" << endl
 		<< "    -F,--farm <url>  Put into mining farm mode with the work server at URL. Use with -G/--opencl." << endl
 		<< "    --farm-recheck <n>  Leave n ms between checks for changed work (default: 500)." << endl
 #endif
+		<< endl
 		<< "Ethash verify mode:" << endl
 		<< "    -w,--check-pow <headerHash> <seedHash> <difficulty> <nonce>  Check PoW credentials for validity." << endl
+		<< endl
 		<< "Benchmarking mode:" << endl
 		<< "    -M,--benchmark  Benchmark for mining and exit; use with --cpu and --opencl." << endl
 		<< "    --benchmark-warmup <seconds>  Set the duration of warmup for the benchmark tests (default: 3)." << endl
@@ -167,14 +181,17 @@ void help()
 #if ETH_JSONRPC || !ETH_TRUE
 		<< "    --phone-home <on/off>  When benchmarking, publish results (default: on)" << endl
 #endif
+		<< endl
 		<< "DAG creation mode:" << endl
 		<< "    -D,--create-dag <this/next/number>  Create the DAG in preparation for mining on given block and exit." << endl
+		<< endl
 		<< "Import/export modes:" << endl
 		<< "    -I,--import <file>  Import file as a concatenated series of blocks and exit." << endl
 		<< "    -E,--export <file>  Export file as a concatenated series of blocks and exit." << endl
 		<< "    --from <n>  Export only from block n; n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl
 		<< "    --to <n>  Export only to block n (inclusive); n may be a decimal, a '0x' prefixed hash, or 'latest'." << endl
 		<< "    --only <n>  Equivalent to --export-from n --export-to n." << endl
+		<< endl
 		<< "General Options:" << endl
 		<< "    -d,--db-path <path>  Load database from path (default: " << getDataDir() << ")" << endl
 #if ETH_EVMJIT || !ETH_TRUE
@@ -537,13 +554,14 @@ int main(int argc, char** argv)
 	/// Mining params
 	unsigned mining = 0;
 	bool forceMining = false;
-	KeyPair sigKey = KeyPair::create();
-	Secret sessionSecret;
-	Address coinbase = sigKey.address();
+	Address signingKey;
+	Address sessionKey;
+	Address beneficiary = signingKey;
 
 	/// Structured logging params
 	bool structuredLogging = false;
 	string structuredLoggingFormat = "%Y-%m-%dT%H:%M:%S";
+	string structuredLoggingURL;
 
 	/// Transaction params
 	TransactionPriority priority = TransactionPriority::Medium;
@@ -568,11 +586,20 @@ int main(int argc, char** argv)
 
 	string configFile = getDataDir() + "/config.rlp";
 	bytes b = contents(configFile);
+
+	strings passwordsToNote;
+	Secrets toImport;
 	if (b.size())
 	{
 		RLP config(b);
-		sigKey = KeyPair(config[0].toHash<Secret>());
-		coinbase = config[1].toHash<Address>();
+		if (config[0].size() == 32)	// secret key - import and forget.
+		{
+			Secret s = config[0].toHash<Secret>();
+			toImport.push_back(s);
+		}
+		else							// new format - just use it as an address.
+			signingKey = config[0].toHash<Address>();
+		beneficiary = config[1].toHash<Address>();
 	}
 
 	for (int i = 1; i < argc; ++i)
@@ -600,6 +627,8 @@ int main(int argc, char** argv)
 				cerr << "-p is DEPRECATED. It will be removed for the Frontier. Use --port instead (or place directly as host:port)." << endl;
 			remotePort = (short)atoi(argv[++i]);
 		}
+		else if (arg == "--password" && i + 1 < argc)
+			passwordsToNote.push_back(argv[++i]);
 		else if (arg == "--master" && i + 1 < argc)
 			masterPassword = argv[++i];
 		else if ((arg == "-I" || arg == "--import") && i + 1 < argc)
@@ -742,7 +771,7 @@ int main(int argc, char** argv)
 		}
 		else if ((arg == "-a" || arg == "--address" || arg == "--coinbase-address") && i + 1 < argc)
 			try {
-				coinbase = h160(fromHex(argv[++i], WhenError::Throw));
+				beneficiary = h160(fromHex(argv[++i], WhenError::Throw));
 			}
 			catch (BadHexCharacter&)
 			{
@@ -758,14 +787,35 @@ int main(int argc, char** argv)
 			minerType = MinerType::CPU;
 		else if (arg == "-G" || arg == "--opencl")
 			minerType = MinerType::GPU;
-		else if ((arg == "-s" || arg == "--secret") && i + 1 < argc)
-			sigKey = KeyPair(h256(fromHex(argv[++i])));
-		else if ((arg == "-S" || arg == "--session-secret") && i + 1 < argc)
-			sessionSecret = h256(fromHex(argv[++i]));
+		/*<< "    -s,--import-secret <secret>  Import a secret key into the key store and use as the default." << endl
+		<< "    -S,--import-session-secret <secret>  Import a secret key into the key store and use as the default for this session only." << endl
+		<< "    --sign-key <address>  Sign all transactions with the key of the given address." << endl
+		<< "    --session-sign-key <address>  Sign all transactions with the key of the given address for this session only." << endl*/
+		else if ((arg == "-s" || arg == "--import-secret") && i + 1 < argc)
+		{
+			Secret s(fromHex(argv[++i]));
+			toImport.push_back(s);
+			signingKey = toAddress(s);
+		}
+		else if ((arg == "-S" || arg == "--import-session-secret") && i + 1 < argc)
+		{
+			Secret s(fromHex(argv[++i]));
+			toImport.push_back(s);
+			sessionKey = toAddress(s);
+		}
+		else if ((arg == "--sign-key") && i + 1 < argc)
+			sessionKey = Address(fromHex(argv[++i]));
+		else if ((arg == "--session-sign-key") && i + 1 < argc)
+			sessionKey = Address(fromHex(argv[++i]));
 		else if (arg == "--structured-logging-format" && i + 1 < argc)
 			structuredLoggingFormat = string(argv[++i]);
 		else if (arg == "--structured-logging")
 			structuredLogging = true;
+		else if (arg == "--structured-logging-url" && i + 1 < argc)
+		{
+			structuredLogging = true;
+			structuredLoggingURL = argv[++i];
+		}
 		else if ((arg == "-d" || arg == "--path" || arg == "--db-path") && i + 1 < argc)
 			dbPath = argv[++i];
 		else if ((arg == "-D" || arg == "--create-dag") && i + 1 < argc)
@@ -806,7 +856,7 @@ int main(int argc, char** argv)
 				auto boundary = bi.boundary();
 				m = boost::to_lower_copy(string(argv[++i]));
 				bi.nonce = h64(m);
-				auto r = EthashAux::eval(seedHash, powHash, bi.nonce);
+				auto r = EthashAux::eval((uint64_t)bi.number, powHash, bi.nonce);
 				bool valid = r.value < boundary;
 				cout << (valid ? "VALID :-)" : "INVALID :-(") << endl;
 				cout << r.value << (valid ? " < " : " >= ") << boundary << endl;
@@ -815,7 +865,7 @@ int main(int argc, char** argv)
 				cout << "  with seed as " << seedHash << endl;
 				if (valid)
 					cout << "(mixHash = " << r.mixHash << ")" << endl;
-				cout << "SHA3( light(seed) ) = " << sha3(EthashAux::light(seedHash)->data()) << endl;
+				cout << "SHA3( light(seed) ) = " << sha3(EthashAux::light((uint64_t)bi.number)->data()) << endl;
 				exit(0);
 			}
 			catch (...)
@@ -949,16 +999,24 @@ int main(int argc, char** argv)
 		}
 	}
 
+	KeyManager keyManager;
+	for (auto const& s: passwordsToNote)
+		keyManager.notePassword(s);
+	for (auto const& s: toImport)
+	{
+		keyManager.import(s, "Imported key");
+		if (!signingKey)
+			signingKey = toAddress(s);
+	}
+
 	{
 		RLPStream config(2);
-		config << sigKey.secret() << coinbase;
+		config << signingKey << beneficiary;
 		writeFile(configFile, config.out());
 	}
 
-	if (sessionSecret)
-		sigKey = KeyPair(sessionSecret);
-
-	
+	if (sessionKey)
+		signingKey = sessionKey;
 
 	if (minerType == MinerType::CPU)
 		ProofOfWork::CPUMiner::setNumInstances(miningThreads);
@@ -983,7 +1041,29 @@ int main(int argc, char** argv)
 	if (!clientName.empty())
 		clientName += "/";
 
-	StructuredLogger::get().initialize(structuredLogging, structuredLoggingFormat);
+	string logbuf;
+	bool silence = false;
+	std::string additional;
+	g_logPost = [&](std::string const& a, char const*){
+		if (silence)
+			logbuf += a + "\n";
+		else
+			cout << "\r           \r" << a << endl << additional << flush;
+	};
+
+	auto getPassword = [&](string const& prompt){
+		auto s = silence;
+		silence = true;
+		cout << endl;
+		string ret = dev::getPassword(prompt);
+		silence = s;
+		return ret;
+	};
+	auto getAccountPassword = [&](Address const& a){
+		return getPassword("Enter password for address " + keyManager.accountDetails()[a].first + " (" + a.abridged() + "; hint:" + keyManager.accountDetails()[a].second + "): ");
+	};
+
+	StructuredLogger::get().initialize(structuredLogging, structuredLoggingFormat, structuredLoggingURL);
 	VMFactory::setKind(jit ? VMKind::JIT : VMKind::Interpreter);
 	auto netPrefs = publicIP.empty() ? NetworkPreferences(listenIP ,listenPort, upnp) : NetworkPreferences(publicIP, listenIP ,listenPort, upnp);
 	auto nodesState = contents((dbPath.size() ? dbPath : getDataDir()) + "/network.rlp");
@@ -992,13 +1072,38 @@ int main(int argc, char** argv)
 		clientImplString,
 		dbPath,
 		killChain,
-		nodeMode == NodeMode::Full ? set<string>{"eth", "shh"} : set<string>(),
+		nodeMode == NodeMode::Full ? set<string>{"eth"/*, "shh"*/} : set<string>(),
 		netPrefs,
 		&nodesState);
 	
 	if (mode == OperationMode::DAGInit)
 		doInitDAG(web3.ethereum()->blockChain().number() + (initDAG == PendingBlock ? 30000 : 0));
 
+	if (keyManager.exists())
+		while (masterPassword.empty())
+		{
+			masterPassword = getPassword("Please enter your MASTER password: ");
+			if (!keyManager.load(masterPassword))
+			{
+				cout << "Password invalid. Try again." << endl;
+				masterPassword.clear();
+			}
+		}
+	else
+	{
+		while (masterPassword.empty())
+		{
+			masterPassword = getPassword("Please enter a MASTER password to protect your key store (make it strong!): ");
+			string confirm = getPassword("Please confirm the password by entering it again: ");
+			if (masterPassword != confirm)
+			{
+				cout << "Passwords were different. Try again." << endl;
+				masterPassword.clear();
+			}
+		}
+		keyManager.create(masterPassword);
+	}
+
 	auto toNumber = [&](string const& s) -> unsigned {
 		if (s == "latest")
 			return web3.ethereum()->number();
@@ -1072,27 +1177,28 @@ int main(int argc, char** argv)
 		c->setGasPricer(gasPricer);
 		c->setForceMining(forceMining);
 		c->setTurboMining(minerType == MinerType::GPU);
-		c->setAddress(coinbase);
+		c->setAddress(beneficiary);
 		c->setNetworkId(networkId);
 	}
 
-	cout << "Transaction Signer: " << sigKey.address() << endl;
-	cout << "Mining Benefactor: " << coinbase << endl;
+	cout << "Transaction Signer: " << signingKey << endl;
+	cout << "Mining Benefactor: " << beneficiary << endl;
 	web3.startNetwork();
 	cout << "Node ID: " << web3.enode() << endl;
 
 	if (bootstrap)
-		web3.addNode(p2p::NodeId(), Host::pocHost());
+		for (auto const& i: Host::pocHosts())
+			web3.requirePeer(i.first, i.second);
 	if (remoteHost.size())
 		web3.addNode(p2p::NodeId(), remoteHost + ":" + toString(remotePort));
 
-#if ETH_JSONRPC
+#if ETH_JSONRPC || !ETH_TRUE
 	shared_ptr<WebThreeStubServer> jsonrpcServer;
 	unique_ptr<jsonrpc::AbstractServerConnector> jsonrpcConnector;
 	if (jsonrpc > -1)
 	{
 		jsonrpcConnector = unique_ptr<jsonrpc::AbstractServerConnector>(new jsonrpc::HttpServer(jsonrpc, "", "", SensibleHttpThreads));
-		jsonrpcServer = shared_ptr<WebThreeStubServer>(new WebThreeStubServer(*jsonrpcConnector.get(), web3, vector<KeyPair>({sigKey})));
+		jsonrpcServer = shared_ptr<WebThreeStubServer>(new WebThreeStubServer(*jsonrpcConnector.get(), web3, make_shared<SimpleAccountHolder>([&](){return web3.ethereum();}, getAccountPassword, keyManager), vector<KeyPair>()));
 		jsonrpcServer->StartListening();
 	}
 #endif
@@ -1103,15 +1209,15 @@ int main(int argc, char** argv)
 
 	if (interactive)
 	{
-		string logbuf;
+		additional = "Press Enter";
 		string l;
 		while (!g_exit)
 		{
-			g_logPost = [](std::string const& a, char const*) { cout << "\r           \r" << a << endl << "Press Enter" << flush; };
+			silence = false;
 			cout << logbuf << "Press Enter" << flush;
 			std::getline(cin, l);
 			logbuf.clear();
-			g_logPost = [&](std::string const& a, char const*) { logbuf += a + "\n"; };
+			silence = true;
 
 #if ETH_READLINE
 			if (l.size())
@@ -1224,7 +1330,7 @@ int main(int argc, char** argv)
 					iss >> g_logVerbosity;
 				cout << "Verbosity: " << g_logVerbosity << endl;
 			}
-#if ETH_JSONRPC
+#if ETH_JSONRPC || !ETH_TRUE
 			else if (cmd == "jsonport")
 			{
 				if (iss.peek() != -1)
@@ -1236,7 +1342,7 @@ int main(int argc, char** argv)
 				if (jsonrpc < 0)
 					jsonrpc = SensibleHttpPort;
 				jsonrpcConnector = unique_ptr<jsonrpc::AbstractServerConnector>(new jsonrpc::HttpServer(jsonrpc, "", "", SensibleHttpThreads));
-				jsonrpcServer = shared_ptr<WebThreeStubServer>(new WebThreeStubServer(*jsonrpcConnector.get(), web3, vector<KeyPair>({sigKey})));
+				jsonrpcServer = shared_ptr<WebThreeStubServer>(new WebThreeStubServer(*jsonrpcConnector.get(), web3, make_shared<SimpleAccountHolder>([&](){return web3.ethereum();}, getAccountPassword, keyManager), vector<KeyPair>()));
 				jsonrpcServer->StartListening();
 			}
 			else if (cmd == "jsonstop")
@@ -1248,11 +1354,8 @@ int main(int argc, char** argv)
 #endif
 			else if (cmd == "address")
 			{
-				cout << "Current address:" << endl << sigKey.address() << endl;
-			}
-			else if (cmd == "secret")
-			{
-				cout << "Secret Key: " << sigKey.secret() << endl;
+				cout << "Current mining beneficiary:" << endl << beneficiary << endl;
+				cout << "Current signing account:" << endl << signingKey << endl;
 			}
 			else if (c && cmd == "block")
 			{
@@ -1267,7 +1370,15 @@ int main(int argc, char** argv)
 			}
 			else if (c && cmd == "balance")
 			{
-				cout << "Current balance: " << formatBalance( c->balanceAt(sigKey.address())) << " = " <<c->balanceAt(sigKey.address()) << " wei" << endl;
+				cout << "Current balance:" << endl;
+				u256 total = 0;
+				for (auto const& i: keyManager.accountDetails())
+				{
+					auto b = c->balanceAt(i.first);
+					cout << ((i.first == signingKey) ? "SIGNING " : "        ") << ((i.first == beneficiary) ? "COINBASE " : "         ") << i.second.first << " (" << i.first << "): " << formatBalance(b) << " = " << b << " wei" << endl;
+					total += b;
+				}
+				cout << "Total: " << formatBalance(total) << " = " << total << " wei" << endl;
 			}
 			else if (c && cmd == "transact")
 			{
@@ -1383,7 +1494,7 @@ int main(int argc, char** argv)
 						try
 						{
 							Address dest = h160(fromHex(hexAddr, WhenError::Throw));
-							c->submitTransaction(sigKey.secret(), amount, dest, bytes(), minGas);
+							c->submitTransaction(keyManager.secret(signingKey, [&](){ return getAccountPassword(signingKey); }), amount, dest, bytes(), minGas);
 						}
 						catch (BadHexCharacter& _e)
 						{
@@ -1452,7 +1563,7 @@ int main(int argc, char** argv)
 					else if (gas < minGas)
 						cwarn << "Minimum gas amount is" << minGas;
 					else
-						c->submitTransaction(sigKey.secret(), endowment, init, gas, gasPrice);
+						c->submitTransaction(keyManager.secret(signingKey, [&](){ return getAccountPassword(signingKey); }), endowment, init, gas, gasPrice);
 				}
 				else
 					cwarn << "Require parameters: contract ENDOWMENT GASPRICE GAS CODEHEX";
@@ -1563,13 +1674,13 @@ int main(int argc, char** argv)
 					}
 				}
 			}
-			else if (cmd == "setsecret")
+			else if (cmd == "setsigningkey")
 			{
 				if (iss.peek() != -1)
 				{
 					string hexSec;
 					iss >> hexSec;
-					sigKey = KeyPair(h256(fromHex(hexSec)));
+					signingKey = Address(fromHex(hexSec));
 				}
 				else
 					cwarn << "Require parameter: setSecret HEXSECRETKEY";
@@ -1586,7 +1697,7 @@ int main(int argc, char** argv)
 					{
 						try
 						{
-							coinbase = h160(fromHex(hexAddr, WhenError::Throw));
+							beneficiary = h160(fromHex(hexAddr, WhenError::Throw));
 						}
 						catch (BadHexCharacter& _e)
 						{
@@ -1609,7 +1720,7 @@ int main(int argc, char** argv)
 					string path;
 					iss >> path;
 					RLPStream config(2);
-					config << sigKey.secret() << coinbase;
+					config << signingKey << beneficiary;
 					writeFile(path, config.out());
 				}
 				else
@@ -1625,8 +1736,8 @@ int main(int argc, char** argv)
 					if (b.size())
 					{
 						RLP config(b);
-						sigKey = KeyPair(config[0].toHash<Secret>());
-						coinbase = config[1].toHash<Address>();
+						signingKey = config[0].toHash<Address>();
+						beneficiary = config[1].toHash<Address>();
 					}
 					else
 						cwarn << path << "has no content!";

ethminer/main.cpp

@@ -417,7 +417,7 @@ int main(int argc, char** argv)
 				auto boundary = bi.boundary();
 				m = boost::to_lower_copy(string(argv[++i]));
 				bi.nonce = h64(m);
-				auto r = EthashAux::eval(seedHash, powHash, bi.nonce);
+				auto r = EthashAux::eval((uint64_t)bi.number, powHash, bi.nonce);
 				bool valid = r.value < boundary;
 				cout << (valid ? "VALID :-)" : "INVALID :-(") << endl;
 				cout << r.value << (valid ? " < " : " >= ") << boundary << endl;
@@ -426,7 +426,7 @@ int main(int argc, char** argv)
 				cout << "  with seed as " << seedHash << endl;
 				if (valid)
 					cout << "(mixHash = " << r.mixHash << ")" << endl;
-				cout << "SHA3( light(seed) ) = " << sha3(EthashAux::light(seedHash)->data()) << endl;
+				cout << "SHA3( light(seed) ) = " << sha3(EthashAux::light((uint64_t)bi.number)->data()) << endl;
 				exit(0);
 			}
 			catch (...)

evmjit/CMakeLists.txt

@@ -33,6 +33,8 @@ else()
 	link_directories(/usr/lib/llvm-3.5/lib)
 endif()
 
+get_filename_component(EVMJIT_INCLUDE_DIR include ABSOLUTE)
+
 add_subdirectory(libevmjit)
 
 if(EVMJIT_CPP)

evmjit/include/evmjit/DataTypes.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <cstdint>
+#include <functional>
+
+namespace dev
+{
+namespace evmjit
+{
+
+struct h256
+{
+	uint64_t words[4];
+};
+
+inline bool operator==(h256 _h1, h256 _h2)
+{
+	return 	_h1.words[0] == _h2.words[0] &&
+			_h1.words[1] == _h2.words[1] &&
+			_h1.words[2] == _h2.words[2] &&
+			_h1.words[3] == _h2.words[3];
+}
+
+/// Representation of 256-bit value binary compatible with LLVM i256
+struct i256
+{
+	uint64_t a = 0;
+	uint64_t b = 0;
+	uint64_t c = 0;
+	uint64_t d = 0;
+
+	i256() = default;
+	i256(h256 _h)
+	{
+		a = _h.words[0];
+		b = _h.words[1];
+		c = _h.words[2];
+		d = _h.words[3];
+	}
+};
+
+}
+}
+
+namespace std
+{
+template<> struct hash<dev::evmjit::h256>
+{
+	size_t operator()(dev::evmjit::h256 const& _h) const
+	{
+		/// This implementation expects the argument to be a full 256-bit Keccak hash.
+		/// It does nothing more than returning a slice of the input hash.
+		return static_cast<size_t>(_h.words[0]);
+	};
+};
+}

evmjit/include/evmjit/JIT.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "evmjit/DataTypes.h"
+
+namespace dev
+{
+namespace eth
+{
+namespace jit
+{
+	class ExecutionEngine;
+}
+}
+
+namespace evmjit
+{
+
+class JIT
+{
+public:
+
+	/// Ask JIT if the EVM code is ready for execution.
+	/// Returns `true` if the EVM code has been compiled and loaded into memory.
+	/// In this case the code can be executed without overhead.
+	/// \param _codeHash	The Keccak hash of the EVM code.
+	static bool isCodeReady(h256 _codeHash);
+
+private:
+	friend class dev::eth::jit::ExecutionEngine;
+
+	static void* getCode(h256 _codeHash);
+	static void mapCode(h256 _codeHash, void* _funcAddr);
+};
+
+}
+}

evmjit/libevmjit-cpp/CMakeLists.txt

@@ -19,6 +19,7 @@ add_library(${TARGET_NAME} STATIC ${SOURCES})
 set_property(TARGET ${TARGET_NAME} PROPERTY FOLDER "libs")
 
 include_directories(../..)
+include_directories(${EVMJIT_INCLUDE_DIR})
 include_directories(${LLVM_INCLUDE_DIRS})
 include_directories(${Boost_INCLUDE_DIRS})
 

evmjit/libevmjit-cpp/Env.cpp

@@ -3,6 +3,7 @@
 #include <libdevcrypto/SHA3.h>
 #include <libevmcore/Params.h>
 #include <libevm/ExtVMFace.h>
+#include <evmjit/DataTypes.h>
 
 #include "Utils.h"
 
@@ -16,7 +17,7 @@ extern "C"
 
 	using namespace dev;
 	using namespace dev::eth;
-	using jit::i256;
+	using evmjit::i256;
 
 	EXPORT void env_sload(ExtVMFace* _env, i256* _index, i256* o_value)
 	{

evmjit/libevmjit-cpp/JitVM.cpp

@@ -32,9 +32,7 @@ bytesConstRef JitVM::go(ExtVMFace& _ext, OnOpFunc const& _onOp, uint64_t _step)
 	if (rejected)
 	{
 		cwarn << "Execution rejected by EVM JIT (gas limit: " << m_gas << "), executing with interpreter";
-		VMFactory::setKind(VMKind::Interpreter);
-		m_fallbackVM = VMFactory::create(m_gas);
-		VMFactory::setKind(VMKind::JIT);
+		m_fallbackVM = VMFactory::create(VMKind::Interpreter, m_gas);
 		auto&& output = m_fallbackVM->go(_ext, _onOp, _step);
 		m_gas = m_fallbackVM->gas(); // copy remaining gas, Executive expects it
 		return output;

evmjit/libevmjit-cpp/Utils.h

@@ -1,13 +1,13 @@
 #pragma once
 
-#include <evmjit/libevmjit/Common.h>
+#include <evmjit/DataTypes.h>
 
 namespace dev
 {
 namespace eth
 {
 
-inline u256 llvm2eth(jit::i256 _i)
+inline u256 llvm2eth(evmjit::i256 _i)
 {
 	u256 u = 0;
 	u |= _i.d;
@@ -20,9 +20,9 @@ inline u256 llvm2eth(jit::i256 _i)
 	return u;
 }
 
-inline jit::i256 eth2llvm(u256 _u)
+inline evmjit::i256 eth2llvm(u256 _u)
 {
-	jit::i256 i;
+	evmjit::i256 i;
 	u256 mask = 0xFFFFFFFFFFFFFFFF;
 	i.a = static_cast<uint64_t>(_u & mask);
 	_u >>= 64;
@@ -34,5 +34,11 @@ inline jit::i256 eth2llvm(u256 _u)
 	return i;
 }
 
+inline evmjit::h256 eth2llvm(h256 _u)
+{
+	/// Just directly copies memory
+	return *(evmjit::h256*)&_u;
+}
+
 }
 }

evmjit/libevmjit/CMakeLists.txt

@@ -8,6 +8,7 @@ set(SOURCES
     Common.h
     Compiler.cpp		Compiler.h
     CompilerHelper.cpp	CompilerHelper.h
+    ${EVMJIT_INCLUDE_DIR}/evmjit/DataTypes.h
     Endianness.cpp		Endianness.h
     ExecStats.cpp		ExecStats.h
     ExecutionEngine.cpp	ExecutionEngine.h
@@ -15,6 +16,7 @@ set(SOURCES
     GasMeter.cpp		GasMeter.h
     Instruction.cpp		Instruction.h
     interface.cpp		interface.h
+    JIT.cpp             ${EVMJIT_INCLUDE_DIR}/evmjit/JIT.h
     Memory.cpp			Memory.h
     Optimizer.cpp		Optimizer.h
     Runtime.cpp			Runtime.h
@@ -79,6 +81,7 @@ set_target_properties(${TARGET_NAME} PROPERTIES
 						VERSION ${EVMJIT_VERSION} SOVERSION ${EVMJIT_SOVERSION}
 						FOLDER "libs")
 
+include_directories(${EVMJIT_INCLUDE_DIR})
 include_directories(${LLVM_INCLUDE_DIRS})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/gen)
 

evmjit/libevmjit/Common.h

@@ -31,7 +31,7 @@ enum class ReturnCode
 
 	// Standard error codes
 	OutOfGas           = -1,
-	StackUnderflow      = -2,
+	StackUnderflow     = -2,
 	BadJumpDestination = -3,
 	BadInstruction     = -4,
 	Rejected           = -5, ///< Input data (code, gas, block info, etc.) does not meet JIT requirement and execution request has been rejected
@@ -46,16 +46,6 @@ enum class ReturnCode
 	LinkerWorkaround = -299,
 };
 
-/// Representation of 256-bit value binary compatible with LLVM i256
-struct i256
-{
-	uint64_t a = 0;
-	uint64_t b = 0;
-	uint64_t c = 0;
-	uint64_t d = 0;
-};
-static_assert(sizeof(i256) == 32, "Wrong i265 size");
-
 #define UNTESTED assert(false)
 
 }

evmjit/libevmjit/ExecutionEngine.cpp

@@ -19,6 +19,7 @@
 #include <llvm/Support/ManagedStatic.h>
 #include "preprocessor/llvm_includes_end.h"
 
+#include "evmjit/JIT.h"
 #include "Runtime.h"
 #include "Compiler.h"
 #include "Optimizer.h"
@@ -33,6 +34,7 @@ namespace eth
 {
 namespace jit
 {
+using evmjit::JIT;
 
 namespace
 {
@@ -119,8 +121,6 @@ ReturnCode ExecutionEngine::run(RuntimeData* _data, Env* _env)
 	// TODO: Do not pseudo-init the cache every time
 	auto objectCache = (g_cache != CacheMode::off && g_cache != CacheMode::clear) ? Cache::getObjectCache(g_cache, listener.get()) : nullptr;
 
-	static std::unordered_map<std::string, uint64_t> funcCache;
-
 	static std::unique_ptr<llvm::ExecutionEngine> ee;
 	if (!ee)
 	{
@@ -147,8 +147,9 @@ ReturnCode ExecutionEngine::run(RuntimeData* _data, Env* _env)
 		module.release();  // Successfully created llvm::ExecutionEngine takes ownership of the module
 		ee->setObjectCache(objectCache);
 
-		if (preloadCache)
-			Cache::preload(*ee, funcCache);
+		// FIXME: Disabled during API changes
+		//if (preloadCache)
+		//	Cache::preload(*ee, funcCache);
 	}
 
 	static StatsCollector statsCollector;
@@ -156,11 +157,8 @@ ReturnCode ExecutionEngine::run(RuntimeData* _data, Env* _env)
 	auto mainFuncName = codeHash(_data->codeHash);
 	m_runtime.init(_data, _env);
 
-	EntryFuncPtr entryFuncPtr = nullptr;
-	auto it = funcCache.find(mainFuncName);
-	if (it != funcCache.end())
-		entryFuncPtr = (EntryFuncPtr) it->second;
-
+	// TODO: Remove cast
+	auto entryFuncPtr = (EntryFuncPtr) JIT::getCode(_data->codeHash);
 	if (!entryFuncPtr)
 	{
 		auto module = objectCache ? Cache::getObject(mainFuncName) : nullptr;
@@ -183,12 +181,10 @@ ReturnCode ExecutionEngine::run(RuntimeData* _data, Env* _env)
 		module.release();
 		listener->stateChanged(ExecState::CodeGen);
 		entryFuncPtr = (EntryFuncPtr)ee->getFunctionAddress(mainFuncName);
+		if (!CHECK(entryFuncPtr))
+			return ReturnCode::LLVMLinkError;
+		JIT::mapCode(_data->codeHash, (void*)entryFuncPtr); // FIXME: Remove cast
 	}
-	if (!CHECK(entryFuncPtr))
-		return ReturnCode::LLVMLinkError;
-
-	if (it == funcCache.end())
-		funcCache[mainFuncName] = (uint64_t) entryFuncPtr;
 
 	listener->stateChanged(ExecState::Execution);
 	auto returnCode = entryFuncPtr(&m_runtime);

evmjit/libevmjit/JIT.cpp

@@ -0,0 +1,46 @@
+#include "evmjit/JIT.h"
+
+#include <unordered_map>
+
+namespace dev
+{
+namespace evmjit
+{
+namespace
+{
+
+class JITImpl: JIT
+{
+public:
+	std::unordered_map<h256, void*> codeMap;
+
+	static JITImpl& instance()
+	{
+		static JITImpl s_instance;
+		return s_instance;
+	}
+};
+
+} // anonymous namespace
+
+bool JIT::isCodeReady(h256 _codeHash)
+{
+	return JITImpl::instance().codeMap.count(_codeHash) != 0;
+}
+
+void* JIT::getCode(h256 _codeHash)
+{
+	auto& codeMap = JITImpl::instance().codeMap;
+	auto it = codeMap.find(_codeHash);
+	if (it != codeMap.end())
+		return it->second;
+	return nullptr;
+}
+
+void JIT::mapCode(h256 _codeHash, void* _funcAddr)
+{
+	JITImpl::instance().codeMap.insert(std::make_pair(_codeHash, _funcAddr));
+}
+
+}
+}

evmjit/libevmjit/RuntimeData.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "evmjit/DataTypes.h"
 #include "Common.h"
 
 namespace dev
@@ -8,7 +9,9 @@ namespace eth
 {
 namespace jit
 {
-	
+using evmjit::i256;
+using evmjit::h256;
+
 struct RuntimeData
 {
 	enum Index
@@ -49,7 +52,7 @@ struct RuntimeData
 	int64_t 	timestamp = 0;
 	byte const* code = nullptr;
 	uint64_t 	codeSize = 0;
-	i256		codeHash;
+	h256		codeHash;
 };
 
 /// VM Environment (ExtVM) opaque type

exp/main.cpp

@@ -81,7 +81,7 @@ int main()
 //	cdebug << toString(a2);
 	Address a2("19c486071651b2650449ba3c6a807f316a73e8fe");
 
-	cdebug << keyman.keys();
+	cdebug << keyman.accountDetails();
 
 	cdebug << "Secret key for " << a << "is" << keyman.secret(a, [](){ return "bar"; });
 	cdebug << "Secret key for " << a2 << "is" << keyman.secret(a2);

libdevcore/Common.cpp

@@ -28,7 +28,9 @@ using namespace dev;
 namespace dev
 {
 
-char const* Version = "0.9.17";
+char const* Version = "0.9.19";
+
+const u256 UndefinedU256 = ~(u256)0;
 
 void HasInvariants::checkInvariants() const
 {

libdevcore/Common.h

@@ -82,6 +82,8 @@ using u160s = std::vector<u160>;
 using u256Set = std::set<u256>;
 using u160Set = std::set<u160>;
 
+extern const u256 UndefinedU256;
+
 // Map types.
 using StringMap = std::map<std::string, std::string>;
 using u256Map = std::map<u256, u256>;
@@ -197,12 +199,12 @@ private:
 #define DEV_TIMED_FUNCTION DEV_TIMED_SCOPE(__PRETTY_FUNCTION__)
 #endif
 
-#define DEV_TIMED_IF(S, MS) for (::std::pair<::dev::TimerHelper, bool> __eth_t(::dev::TimerHelper(#S, MS), true); __eth_t.second; __eth_t.second = false)
-#define DEV_TIMED_SCOPE_IF(S) ::dev::TimerHelper __eth_t(S, MS)
+#define DEV_TIMED_ABOVE(S, MS) for (::std::pair<::dev::TimerHelper, bool> __eth_t(::dev::TimerHelper(#S, MS), true); __eth_t.second; __eth_t.second = false)
+#define DEV_TIMED_SCOPE_ABOVE(S) ::dev::TimerHelper __eth_t(S, MS)
 #if WIN32
-#define DEV_TIMED_FUNCTION_IF(MS) DEV_TIMED_SCOPE_IF(__FUNCSIG__, MS)
+#define DEV_TIMED_FUNCTION_ABOVE(MS) DEV_TIMED_SCOPE_ABOVE(__FUNCSIG__, MS)
 #else
-#define DEV_TIMED_FUNCTION_IF(MS) DEV_TIMED_SCOPE_IF(__PRETTY_FUNCTION__, MS)
+#define DEV_TIMED_FUNCTION_ABOVE(MS) DEV_TIMED_SCOPE_ABOVE(__PRETTY_FUNCTION__, MS)
 #endif
 
 enum class WithExisting: int

libdevcore/CommonIO.cpp

@@ -20,7 +20,8 @@
  */
 
 #include "CommonIO.h"
-
+#include <iostream>
+#include <cstdlib>
 #include <fstream>
 #include "Exceptions.h"
 using namespace std;
@@ -117,3 +118,14 @@ void dev::writeFile(std::string const& _file, bytesConstRef _data)
 	ofstream(_file, ios::trunc|ios::binary).write((char const*)_data.data(), _data.size());
 }
 
+std::string dev::getPassword(std::string const& _prompt)
+{
+#if WIN32
+	cout << _prompt << flush;
+	std::string ret;
+	std::getline(cin, ret);
+	return ret;
+#else
+	return getpass(_prompt.c_str());
+#endif
+}

libdevcore/CommonIO.h

@@ -42,6 +42,8 @@
 namespace dev
 {
 
+std::string getPassword(std::string const& _prompt);
+
 /// Retrieve and returns the contents of the given file. If the file doesn't exist or isn't readable, returns an empty bytes.
 bytes contents(std::string const& _file);
 std::string contentsString(std::string const& _file);

libdevcore/FixedHash.h

@@ -282,6 +282,7 @@ namespace std
 {
 	/// Forward std::hash<dev::FixedHash> to dev::FixedHash::hash.
 	template<> struct hash<dev::h64>: dev::h64::hash {};
+	template<> struct hash<dev::h128>: dev::h128::hash {};
 	template<> struct hash<dev::h160>: dev::h160::hash {};
 	template<> struct hash<dev::h256>: dev::h256::hash {};
 	template<> struct hash<dev::h512>: dev::h512::hash {};

libdevcore/StructuredLogger.cpp

@@ -34,6 +34,15 @@ using namespace std;
 namespace dev
 {
 
+void StructuredLogger::initialize(bool _enabled, std::string const& _timeFormat, std::string const& _destinationURL)
+{
+	m_enabled = _enabled;
+	m_timeFormat = _timeFormat;
+	if (_destinationURL.size() > 7 && _destinationURL.substr(0, 7) == "file://")
+		m_out.open(_destinationURL.substr(7));
+	// TODO: support tcp://
+}
+
 void StructuredLogger::outputJson(Json::Value const& _value, std::string const& _name) const
 {
 	Json::Value event;
@@ -41,7 +50,7 @@ void StructuredLogger::outputJson(Json::Value const& _value, std::string const&
 	Json::FastWriter fastWriter;
 	Guard l(s_lock);
 	event[_name] = _value;
-	cout << fastWriter.write(event) << endl;
+	(m_out.is_open() ? m_out : cout) << fastWriter.write(event) << endl;
 }
 
 void StructuredLogger::starting(string const& _clientImpl, const char* _ethVersion)
@@ -51,6 +60,7 @@ void StructuredLogger::starting(string const& _clientImpl, const char* _ethVersi
 		Json::Value event;
 		event["client_impl"] = _clientImpl;
 		event["eth_version"] = std::string(_ethVersion);
+		// TODO net_version
 		event["ts"] = dev::toString(chrono::system_clock::now(), get().m_timeFormat.c_str());
 
 		get().outputJson(event, "starting");
@@ -64,6 +74,7 @@ void StructuredLogger::stopping(string const& _clientImpl, const char* _ethVersi
 		Json::Value event;
 		event["client_impl"] = _clientImpl;
 		event["eth_version"] = std::string(_ethVersion);
+		// TODO net_version
 		event["ts"] = dev::toString(chrono::system_clock::now(), get().m_timeFormat.c_str());
 
 		get().outputJson(event, "stopping");

libdevcore/StructuredLogger.h

@@ -25,6 +25,7 @@
 
 #pragma once
 
+#include <fstream>
 #include <string>
 #include <chrono>
 
@@ -46,11 +47,7 @@ public:
 	 *                        http://en.cppreference.com/w/cpp/chrono/c/strftime
 	 *                        with which to display timestamps
 	 */
-	void initialize(bool _enabled, std::string const& _timeFormat)
-	{
-		m_enabled = _enabled;
-		m_timeFormat = _timeFormat;
-	}
+	void initialize(bool _enabled, std::string const& _timeFormat, std::string const& _destinationURL = "");
 
 	static StructuredLogger& get()
 	{
@@ -92,6 +89,11 @@ public:
 		std::string const& _prevHash
 	);
 	static void transactionReceived(std::string const& _hash, std::string const& _remoteId);
+	// TODO: static void pendingQueueChanged(std::vector<h256> const& _hashes);
+	// TODO: static void miningStarted();
+	// TODO: static void stillMining(unsigned _hashrate);
+	// TODO: static void miningStopped();
+
 private:
 	// Singleton class. Private default ctor and no copying
 	StructuredLogger() = default;
@@ -102,6 +104,8 @@ private:
 
 	bool m_enabled = false;
 	std::string m_timeFormat = "%Y-%m-%dT%H:%M:%S";
+
+	mutable std::ofstream m_out;
 };
 
 }

libdevcore/Worker.cpp

@@ -65,15 +65,15 @@ void Worker::startWorking()
 					m_state.exchange(ex);
 
 //				cnote << "Waiting until not Stopped...";
-				DEV_TIMED_IF(Worker stopping, 100)
+				DEV_TIMED_ABOVE(Worker stopping, 100)
 					while (m_state == WorkerState::Stopped)
 						this_thread::sleep_for(chrono::milliseconds(20));
 			}
 		}));
 //		cnote << "Spawning" << m_name;
 	}
-	DEV_TIMED_IF(Start worker, 100)
-		while (m_state != WorkerState::Started)
+	DEV_TIMED_ABOVE(Start worker, 100)
+		while (m_state == WorkerState::Starting)
 			this_thread::sleep_for(chrono::microseconds(20));
 }
 
@@ -85,7 +85,7 @@ void Worker::stopWorking()
 			WorkerState ex = WorkerState::Started;
 			m_state.compare_exchange_strong(ex, WorkerState::Stopping);
 
-			DEV_TIMED_IF(Stop worker, 100)
+			DEV_TIMED_ABOVE(Stop worker, 100)
 				while (m_state != WorkerState::Stopped)
 					this_thread::sleep_for(chrono::microseconds(20));
 		}
@@ -99,7 +99,7 @@ void Worker::terminate()
 		{
 			m_state.exchange(WorkerState::Killing);
 
-			DEV_TIMED_IF(Terminate worker, 100)
+			DEV_TIMED_ABOVE(Terminate worker, 100)
 				m_work->join();
 
 			m_work.reset();

libdevcrypto/MemoryDB.cpp

@@ -32,6 +32,7 @@ const char* DBWarn::name() { return "TDB"; }
 
 std::unordered_map<h256, std::string> MemoryDB::get() const
 {
+	ReadGuard l(x_this);
 	std::unordered_map<h256, std::string> ret;
 	for (auto const& i: m_main)
 		if (!m_enforceRefs || i.second.second > 0)
@@ -39,21 +40,34 @@ std::unordered_map<h256, std::string> MemoryDB::get() const
 	return ret;
 }
 
+MemoryDB& MemoryDB::operator=(MemoryDB const& _c)
+{
+	if (this == &_c)
+		return *this;
+	ReadGuard l(_c.x_this);
+	WriteGuard l2(x_this);
+	m_main = _c.m_main;
+	m_aux = _c.m_aux;
+	return *this;
+}
+
 std::string MemoryDB::lookup(h256 const& _h) const
 {
+	ReadGuard l(x_this);
 	auto it = m_main.find(_h);
 	if (it != m_main.end())
 	{
 		if (!m_enforceRefs || it->second.second > 0)
 			return it->second.first;
-//		else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first))
-//			cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h;
+		else
+			cwarn << "Lookup required for value with refcount == 0. This is probably a critical trie issue" << _h;
 	}
 	return std::string();
 }
 
 bool MemoryDB::exists(h256 const& _h) const
 {
+	ReadGuard l(x_this);
 	auto it = m_main.find(_h);
 	if (it != m_main.end() && (!m_enforceRefs || it->second.second > 0))
 		return true;
@@ -62,6 +76,7 @@ bool MemoryDB::exists(h256 const& _h) const
 
 void MemoryDB::insert(h256 const& _h, bytesConstRef _v)
 {
+	WriteGuard l(x_this);
 	auto it = m_main.find(_h);
 	if (it != m_main.end())
 	{
@@ -77,34 +92,34 @@ void MemoryDB::insert(h256 const& _h, bytesConstRef _v)
 
 bool MemoryDB::kill(h256 const& _h)
 {
+	ReadGuard l(x_this);
 	if (m_main.count(_h))
 	{
 		if (m_main[_h].second > 0)
+		{
 			m_main[_h].second--;
+			return true;
+		}
 #if ETH_PARANOIA
 		else
 		{
 			// If we get to this point, then there was probably a node in the level DB which we need to remove and which we have previously
 			// used as part of the memory-based MemoryDB. Nothing to be worried about *as long as the node exists in the DB*.
 			dbdebug << "NOKILL-WAS" << _h;
-			return false;
 		}
 		dbdebug << "KILL" << _h << "=>" << m_main[_h].second;
-		return true;
 	}
 	else
 	{
 		dbdebug << "NOKILL" << _h;
-		return false;
-	}
-#else
-	}
-	return true;
 #endif
+	}
+	return false;
 }
 
 void MemoryDB::purge()
 {
+	WriteGuard l(x_this);
 	for (auto it = m_main.begin(); it != m_main.end(); )
 		if (it->second.second)
 			++it;
@@ -114,6 +129,7 @@ void MemoryDB::purge()
 
 h256Hash MemoryDB::keys() const
 {
+	ReadGuard l(x_this);
 	h256Hash ret;
 	for (auto const& i: m_main)
 		if (i.second.second)

libdevcrypto/MemoryDB.h

@@ -23,6 +23,7 @@
 
 #include <unordered_map>
 #include <libdevcore/Common.h>
+#include <libdevcore/Guards.h>
 #include <libdevcore/FixedHash.h>
 #include <libdevcore/Log.h>
 #include <libdevcore/RLP.h>
@@ -43,6 +44,9 @@ class MemoryDB
 
 public:
 	MemoryDB() {}
+	MemoryDB(MemoryDB const& _c) { operator=(_c); }
+
+	MemoryDB& operator=(MemoryDB const& _c);
 
 	void clear() { m_main.clear(); }	// WARNING !!!! didn't originally clear m_refCount!!!
 	std::unordered_map<h256, std::string> get() const;
@@ -53,13 +57,14 @@ public:
 	bool kill(h256 const& _h);
 	void purge();
 
-	bytes lookupAux(h256 const& _h) const { try { return m_aux.at(_h).first; } catch (...) { return bytes(); } }
-	void removeAux(h256 const& _h) { m_aux[_h].second = false; }
-	void insertAux(h256 const& _h, bytesConstRef _v) { m_aux[_h] = make_pair(_v.toBytes(), true); }
+	bytes lookupAux(h256 const& _h) const { ReadGuard l(x_this); auto it = m_aux.find(_h); if (it != m_aux.end() && (!m_enforceRefs || it->second.second)) return it->second.first; return bytes(); }
+	void removeAux(h256 const& _h) { WriteGuard l(x_this); m_aux[_h].second = false; }
+	void insertAux(h256 const& _h, bytesConstRef _v) { WriteGuard l(x_this); m_aux[_h] = make_pair(_v.toBytes(), true); }
 
 	h256Hash keys() const;
 
 protected:
+	mutable SharedMutex x_this;
 	std::unordered_map<h256, std::pair<std::string, unsigned>> m_main;
 	std::unordered_map<h256, std::pair<bytes, bool>> m_aux;
 

libdevcrypto/OverlayDB.cpp

@@ -19,6 +19,7 @@
  * @date 2014
  */
 
+#include <thread>
 #include <leveldb/db.h>
 #include <leveldb/write_batch.h>
 #include <libdevcore/Common.h>
@@ -29,6 +30,8 @@ using namespace dev;
 namespace dev
 {
 
+h256 const EmptyTrie = sha3(rlp(""));
+
 OverlayDB::~OverlayDB()
 {
 	if (m_db.use_count() == 1 && m_db.get())
@@ -41,30 +44,41 @@ void OverlayDB::commit()
 	{
 		ldb::WriteBatch batch;
 //		cnote << "Committing nodes to disk DB:";
-		for (auto const& i: m_main)
+		DEV_READ_GUARDED(x_this)
 		{
-//			cnote << i.first << "#" << m_main[i.first].second;
-			if (i.second.second)
-				batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.first.data(), i.second.first.size()));
-		}
-		for (auto const& i: m_aux)
-			if (i.second.second)
+			for (auto const& i: m_main)
 			{
-				bytes b = i.first.asBytes();
-				b.push_back(255);	// for aux
-				batch.Put(bytesConstRef(&b), bytesConstRef(&i.second.first));
+				if (i.second.second)
+					batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.first.data(), i.second.first.size()));
+//				cnote << i.first << "#" << m_main[i.first].second;
 			}
-		m_db->Write(m_writeOptions, &batch);
+			for (auto const& i: m_aux)
+				if (i.second.second)
+				{
+					bytes b = i.first.asBytes();
+					b.push_back(255);	// for aux
+					batch.Put(bytesConstRef(&b), bytesConstRef(&i.second.first));
+				}
+		}
+
+		while (true)
+		{
+			ldb::Status o = m_db->Write(m_writeOptions, &batch);
+			if (o.ok())
+				break;
+			cwarn << "Error writing to database. Sleeping a while then retrying. If it keeps saying this, free up some space!";
+			this_thread::sleep_for(chrono::milliseconds(500));
+		}
 		m_aux.clear();
 		m_main.clear();
 	}
 }
 
-bytes OverlayDB::lookupAux(h256 _h) const
+bytes OverlayDB::lookupAux(h256 const& _h) const
 {
 	bytes ret = MemoryDB::lookupAux(_h);
 	if (!ret.empty())
-		return ret;
+		return move(ret);
 	std::string v;
 	bytes b = _h.asBytes();
 	b.push_back(255);	// for aux
@@ -76,18 +90,19 @@ bytes OverlayDB::lookupAux(h256 _h) const
 
 void OverlayDB::rollback()
 {
+	WriteGuard l(x_this);
 	m_main.clear();
 }
 
-std::string OverlayDB::lookup(h256 _h) const
+std::string OverlayDB::lookup(h256 const& _h) const
 {
 	std::string ret = MemoryDB::lookup(_h);
 	if (ret.empty() && m_db)
 		m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret);
-	return ret;
+	return move(ret);
 }
 
-bool OverlayDB::exists(h256 _h) const
+bool OverlayDB::exists(h256 const& _h) const
 {
 	if (MemoryDB::exists(_h))
 		return true;
@@ -97,16 +112,20 @@ bool OverlayDB::exists(h256 _h) const
 	return !ret.empty();
 }
 
-void OverlayDB::kill(h256 _h)
+void OverlayDB::kill(h256 const& _h)
 {
-#if ETH_PARANOIA
+#if ETH_PARANOIA || 1
 	if (!MemoryDB::kill(_h))
 	{
 		std::string ret;
 		if (m_db)
 			m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret);
-		if (ret.empty())
+		// No point node ref decreasing for EmptyTrie since we never bother incrementing it in the first place for
+		// empty storage tries.
+		if (ret.empty() && _h != EmptyTrie)
 			cnote << "Decreasing DB node ref count below zero with no DB node. Probably have a corrupt Trie." << _h;
+
+		// TODO: for 1.1: ref-counted triedb.
 	}
 #else
 	MemoryDB::kill(_h);

libdevcrypto/OverlayDB.h

@@ -46,11 +46,11 @@ public:
 	void commit();
 	void rollback();
 
-	std::string lookup(h256 _h) const;
-	bool exists(h256 _h) const;
-	void kill(h256 _h);
+	std::string lookup(h256 const& _h) const;
+	bool exists(h256 const& _h) const;
+	void kill(h256 const& _h);
 
-	bytes lookupAux(h256 _h) const;
+	bytes lookupAux(h256 const& _h) const;
 
 private:
 	using MemoryDB::clear;

libdevcrypto/SecretStore.cpp

@@ -104,6 +104,7 @@ void SecretStore::save(std::string const& _keysPath)
 void SecretStore::load(std::string const& _keysPath)
 {
 	fs::path p(_keysPath);
+	boost::filesystem::create_directories(p);
 	js::mValue v;
 	for (fs::directory_iterator it(p); it != fs::directory_iterator(); ++it)
 		if (is_regular_file(it->path()))

libdevcrypto/SecretStore.h

@@ -23,6 +23,7 @@
 
 #include <functional>
 #include <mutex>
+#include <libdevcore/FixedHash.h>
 #include "Common.h"
 #include "FileSystem.h"
 
@@ -48,8 +49,8 @@ private:
 	static std::string encrypt(bytes const& _v, std::string const& _pass);
 	static bytes decrypt(std::string const& _v, std::string const& _pass);
 
-	mutable std::map<h128, bytes> m_cached;
-	std::map<h128, std::pair<std::string, std::string>> m_keys;
+	mutable std::unordered_map<h128, bytes> m_cached;
+	std::unordered_map<h128, std::pair<std::string, std::string>> m_keys;
 };
 
 }

libdevcrypto/TrieDB.cpp

@@ -25,6 +25,6 @@ using namespace std;
 using namespace dev;
 
 h256 const dev::c_shaNull = sha3(rlp(""));
-h256 const dev::EmptyTrie = c_shaNull;
+h256 const dev::EmptyTrie = sha3(rlp(""));
 
 const char* TrieDBChannel::name() { return "-T-"; }

libdevcrypto/TrieDB.h

@@ -79,7 +79,7 @@ public:
 	void open(DB* _db) { m_db = _db; }
 	void open(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { m_db = _db; setRoot(_root, _v); }
 
-	void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); }
+	void init() { setRoot(forceInsertNode(&RLPNull)); assert(node(m_root).size()); }
 
 	void setRoot(h256 const& _root, Verification _v = Verification::Normal)
 	{
@@ -88,11 +88,13 @@ public:
 		{
 			if (m_root == c_shaNull && !m_db->exists(m_root))
 				init();
-
-			/*std::cout << "Setting root to " << _root << " (patched to " << m_root << ")" << std::endl;*/
+		}
+		/*std::cout << "Setting root to " << _root << " (patched to " << m_root << ")" << std::endl;*/
+#if ETH_DEBUG
+		if (_v == Verification::Normal)
+#endif
 			if (!node(m_root).size())
 				BOOST_THROW_EXCEPTION(RootNotFound());
-		}
 	}
 
 	/// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node).
@@ -282,11 +284,17 @@ private:
 	std::string deref(RLP const& _n) const;
 
 	std::string node(h256 _h) const { return m_db->lookup(_h); }
-	void insertNode(h256 _h, bytesConstRef _v) { m_db->insert(_h, _v); }
-	void killNode(h256 _h) { m_db->kill(_h); }
 
-	h256 insertNode(bytesConstRef _v) { auto h = sha3(_v); insertNode(h, _v); return h; }
-	void killNode(RLP const& _d) { if (_d.data().size() >= 32) killNode(sha3(_d.data())); }
+	// These are low-level node insertion functions that just go straight through into the DB.
+	h256 forceInsertNode(bytesConstRef _v) { auto h = sha3(_v); forceInsertNode(h, _v); return h; }
+	void forceInsertNode(h256 _h, bytesConstRef _v) { m_db->insert(_h, _v); }
+	void forceKillNode(h256 _h) { m_db->kill(_h); }
+
+	// This are semantically-aware node insertion functions that only kills when the node's
+	// data is < 32 bytes. It can safely be used when pruning the trie but won't work correctly
+	// for the special case of the root (which is always looked up via a hash). In that case,
+	// use forceKillNode().
+	void killNode(RLP const& _d) { if (_d.data().size() >= 32) forceKillNode(sha3(_d.data())); }
 
 	h256 m_root;
 	DB* m_db = nullptr;
@@ -743,8 +751,8 @@ template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesCons
 	// However, we know it's the root node and thus always hashed.
 	// So, if it's less than 32 (and thus should have been deleted but wasn't) then we delete it here.
 	if (rv.size() < 32)
-		killNode(m_root);
-	m_root = insertNode(&b);
+		forceKillNode(m_root);
+	m_root = forceInsertNode(&b);
 }
 
 template <class DB> std::string GenericTrieDB<DB>::at(bytesConstRef _key) const
@@ -890,8 +898,8 @@ template <class DB> void GenericTrieDB<DB>::remove(bytesConstRef _key)
 	if (b.size())
 	{
 		if (rv.size() < 32)
-			killNode(m_root);
-		m_root = insertNode(&b);
+			forceKillNode(m_root);
+		m_root = forceInsertNode(&b);
 	}
 }
 
@@ -1081,7 +1089,7 @@ template <class DB> RLPStream& GenericTrieDB<DB>::streamNode(RLPStream& _s, byte
 	if (_b.size() < 32)
 		_s.appendRaw(_b);
 	else
-		_s.append(insertNode(&_b));
+		_s.append(forceInsertNode(&_b));
 	return _s;
 }
 
@@ -1122,7 +1130,7 @@ template <class DB> bytes GenericTrieDB<DB>::graft(RLP const& _orig)
 		// remove second item from the trie after derefrencing it into s & n.
 		auto lh = _orig[1].toHash<h256>();
 		s = node(lh);
-		killNode(lh);
+		forceKillNode(lh);
 		n = RLP(s);
 	}
 	assert(n.itemCount() == 2);

libethash-cl/ethash_cl_miner.cpp

@@ -27,6 +27,7 @@
 #include <iostream>
 #include <assert.h>
 #include <queue>
+#include <random>
 #include <vector>
 #include <libethash/util.h>
 #include <libethash/ethash.h>
@@ -119,9 +120,7 @@ unsigned ethash_cl_miner::get_num_devices(unsigned _platformId)
 void ethash_cl_miner::finish()
 {
 	if (m_queue())
-	{
 		m_queue.finish();
-	}
 }
 
 bool ethash_cl_miner::init(uint8_t const* _dag, uint64_t _dagSize, unsigned workgroup_size, unsigned _platformId, unsigned _deviceId)
@@ -161,9 +160,7 @@ bool ethash_cl_miner::init(uint8_t const* _dag, uint64_t _dagSize, unsigned work
 		return false;
 	}
 	if (strncmp("OpenCL 1.1", device_version.c_str(), 10) == 0)
-	{
 		m_opencl_1_1 = true;
-	}
 
 	// create context
 	m_context = cl::Context(std::vector<cl::Device>(&device, &device + 1));
@@ -306,21 +303,15 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook
 	// update header constant buffer
 	m_queue.enqueueWriteBuffer(m_header, false, 0, 32, header);
 	for (unsigned i = 0; i != c_num_buffers; ++i)
-	{
 		m_queue.enqueueWriteBuffer(m_search_buf[i], false, 0, 4, &c_zero);
-	}
 
 #if CL_VERSION_1_2 && 0
 	cl::Event pre_return_event;
 	if (!m_opencl_1_1)
-	{
 		m_queue.enqueueBarrierWithWaitList(NULL, &pre_return_event);
-	}
 	else
 #endif
-	{
 		m_queue.finish();
-	}
 
 	/*
 	__kernel void ethash_combined_search(
@@ -341,7 +332,9 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook
 
 
 	unsigned buf = 0;
-	for (uint64_t start_nonce = 0; ; start_nonce += c_search_batch_size)
+	std::random_device engine;
+	uint64_t start_nonce = std::uniform_int_distribution<uint64_t>()(engine);
+	for (; ; start_nonce += c_search_batch_size)
 	{
 		// supply output buffer to kernel
 		m_search_kernel.setArg(0, m_search_buf[buf]);
@@ -386,9 +379,7 @@ void ethash_cl_miner::search(uint8_t const* header, uint64_t target, search_hook
 	// not safe to return until this is ready
 #if CL_VERSION_1_2 && 0
 	if (!m_opencl_1_1)
-	{
 		pre_return_event.wait();
-	}
 #endif
 }
 

libethash/CMakeLists.txt

@@ -10,8 +10,7 @@ if (NOT MSVC)
 	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
 endif()
 
-set(FILES 	util.c
-          	util.h
+set(FILES 	util.h
           	io.c
           	internal.c
           	ethash.h
@@ -21,7 +20,7 @@ set(FILES 	util.c
           	data_sizes.h)
 
 if (MSVC)
-	list(APPEND FILES io_win32.c)
+	list(APPEND FILES util_win32.c io_win32.c mmap_win32.c)
 else()
 	list(APPEND FILES io_posix.c)
 endif()
@@ -46,4 +45,4 @@ endif()
 
 if (NOT ETHASHCL)
 	install( TARGETS ${LIBRARY} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib )
-endif ()
+endif ()

libethash/endian.h

@@ -3,38 +3,6 @@
 #include <stdint.h>
 #include "compiler.h"
 
-static const uint8_t BitReverseTable256[] =
-        {
-                0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
-                0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
-                0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
-                0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
-                0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
-                0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
-                0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
-                0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
-                0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
-                0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
-                0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
-                0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
-                0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
-                0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
-                0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
-                0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF
-        };
-
-static inline uint32_t bitfn_swap32(uint32_t a) {
-    return (BitReverseTable256[a & 0xff] << 24) |
-            (BitReverseTable256[(a >> 8) & 0xff] << 16) |
-            (BitReverseTable256[(a >> 16) & 0xff] << 8) |
-            (BitReverseTable256[(a >> 24) & 0xff]);
-}
-
-static inline uint64_t bitfn_swap64(uint64_t a) {
-    return ((uint64_t) bitfn_swap32((uint32_t) (a >> 32))) |
-            (((uint64_t) bitfn_swap32((uint32_t) a)) << 32);
-}
-
 #if defined(__MINGW32__) || defined(_WIN32)
   # define LITTLE_ENDIAN 1234
   # define BYTE_ORDER    LITTLE_ENDIAN
@@ -53,22 +21,52 @@ static inline uint64_t bitfn_swap64(uint64_t a) {
   # define BIG_ENDIAN 1234
   # define BYTE_ORDER    BIG_ENDIAN
 #else
-
 # include <endian.h>
+#endif
 
+#if defined(_WIN32)
+#include <stdlib.h>
+#define ethash_swap_u32(input_) _byteswap_ulong(input_)
+#define ethash_swap_u64(input_) _byteswap_uint64(input_)
+#elif defined(__APPLE__)
+#include <libkern/OSByteOrder.h>
+#define ethash_swap_u32(input_) OSSwapInt32(input_)
+#define ethash_swap_u64(input_) OSSwapInt64(input_)
+#else // posix
+#include <byteswap.h>
+#define ethash_swap_u32(input_) __bswap_32(input_)
+#define ethash_swap_u64(input_) __bswap_64(input_)
 #endif
 
 
 #if LITTLE_ENDIAN == BYTE_ORDER
 
-#define fix_endian32(x) (x)
-#define fix_endian64(x) (x)
+#define fix_endian32(dst_ ,src_) dst_ = src_
+#define fix_endian32_same(val_)
+#define fix_endian64(dst_, src_) dst_ = src_
+#define fix_endian64_same(val_)
+#define fix_endian_arr32(arr_, size_)
+#define fix_endian_arr64(arr_, size_)
 
 #elif BIG_ENDIAN == BYTE_ORDER
 
-#define fix_endian32(x) bitfn_swap32(x)
-#define fix_endian64(x) bitfn_swap64(x)
+#define fix_endian32(dst_, src_) dst_ = ethash_swap_u32(src_)
+#define fix_endian32_same(val_) val_ = ethash_swap_u32(val_)
+#define fix_endian64(dst_, src_) dst_ = ethash_swap_u64(src_
+#define fix_endian64_same(val_) val_ = ethash_swap_u64(val_)
+#define fix_endian_arr32(arr_, size_)			\
+	do {										\
+	for (unsigned i_ = 0; i_ < (size_), ++i_) { \
+		arr_[i_] = ethash_swap_u32(arr_[i_]);	\
+	}											\
+	while (0)
+#define fix_endian_arr64(arr_, size_)			\
+	do {										\
+	for (unsigned i_ = 0; i_ < (size_), ++i_) { \
+		arr_[i_] = ethash_swap_u64(arr_[i_]);	\
+	}											\
+	while (0)									\
 
 #else
 # error "endian not supported"
-#endif // BYTE_ORDER
+#endif // BYTE_ORDER

libethash/ethash.h

@@ -24,7 +24,6 @@
 #include <stdbool.h>
 #include <string.h>
 #include <stddef.h>
-#include <stdlib.h>
 #include "compiler.h"
 
 #define ETHASH_REVISION 23
@@ -38,101 +37,110 @@
 #define ETHASH_DATASET_PARENTS 256
 #define ETHASH_CACHE_ROUNDS 3
 #define ETHASH_ACCESSES 64
+#define ETHASH_DAG_MAGIC_NUM_SIZE 8
+#define ETHASH_DAG_MAGIC_NUM 0xFEE1DEADBADDCAFE
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct ethash_params {
-	uint64_t full_size;               // Size of full data set (in bytes, multiple of mix size (128)).
-	uint64_t cache_size;              // Size of compute cache (in bytes, multiple of node size (64)).
-} ethash_params;
+/// Type of a seedhash/blockhash e.t.c.
+typedef struct ethash_h256 { uint8_t b[32]; } ethash_h256_t;
 
-typedef struct ethash_return_value {
-	uint8_t result[32];
-	uint8_t mix_hash[32];
-} ethash_return_value;
-
-uint64_t ethash_get_datasize(const uint32_t block_number);
-uint64_t ethash_get_cachesize(const uint32_t block_number);
-
-// initialize the parameters
-static inline void ethash_params_init(ethash_params *params, const uint32_t block_number) {
-	params->full_size = ethash_get_datasize(block_number);
-	params->cache_size = ethash_get_cachesize(block_number);
-}
-
-/***********************************
- * OLD API *************************
- ***********************************
- ******************** (deprecated) *
- ***********************************/
+// convenience macro to statically initialize an h256_t
+// usage:
+// ethash_h256_t a = ethash_h256_static_init(1, 2, 3, ... )
+// have to provide all 32 values. If you don't provide all the rest
+// will simply be unitialized (not guranteed to be 0)
+#define ethash_h256_static_init(...)			\
+	{ {__VA_ARGS__} }
 
-void ethash_get_seedhash(uint8_t seedhash[32], const uint32_t block_number);
-void ethash_mkcache(void *cache, ethash_params const *params, const uint8_t seed[32]);
-void ethash_light(ethash_return_value *ret, void const *cache, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce);
-void ethash_compute_full_data(void *mem, ethash_params const *params, void const *cache);
-void ethash_full(ethash_return_value *ret, void const *full_mem, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce);
+struct ethash_light;
+typedef struct ethash_light* ethash_light_t;
+struct ethash_full;
+typedef struct ethash_full* ethash_full_t;
+typedef int(*ethash_callback_t)(unsigned);
 
-/***********************************
- * NEW API *************************
- ***********************************/
-
-// TODO: compute params and seed in ethash_new_light; it should take only block_number
-// TODO: store params in ethash_light_t/ethash_full_t to avoid having to repass into compute/new_full
-
-typedef uint8_t const ethash_seedhash_t[32];
-
-typedef void const* ethash_light_t;
-static inline ethash_light_t ethash_new_light(ethash_params const* params, ethash_seedhash_t seed) {
-	void* ret = malloc((size_t)params->cache_size);
-	ethash_mkcache(ret, params, seed);
-	return ret;
-}
-static inline void ethash_compute_light(ethash_return_value *ret, ethash_light_t light, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce) {
-	ethash_light(ret, light, params, header_hash, nonce);
-}
-static inline void ethash_delete_light(ethash_light_t light) {
-	free((void*)light);
-}
-
-typedef void const* ethash_full_t;
-static inline ethash_full_t ethash_new_full(ethash_params const* params, ethash_light_t light) {
-	void* ret = malloc((size_t)params->full_size);
-	ethash_compute_full_data(ret, params, light);
-	return ret;
-}
-static inline void ethash_prep_full(void *full, ethash_params const *params, void const *cache) {
-	ethash_compute_full_data(full, params, cache);
-}
-static inline void ethash_compute_full(ethash_return_value *ret, void const *full, ethash_params const *params, const uint8_t header_hash[32], const uint64_t nonce) {
-	ethash_full(ret, full, params, header_hash, nonce);
-}
-
-/// @brief Compare two s256-bit big-endian values.
-/// @returns 1 if @a a is less than or equal to @a b, 0 otherwise.
-/// Both parameters are 256-bit big-endian values.
-static inline int ethash_leq_be256(const uint8_t a[32], const uint8_t b[32]) {
-	// Boundary is big endian
-	for (int i = 0; i < 32; i++) {
-		if (a[i] == b[i])
-			continue;
-		return a[i] < b[i];
-	}
-	return 1;
-}
-
-/// Perofrms a cursory check on the validity of the nonce.
-/// @returns 1 if the nonce may possibly be valid for the given header_hash & boundary.
-/// @p boundary equivalent to 2 ^ 256 / block_difficulty, represented as a 256-bit big-endian.
-int ethash_preliminary_check_boundary(
-	const uint8_t header_hash[32],
-	const uint64_t nonce,
-	const uint8_t mix_hash[32],
-	const uint8_t boundary[32]);
-
-#define ethash_quick_check_difficulty ethash_preliminary_check_boundary
-#define ethash_check_difficulty ethash_leq_be256
+typedef struct ethash_return_value {
+	ethash_h256_t result;
+	ethash_h256_t mix_hash;
+	bool success;
+} ethash_return_value_t;
+
+/**
+ * Allocate and initialize a new ethash_light handler
+ *
+ * @param block_number   The block number for which to create the handler
+ * @return               Newly allocated ethash_light handler or NULL in case of
+ *                       ERRNOMEM or invalid parameters used for @ref ethash_compute_cache_nodes()
+ */
+ethash_light_t ethash_light_new(uint64_t block_number);
+/**
+ * Frees a previously allocated ethash_light handler
+ * @param light        The light handler to free
+ */
+void ethash_light_delete(ethash_light_t light);
+/**
+ * Calculate the light client data
+ *
+ * @param light          The light client handler
+ * @param header_hash    The header hash to pack into the mix
+ * @param nonce          The nonce to pack into the mix
+ * @return               an object of ethash_return_value_t holding the return values
+ */
+ethash_return_value_t ethash_light_compute(
+	ethash_light_t light,
+	ethash_h256_t const header_hash,
+	uint64_t nonce
+);
+
+/**
+ * Allocate and initialize a new ethash_full handler
+ *
+ * @param light         The light handler containing the cache.
+ * @param callback      A callback function with signature of @ref ethash_callback_t
+ *                      It accepts an unsigned with which a progress of DAG calculation
+ *                      can be displayed. If all goes well the callback should return 0.
+ *                      If a non-zero value is returned then DAG generation will stop.
+ *                      Be advised. A progress value of 100 means that DAG creation is
+ *                      almost complete and that this function will soon return succesfully.
+ *                      It does not mean that the function has already had a succesfull return.
+ * @return              Newly allocated ethash_full handler or NULL in case of
+ *                      ERRNOMEM or invalid parameters used for @ref ethash_compute_full_data()
+ */
+ethash_full_t ethash_full_new(ethash_light_t light, ethash_callback_t callback);
+
+/**
+ * Frees a previously allocated ethash_full handler
+ * @param full    The light handler to free
+ */
+void ethash_full_delete(ethash_full_t full);
+/**
+ * Calculate the full client data
+ *
+ * @param full           The full client handler
+ * @param header_hash    The header hash to pack into the mix
+ * @param nonce          The nonce to pack into the mix
+ * @return               An object of ethash_return_value to hold the return value
+ */
+ethash_return_value_t ethash_full_compute(
+	ethash_full_t full,
+	ethash_h256_t const header_hash,
+	uint64_t nonce
+);
+/**
+ * Get a pointer to the full DAG data
+ */
+void const* ethash_full_dag(ethash_full_t full);
+/**
+ * Get the size of the DAG data
+ */
+uint64_t ethash_full_dag_size(ethash_full_t full);
+
+/**
+ * Calculate the seedhash for a given block number
+ */
+ethash_h256_t ethash_get_seedhash(uint64_t block_number);
 
 #ifdef __cplusplus
 }

libethash/fnv.h

@@ -29,10 +29,11 @@ extern "C" {
 
 #define FNV_PRIME 0x01000193
 
-static inline uint32_t fnv_hash(const uint32_t x, const uint32_t y) {
-	return x*FNV_PRIME ^ y;
+static inline uint32_t fnv_hash(uint32_t const x, uint32_t const y)
+{
+	return x * FNV_PRIME ^ y;
 }
 
 #ifdef __cplusplus
 }
-#endif
+#endif

libethash/internal.c

@@ -8,11 +8,11 @@
 
   ethash is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
   GNU General Public License for more details.
 
   You should have received a copy of the GNU General Public License
-  along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+  along with cpp-ethereum.	If not, see <http://www.gnu.org/licenses/>.
 */
 /** @file internal.c
 * @author Tim Hughes <tim@twistedfury.com>
@@ -23,11 +23,15 @@
 #include <assert.h>
 #include <inttypes.h>
 #include <stddef.h>
+#include <errno.h>
+#include <math.h>
+#include "mmap.h"
 #include "ethash.h"
 #include "fnv.h"
 #include "endian.h"
 #include "internal.h"
 #include "data_sizes.h"
+#include "io.h"
 
 #ifdef WITH_CRYPTOPP
 
@@ -37,264 +41,456 @@
 #include "sha3.h"
 #endif // WITH_CRYPTOPP
 
-uint64_t ethash_get_datasize(const uint32_t block_number) {
-    assert(block_number / ETHASH_EPOCH_LENGTH < 2048);
-    return dag_sizes[block_number / ETHASH_EPOCH_LENGTH];
+uint64_t ethash_get_datasize(uint64_t const block_number)
+{
+	assert(block_number / ETHASH_EPOCH_LENGTH < 2048);
+	return dag_sizes[block_number / ETHASH_EPOCH_LENGTH];
 }
 
-uint64_t ethash_get_cachesize(const uint32_t block_number) {
-    assert(block_number / ETHASH_EPOCH_LENGTH < 2048);
-    return cache_sizes[block_number / ETHASH_EPOCH_LENGTH];
+uint64_t ethash_get_cachesize(uint64_t const block_number)
+{
+	assert(block_number / ETHASH_EPOCH_LENGTH < 2048);
+	return cache_sizes[block_number / ETHASH_EPOCH_LENGTH];
 }
 
 // Follows Sergio's "STRICT MEMORY HARD HASHING FUNCTIONS" (2014)
 // https://bitslog.files.wordpress.com/2013/12/memohash-v0-3.pdf
 // SeqMemoHash(s, R, N)
-void static ethash_compute_cache_nodes(
-        node *const nodes,
-        ethash_params const *params,
-        const uint8_t seed[32]) {
-    assert((params->cache_size % sizeof(node)) == 0);
-    uint32_t const num_nodes = (uint32_t) (params->cache_size / sizeof(node));
-
-    SHA3_512(nodes[0].bytes, seed, 32);
-
-    for (unsigned i = 1; i != num_nodes; ++i) {
-        SHA3_512(nodes[i].bytes, nodes[i - 1].bytes, 64);
-    }
-
-    for (unsigned j = 0; j != ETHASH_CACHE_ROUNDS; j++) {
-        for (unsigned i = 0; i != num_nodes; i++) {
-            uint32_t const idx = nodes[i].words[0] % num_nodes;
-            node data;
-            data = nodes[(num_nodes - 1 + i) % num_nodes];
-            for (unsigned w = 0; w != NODE_WORDS; ++w) {
-                data.words[w] ^= nodes[idx].words[w];
-            }
-            SHA3_512(nodes[i].bytes, data.bytes, sizeof(data));
-        }
-    }
-
-    // now perform endian conversion
-#if BYTE_ORDER != LITTLE_ENDIAN
-    for (unsigned w = 0; w != (num_nodes*NODE_WORDS); ++w)
-    {
-        nodes->words[w] = fix_endian32(nodes->words[w]);
-    }
-#endif
-}
-
-void ethash_mkcache(
-		void *cache,
-        ethash_params const *params,
-        const uint8_t seed[32]) {
-	node *nodes = (node *) cache;
-    ethash_compute_cache_nodes(nodes, params, seed);
+bool static ethash_compute_cache_nodes(
+	node* const nodes,
+	uint64_t cache_size,
+	ethash_h256_t const* seed
+)
+{
+	if (cache_size % sizeof(node) != 0) {
+		return false;
+	}
+	uint32_t const num_nodes = (uint32_t) (cache_size / sizeof(node));
+
+	SHA3_512(nodes[0].bytes, (uint8_t*)seed, 32);
+
+	for (uint32_t i = 1; i != num_nodes; ++i) {
+		SHA3_512(nodes[i].bytes, nodes[i - 1].bytes, 64);
+	}
+
+	for (uint32_t j = 0; j != ETHASH_CACHE_ROUNDS; j++) {
+		for (uint32_t i = 0; i != num_nodes; i++) {
+			uint32_t const idx = nodes[i].words[0] % num_nodes;
+			node data;
+			data = nodes[(num_nodes - 1 + i) % num_nodes];
+			for (uint32_t w = 0; w != NODE_WORDS; ++w) {
+				data.words[w] ^= nodes[idx].words[w];
+			}
+			SHA3_512(nodes[i].bytes, data.bytes, sizeof(data));
+		}
+	}
+
+	// now perform endian conversion
+	fix_endian_arr32(nodes->words, num_nodes * NODE_WORDS);
+	return true;
 }
 
 void ethash_calculate_dag_item(
-        node *const ret,
-        const unsigned node_index,
-        const struct ethash_params *params,
-		const void *cache) {
-
-    uint32_t num_parent_nodes = (uint32_t) (params->cache_size / sizeof(node));
-	node const *cache_nodes = (node const *) cache;
-    node const *init = &cache_nodes[node_index % num_parent_nodes];
-
-    memcpy(ret, init, sizeof(node));
-    ret->words[0] ^= node_index;
-    SHA3_512(ret->bytes, ret->bytes, sizeof(node));
-
+	node* const ret,
+	uint32_t node_index,
+	ethash_light_t const light
+)
+{
+	uint32_t num_parent_nodes = (uint32_t) (light->cache_size / sizeof(node));
+	node const* cache_nodes = (node const *) light->cache;
+	node const* init = &cache_nodes[node_index % num_parent_nodes];
+	memcpy(ret, init, sizeof(node));
+	ret->words[0] ^= node_index;
+	SHA3_512(ret->bytes, ret->bytes, sizeof(node));
 #if defined(_M_X64) && ENABLE_SSE
-    __m128i const fnv_prime = _mm_set1_epi32(FNV_PRIME);
-    __m128i xmm0 = ret->xmm[0];
-    __m128i xmm1 = ret->xmm[1];
-    __m128i xmm2 = ret->xmm[2];
-    __m128i xmm3 = ret->xmm[3];
+	__m128i const fnv_prime = _mm_set1_epi32(FNV_PRIME);
+	__m128i xmm0 = ret->xmm[0];
+	__m128i xmm1 = ret->xmm[1];
+	__m128i xmm2 = ret->xmm[2];
+	__m128i xmm3 = ret->xmm[3];
 #endif
 
-    for (unsigned i = 0; i != ETHASH_DATASET_PARENTS; ++i) {
-        uint32_t parent_index = ((node_index ^ i) * FNV_PRIME ^ ret->words[i % NODE_WORDS]) % num_parent_nodes;
-        node const *parent = &cache_nodes[parent_index];
+	for (uint32_t i = 0; i != ETHASH_DATASET_PARENTS; ++i) {
+		uint32_t parent_index = fnv_hash(node_index ^ i, ret->words[i % NODE_WORDS]) % num_parent_nodes;
+		node const *parent = &cache_nodes[parent_index];
 
 #if defined(_M_X64) && ENABLE_SSE
-        {
-            xmm0 = _mm_mullo_epi32(xmm0, fnv_prime);
-            xmm1 = _mm_mullo_epi32(xmm1, fnv_prime);
-            xmm2 = _mm_mullo_epi32(xmm2, fnv_prime);
-            xmm3 = _mm_mullo_epi32(xmm3, fnv_prime);
-            xmm0 = _mm_xor_si128(xmm0, parent->xmm[0]);
-            xmm1 = _mm_xor_si128(xmm1, parent->xmm[1]);
-            xmm2 = _mm_xor_si128(xmm2, parent->xmm[2]);
-            xmm3 = _mm_xor_si128(xmm3, parent->xmm[3]);
-
-            // have to write to ret as values are used to compute index
-            ret->xmm[0] = xmm0;
-            ret->xmm[1] = xmm1;
-            ret->xmm[2] = xmm2;
-            ret->xmm[3] = xmm3;
-        }
-        #else
-        {
-            for (unsigned w = 0; w != NODE_WORDS; ++w) {
-                ret->words[w] = fnv_hash(ret->words[w], parent->words[w]);
-            }
-        }
+		{
+			xmm0 = _mm_mullo_epi32(xmm0, fnv_prime);
+			xmm1 = _mm_mullo_epi32(xmm1, fnv_prime);
+			xmm2 = _mm_mullo_epi32(xmm2, fnv_prime);
+			xmm3 = _mm_mullo_epi32(xmm3, fnv_prime);
+			xmm0 = _mm_xor_si128(xmm0, parent->xmm[0]);
+			xmm1 = _mm_xor_si128(xmm1, parent->xmm[1]);
+			xmm2 = _mm_xor_si128(xmm2, parent->xmm[2]);
+			xmm3 = _mm_xor_si128(xmm3, parent->xmm[3]);
+
+			// have to write to ret as values are used to compute index
+			ret->xmm[0] = xmm0;
+			ret->xmm[1] = xmm1;
+			ret->xmm[2] = xmm2;
+			ret->xmm[3] = xmm3;
+		}
+		#else
+		{
+			for (unsigned w = 0; w != NODE_WORDS; ++w) {
+				ret->words[w] = fnv_hash(ret->words[w], parent->words[w]);
+			}
+		}
 #endif
-    }
-
-    SHA3_512(ret->bytes, ret->bytes, sizeof(node));
+	}
+	SHA3_512(ret->bytes, ret->bytes, sizeof(node));
 }
 
-void ethash_compute_full_data(
-        void *mem,
-        ethash_params const *params,
-		void const *cache) {
-    assert((params->full_size % (sizeof(uint32_t) * MIX_WORDS)) == 0);
-    assert((params->full_size % sizeof(node)) == 0);
-    node *full_nodes = mem;
-
-    // now compute full nodes
-    for (unsigned n = 0; n != (params->full_size / sizeof(node)); ++n) {
-        ethash_calculate_dag_item(&(full_nodes[n]), n, params, cache);
-    }
+bool ethash_compute_full_data(
+	void* mem,
+	uint64_t full_size,
+	ethash_light_t const light,
+	ethash_callback_t callback
+)
+{
+	if (full_size % (sizeof(uint32_t) * MIX_WORDS) != 0 ||
+		(full_size % sizeof(node)) != 0) {
+		return false;
+	}
+	uint32_t const max_n = (uint32_t)(full_size / sizeof(node));
+	node* full_nodes = mem;
+	double const progress_change = 1.0f / max_n;
+	double progress = 0.0f;
+	// now compute full nodes
+	for (uint32_t n = 0; n != max_n; ++n) {
+		if (callback &&
+			n % (max_n / 100) == 0 &&
+			callback((unsigned int)(ceil(progress * 100.0f))) != 0) {
+
+			return false;
+		}
+		progress += progress_change;
+		ethash_calculate_dag_item(&(full_nodes[n]), n, light);
+	}
+	return true;
 }
 
-static void ethash_hash(
-        ethash_return_value *ret,
-        node const *full_nodes,
-		void const *cache,
-        ethash_params const *params,
-        const uint8_t header_hash[32],
-        const uint64_t nonce) {
-
-    assert((params->full_size % MIX_WORDS) == 0);
+static bool ethash_hash(
+	ethash_return_value_t* ret,
+	node const* full_nodes,
+	ethash_light_t const light,
+	uint64_t full_size,
+	ethash_h256_t const header_hash,
+	uint64_t const nonce
+)
+{
+	if (full_size % MIX_WORDS != 0) {
+		return false;
+	}
+
+	// pack hash and nonce together into first 40 bytes of s_mix
+	assert(sizeof(node) * 8 == 512);
+	node s_mix[MIX_NODES + 1];
+	memcpy(s_mix[0].bytes, &header_hash, 32);
+	fix_endian64(s_mix[0].double_words[4], nonce);
+
+	// compute sha3-512 hash and replicate across mix
+	SHA3_512(s_mix->bytes, s_mix->bytes, 40);
+	fix_endian_arr32(s_mix[0].words, 16);
+
+	node* const mix = s_mix + 1;
+	for (uint32_t w = 0; w != MIX_WORDS; ++w) {
+		mix->words[w] = s_mix[0].words[w % NODE_WORDS];
+	}
+
+	unsigned const page_size = sizeof(uint32_t) * MIX_WORDS;
+	unsigned const num_full_pages = (unsigned) (full_size / page_size);
+
+	for (unsigned i = 0; i != ETHASH_ACCESSES; ++i) {
+		uint32_t const index = fnv_hash(s_mix->words[0] ^ i, mix->words[i % MIX_WORDS]) % num_full_pages;
+
+		for (unsigned n = 0; n != MIX_NODES; ++n) {
+			node const* dag_node;
+			if (full_nodes) {
+				dag_node = &full_nodes[MIX_NODES * index + n];
+			} else {
+				node tmp_node;
+				ethash_calculate_dag_item(&tmp_node, index * MIX_NODES + n, light);
+				dag_node = &tmp_node;
+			}
 
-    // pack hash and nonce together into first 40 bytes of s_mix
-    assert(sizeof(node) * 8 == 512);
-    node s_mix[MIX_NODES + 1];
-    memcpy(s_mix[0].bytes, header_hash, 32);
-
-#if BYTE_ORDER != LITTLE_ENDIAN
-    s_mix[0].double_words[4] = fix_endian64(nonce);
-#else
-    s_mix[0].double_words[4] = nonce;
+#if defined(_M_X64) && ENABLE_SSE
+			{
+				__m128i fnv_prime = _mm_set1_epi32(FNV_PRIME);
+				__m128i xmm0 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[0]);
+				__m128i xmm1 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[1]);
+				__m128i xmm2 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[2]);
+				__m128i xmm3 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[3]);
+				mix[n].xmm[0] = _mm_xor_si128(xmm0, dag_node->xmm[0]);
+				mix[n].xmm[1] = _mm_xor_si128(xmm1, dag_node->xmm[1]);
+				mix[n].xmm[2] = _mm_xor_si128(xmm2, dag_node->xmm[2]);
+				mix[n].xmm[3] = _mm_xor_si128(xmm3, dag_node->xmm[3]);
+			}
+			#else
+			{
+				for (unsigned w = 0; w != NODE_WORDS; ++w) {
+					mix[n].words[w] = fnv_hash(mix[n].words[w], dag_node->words[w]);
+				}
+			}
 #endif
+		}
+
+	}
+
+	// compress mix
+	for (uint32_t w = 0; w != MIX_WORDS; w += 4) {
+		uint32_t reduction = mix->words[w + 0];
+		reduction = reduction * FNV_PRIME ^ mix->words[w + 1];
+		reduction = reduction * FNV_PRIME ^ mix->words[w + 2];
+		reduction = reduction * FNV_PRIME ^ mix->words[w + 3];
+		mix->words[w / 4] = reduction;
+	}
+
+	fix_endian_arr32(mix->words, MIX_WORDS / 4);
+	memcpy(&ret->mix_hash, mix->bytes, 32);
+	// final Keccak hash
+	SHA3_256(&ret->result, s_mix->bytes, 64 + 32); // Keccak-256(s + compressed_mix)
+	return true;
+}
 
-    // compute sha3-512 hash and replicate across mix
-    SHA3_512(s_mix->bytes, s_mix->bytes, 40);
-
-#if BYTE_ORDER != LITTLE_ENDIAN
-    for (unsigned w = 0; w != 16; ++w) {
-        s_mix[0].words[w] = fix_endian32(s_mix[0].words[w]);
-    }
-#endif
+void ethash_quick_hash(
+	ethash_h256_t* return_hash,
+	ethash_h256_t const* header_hash,
+	uint64_t const nonce,
+	ethash_h256_t const* mix_hash
+)
+{
+	uint8_t buf[64 + 32];
+	memcpy(buf, header_hash, 32);
+	fix_endian64_same(nonce);
+	memcpy(&(buf[32]), &nonce, 8);
+	SHA3_512(buf, buf, 40);
+	memcpy(&(buf[64]), mix_hash, 32);
+	SHA3_256(return_hash, buf, 64 + 32);
+}
 
-    node *const mix = s_mix + 1;
-    for (unsigned w = 0; w != MIX_WORDS; ++w) {
-        mix->words[w] = s_mix[0].words[w % NODE_WORDS];
-    }
+ethash_h256_t ethash_get_seedhash(uint64_t block_number)
+{
+	ethash_h256_t ret;
+	ethash_h256_reset(&ret);
+	uint64_t const epochs = block_number / ETHASH_EPOCH_LENGTH;
+	for (uint32_t i = 0; i < epochs; ++i)
+		SHA3_256(&ret, (uint8_t*)&ret, 32);
+	return ret;
+}
 
-    unsigned const
-            page_size = sizeof(uint32_t) * MIX_WORDS,
-            num_full_pages = (unsigned) (params->full_size / page_size);
+bool ethash_quick_check_difficulty(
+	ethash_h256_t const* header_hash,
+	uint64_t const nonce,
+	ethash_h256_t const* mix_hash,
+	ethash_h256_t const* difficulty
+)
+{
+
+	ethash_h256_t return_hash;
+	ethash_quick_hash(&return_hash, header_hash, nonce, mix_hash);
+	return ethash_check_difficulty(&return_hash, difficulty);
+}
 
+ethash_light_t ethash_light_new_internal(uint64_t cache_size, ethash_h256_t const* seed)
+{
+	struct ethash_light *ret;
+	ret = calloc(sizeof(*ret), 1);
+	if (!ret) {
+		return NULL;
+	}
+	ret->cache = malloc((size_t)cache_size);
+	if (!ret->cache) {
+		goto fail_free_light;
+	}
+	node* nodes = (node*)ret->cache;
+	if (!ethash_compute_cache_nodes(nodes, cache_size, seed)) {
+		goto fail_free_cache_mem;
+	}
+	ret->cache_size = cache_size;
+	return ret;
+
+fail_free_cache_mem:
+	free(ret->cache);
+fail_free_light:
+	free(ret);
+	return NULL;
+}
 
-    for (unsigned i = 0; i != ETHASH_ACCESSES; ++i) {
-        uint32_t const index = ((s_mix->words[0] ^ i) * FNV_PRIME ^ mix->words[i % MIX_WORDS]) % num_full_pages;
+ethash_light_t ethash_light_new(uint64_t block_number)
+{
+	ethash_h256_t seedhash = ethash_get_seedhash(block_number);
+	ethash_light_t ret;
+	ret = ethash_light_new_internal(ethash_get_cachesize(block_number), &seedhash);
+	ret->block_number = block_number;
+	return ret;
+}
 
-        for (unsigned n = 0; n != MIX_NODES; ++n) {
-            const node *dag_node = &full_nodes[MIX_NODES * index + n];
+void ethash_light_delete(ethash_light_t light)
+{
+	if (light->cache) {
+		free(light->cache);
+	}
+	free(light);
+}
 
-            if (!full_nodes) {
-                node tmp_node;
-                ethash_calculate_dag_item(&tmp_node, index * MIX_NODES + n, params, cache);
-                dag_node = &tmp_node;
-            }
+ethash_return_value_t ethash_light_compute_internal(
+	ethash_light_t light,
+	uint64_t full_size,
+	ethash_h256_t const header_hash,
+	uint64_t nonce
+)
+{
+  	ethash_return_value_t ret;
+	ret.success = true;
+	if (!ethash_hash(&ret, NULL, light, full_size, header_hash, nonce)) {
+		ret.success = false;
+	}
+	return ret;
+}
 
-#if defined(_M_X64) && ENABLE_SSE
-            {
-                __m128i fnv_prime = _mm_set1_epi32(FNV_PRIME);
-                __m128i xmm0 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[0]);
-                __m128i xmm1 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[1]);
-                __m128i xmm2 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[2]);
-                __m128i xmm3 = _mm_mullo_epi32(fnv_prime, mix[n].xmm[3]);
-                mix[n].xmm[0] = _mm_xor_si128(xmm0, dag_node->xmm[0]);
-                mix[n].xmm[1] = _mm_xor_si128(xmm1, dag_node->xmm[1]);
-                mix[n].xmm[2] = _mm_xor_si128(xmm2, dag_node->xmm[2]);
-                mix[n].xmm[3] = _mm_xor_si128(xmm3, dag_node->xmm[3]);
-            }
-            #else
-            {
-                for (unsigned w = 0; w != NODE_WORDS; ++w) {
-                    mix[n].words[w] = fnv_hash(mix[n].words[w], dag_node->words[w]);
-                }
-            }
-#endif
-        }
-
-    }
-
-    // compress mix
-    for (unsigned w = 0; w != MIX_WORDS; w += 4) {
-        uint32_t reduction = mix->words[w + 0];
-        reduction = reduction * FNV_PRIME ^ mix->words[w + 1];
-        reduction = reduction * FNV_PRIME ^ mix->words[w + 2];
-        reduction = reduction * FNV_PRIME ^ mix->words[w + 3];
-        mix->words[w / 4] = reduction;
-    }
-
-#if BYTE_ORDER != LITTLE_ENDIAN
-    for (unsigned w = 0; w != MIX_WORDS/4; ++w) {
-        mix->words[w] = fix_endian32(mix->words[w]);
-    }
-#endif
+ethash_return_value_t ethash_light_compute(
+	ethash_light_t light,
+	ethash_h256_t const header_hash,
+	uint64_t nonce
+)
+{
+	uint64_t full_size = ethash_get_datasize(light->block_number);
+	return ethash_light_compute_internal(light, full_size, header_hash, nonce);
+}
 
-    memcpy(ret->mix_hash, mix->bytes, 32);
-    // final Keccak hash
-    SHA3_256(ret->result, s_mix->bytes, 64 + 32); // Keccak-256(s + compressed_mix)
+static bool ethash_mmap(struct ethash_full* ret, FILE* f)
+{
+	int fd;
+	char* mmapped_data;
+	ret->file = f;
+	if ((fd = ethash_fileno(ret->file)) == -1) {
+		return false;
+	}
+	mmapped_data= mmap(
+		NULL,
+		(size_t)ret->file_size + ETHASH_DAG_MAGIC_NUM_SIZE,
+		PROT_READ | PROT_WRITE,
+		MAP_SHARED,
+		fd,
+		0
+	);
+	if (mmapped_data == MAP_FAILED) {
+		return false;
+	}
+	ret->data = (node*)(mmapped_data + ETHASH_DAG_MAGIC_NUM_SIZE);
+	return true;
 }
 
-void ethash_quick_hash(
-        uint8_t return_hash[32],
-        const uint8_t header_hash[32],
-        const uint64_t nonce,
-        const uint8_t mix_hash[32]) {
-
-    uint8_t buf[64 + 32];
-    memcpy(buf, header_hash, 32);
-#if BYTE_ORDER != LITTLE_ENDIAN
-    nonce = fix_endian64(nonce);
-#endif
-    memcpy(&(buf[32]), &nonce, 8);
-    SHA3_512(buf, buf, 40);
-    memcpy(&(buf[64]), mix_hash, 32);
-    SHA3_256(return_hash, buf, 64 + 32);
+ethash_full_t ethash_full_new_internal(
+	char const* dirname,
+	ethash_h256_t const seed_hash,
+	uint64_t full_size,
+	ethash_light_t const light,
+	ethash_callback_t callback
+)
+{
+	struct ethash_full* ret;
+	FILE *f = NULL;
+	ret = calloc(sizeof(*ret), 1);
+	if (!ret) {
+		return NULL;
+	}
+	ret->file_size = (size_t)full_size;
+	switch (ethash_io_prepare(dirname, seed_hash, &f, (size_t)full_size, false)) {
+	case ETHASH_IO_FAIL:
+		goto fail_free_full;
+	case ETHASH_IO_MEMO_MATCH:
+		if (!ethash_mmap(ret, f)) {
+			goto fail_close_file;
+		}
+		return ret;
+	case ETHASH_IO_MEMO_SIZE_MISMATCH:
+		// if a DAG of same filename but unexpected size is found, silently force new file creation
+		if (ethash_io_prepare(dirname, seed_hash, &f, (size_t)full_size, true) != ETHASH_IO_MEMO_MISMATCH) {
+			goto fail_free_full;
+		}
+		// fallthrough to the mismatch case here, DO NOT go through match
+	case ETHASH_IO_MEMO_MISMATCH:
+		if (!ethash_mmap(ret, f)) {
+			goto fail_close_file;
+		}
+		break;
+	}
+
+	if (!ethash_compute_full_data(ret->data, full_size, light, callback)) {
+		goto fail_free_full_data;
+	}
+
+	// after the DAG has been filled then we finalize it by writting the magic number at the beginning
+	if (fseek(f, 0, SEEK_SET) != 0) {
+		goto fail_free_full_data;
+	}
+	uint64_t const magic_num = ETHASH_DAG_MAGIC_NUM;
+	if (fwrite(&magic_num, ETHASH_DAG_MAGIC_NUM_SIZE, 1, f) != 1) {
+		goto fail_free_full_data;
+	}
+	fflush(f); // make sure the magic number IS there
+	return ret;
+
+fail_free_full_data:
+	// could check that munmap(..) == 0 but even if it did not can't really do anything here
+	munmap(ret->data, (size_t)full_size);
+fail_close_file:
+	fclose(ret->file);
+fail_free_full:
+	free(ret);
+	return NULL;
 }
 
-void ethash_get_seedhash(uint8_t seedhash[32], const uint32_t block_number) {
-    memset(seedhash, 0, 32);
-    const uint32_t epochs = block_number / ETHASH_EPOCH_LENGTH;
-    for (uint32_t i = 0; i < epochs; ++i)
-        SHA3_256(seedhash, seedhash, 32);
+ethash_full_t ethash_full_new(ethash_light_t light, ethash_callback_t callback)
+{
+	char strbuf[256];
+	if (!ethash_get_default_dirname(strbuf, 256)) {
+		return NULL;
+	}
+	uint64_t full_size = ethash_get_datasize(light->block_number);
+	ethash_h256_t seedhash = ethash_get_seedhash(light->block_number);
+	return ethash_full_new_internal(strbuf, seedhash, full_size, light, callback);
 }
 
-int ethash_preliminary_check_boundary(
-        const uint8_t header_hash[32],
-        const uint64_t nonce,
-        const uint8_t mix_hash[32],
-		const uint8_t difficulty[32]) {
+void ethash_full_delete(ethash_full_t full)
+{
+	// could check that munmap(..) == 0 but even if it did not can't really do anything here
+	munmap(full->data, (size_t)full->file_size);
+	if (full->file) {
+		fclose(full->file);
+	}
+	free(full);
+}
 
-	uint8_t return_hash[32];
-    ethash_quick_hash(return_hash, header_hash, nonce, mix_hash);
-	return ethash_leq_be256(return_hash, difficulty);
+ethash_return_value_t ethash_full_compute(
+	ethash_full_t full,
+	ethash_h256_t const header_hash,
+	uint64_t nonce
+)
+{
+	ethash_return_value_t ret;
+	ret.success = true;
+	if (!ethash_hash(
+		&ret,
+		(node const*)full->data,
+		NULL,
+		full->file_size,
+		header_hash,
+		nonce)) {
+		ret.success = false;
+	}
+	return ret;
 }
 
-void ethash_full(ethash_return_value *ret, void const *full_mem, ethash_params const *params, const uint8_t previous_hash[32], const uint64_t nonce) {
-    ethash_hash(ret, (node const *) full_mem, NULL, params, previous_hash, nonce);
+void const* ethash_full_dag(ethash_full_t full)
+{
+	return full->data;
 }
 
-void ethash_light(ethash_return_value *ret, void const *cache, ethash_params const *params, const uint8_t previous_hash[32], const uint64_t nonce) {
-    ethash_hash(ret, NULL, cache, params, previous_hash, nonce);
+uint64_t ethash_full_dag_size(ethash_full_t full)
+{
+	return full->file_size;
 }

libethash/internal.h

@@ -2,6 +2,7 @@
 #include "compiler.h"
 #include "endian.h"
 #include "ethash.h"
+#include <stdio.h>
 
 #define ENABLE_SSE 0
 
@@ -20,9 +21,9 @@ extern "C" {
 #include <stdint.h>
 
 typedef union node {
-    uint8_t bytes[NODE_WORDS * 4];
-    uint32_t words[NODE_WORDS];
-    uint64_t double_words[NODE_WORDS / 2];
+	uint8_t bytes[NODE_WORDS * 4];
+	uint32_t words[NODE_WORDS];
+	uint64_t double_words[NODE_WORDS / 2];
 
 #if defined(_M_X64) && ENABLE_SSE
 	__m128i xmm[NODE_WORDS/4];
@@ -30,18 +31,139 @@ typedef union node {
 
 } node;
 
+static inline uint8_t ethash_h256_get(ethash_h256_t const* hash, unsigned int i)
+{
+	return hash->b[i];
+}
+
+static inline void ethash_h256_set(ethash_h256_t* hash, unsigned int i, uint8_t v)
+{
+	hash->b[i] = v;
+}
+
+static inline void ethash_h256_reset(ethash_h256_t* hash)
+{
+	memset(hash, 0, 32);
+}
+
+// Returns if hash is less than or equal to difficulty
+static inline bool ethash_check_difficulty(
+	ethash_h256_t const* hash,
+	ethash_h256_t const* difficulty
+)
+{
+	// Difficulty is big endian
+	for (int i = 0; i < 32; i++) {
+		if (ethash_h256_get(hash, i) == ethash_h256_get(difficulty, i)) {
+			continue;
+		}
+		return ethash_h256_get(hash, i) < ethash_h256_get(difficulty, i);
+	}
+	return true;
+}
+
+bool ethash_quick_check_difficulty(
+	ethash_h256_t const* header_hash,
+	uint64_t const nonce,
+	ethash_h256_t const* mix_hash,
+	ethash_h256_t const* difficulty
+);
+
+struct ethash_light {
+	void* cache;
+	uint64_t cache_size;
+	uint64_t block_number;
+};
+
+/**
+ * Allocate and initialize a new ethash_light handler. Internal version
+ *
+ * @param cache_size    The size of the cache in bytes
+ * @param seed          Block seedhash to be used during the computation of the
+ *                      cache nodes
+ * @return              Newly allocated ethash_light handler or NULL in case of
+ *                      ERRNOMEM or invalid parameters used for @ref ethash_compute_cache_nodes()
+ */
+ethash_light_t ethash_light_new_internal(uint64_t cache_size, ethash_h256_t const* seed);
+
+/**
+ * Calculate the light client data. Internal version.
+ *
+ * @param light          The light client handler
+ * @param full_size      The size of the full data in bytes.
+ * @param header_hash    The header hash to pack into the mix
+ * @param nonce          The nonce to pack into the mix
+ * @return               The resulting hash.
+ */
+ethash_return_value_t ethash_light_compute_internal(
+	ethash_light_t light,
+	uint64_t full_size,
+	ethash_h256_t const header_hash,
+	uint64_t nonce
+);
+
+struct ethash_full {
+	FILE* file;
+	uint64_t file_size;
+	node* data;
+};
+
+/**
+ * Allocate and initialize a new ethash_full handler. Internal version.
+ *
+ * @param dirname        The directory in which to put the DAG file.
+ * @param seedhash       The seed hash of the block. Used in the DAG file naming.
+ * @param full_size      The size of the full data in bytes.
+ * @param cache          A cache object to use that was allocated with @ref ethash_cache_new().
+ *                       Iff this function succeeds the ethash_full_t will take memory
+ *                       memory ownership of the cache and free it at deletion. If
+ *                       not then the user still has to handle freeing of the cache himself.
+ * @param callback       A callback function with signature of @ref ethash_callback_t
+ *                       It accepts an unsigned with which a progress of DAG calculation
+ *                       can be displayed. If all goes well the callback should return 0.
+ *                       If a non-zero value is returned then DAG generation will stop.
+ * @return               Newly allocated ethash_full handler or NULL in case of
+ *                       ERRNOMEM or invalid parameters used for @ref ethash_compute_full_data()
+ */
+ethash_full_t ethash_full_new_internal(
+	char const* dirname,
+	ethash_h256_t const seed_hash,
+	uint64_t full_size,
+	ethash_light_t const light,
+	ethash_callback_t callback
+);
+
 void ethash_calculate_dag_item(
-        node *const ret,
-        const unsigned node_index,
-        ethash_params const *params,
-		void const *cache
+	node* const ret,
+	uint32_t node_index,
+	ethash_light_t const cache
 );
 
 void ethash_quick_hash(
-        uint8_t return_hash[32],
-        const uint8_t header_hash[32],
-        const uint64_t nonce,
-        const uint8_t mix_hash[32]);
+	ethash_h256_t* return_hash,
+	ethash_h256_t const* header_hash,
+	const uint64_t nonce,
+	ethash_h256_t const* mix_hash
+);
+
+uint64_t ethash_get_datasize(uint64_t const block_number);
+uint64_t ethash_get_cachesize(uint64_t const block_number);
+
+/**
+ * Compute the memory data for a full node's memory
+ *
+ * @param mem         A pointer to an ethash full's memory
+ * @param full_size   The size of the full data in bytes
+ * @param cache       A cache object to use in the calculation
+ * @param callback    The callback function. Check @ref ethash_full_new() for details.
+ * @return            true if all went fine and false for invalid parameters
+ */
+bool ethash_compute_full_data(
+	void* mem,
+	uint64_t full_size,
+	ethash_light_t const light,
+	ethash_callback_t callback
+);
 
 #ifdef __cplusplus
 }

libethash/io.c

@@ -22,68 +22,81 @@
 #include <string.h>
 #include <stdio.h>
 
-// silly macro to save some typing
-#define PASS_ARR(c_) (c_), sizeof(c_)
-
-static bool ethash_io_write_file(char const *dirname,
-                                 char const* filename,
-                                 size_t filename_length,
-                                 void const* data,
-                                 size_t data_size)
-{
-    bool ret = false;
-    char *fullname = ethash_io_create_filename(dirname, filename, filename_length);
-    if (!fullname) {
-        return false;
-    }
-    FILE *f = fopen(fullname, "wb");
-    if (!f) {
-        goto free_name;
-    }
-    if (data_size != fwrite(data, 1, data_size, f)) {
-        goto close;
-    }
-
-    ret = true;
-close:
-    fclose(f);
-free_name:
-    free(fullname);
-    return ret;
-}
-
-bool ethash_io_write(char const *dirname,
-                     ethash_params const* params,
-                     ethash_blockhash_t seedhash,
-                     void const* cache,
-                     uint8_t **data,
-                     size_t *data_size)
+enum ethash_io_rc ethash_io_prepare(
+	char const* dirname,
+	ethash_h256_t const seedhash,
+	FILE** output_file,
+	uint64_t file_size,
+	bool force_create
+)
 {
-    char info_buffer[DAG_MEMO_BYTESIZE];
-    // allocate the bytes
-	uint8_t *temp_data_ptr = malloc((size_t)params->full_size);
-    if (!temp_data_ptr) {
-        goto end;
-    }
-    ethash_compute_full_data(temp_data_ptr, params, cache);
+	char mutable_name[DAG_MUTABLE_NAME_MAX_SIZE];
+	enum ethash_io_rc ret = ETHASH_IO_FAIL;
 
-	if (!ethash_io_write_file(dirname, PASS_ARR(DAG_FILE_NAME), temp_data_ptr, (size_t)params->full_size)) {
-        goto fail_free;
-    }
+	// assert directory exists
+	if (!ethash_mkdir(dirname)) {
+		goto end;
+	}
 
-    ethash_io_serialize_info(ETHASH_REVISION, seedhash, info_buffer);
-    if (!ethash_io_write_file(dirname, PASS_ARR(DAG_MEMO_NAME), info_buffer, DAG_MEMO_BYTESIZE)) {
-        goto fail_free;
-    }
+	ethash_io_mutable_name(ETHASH_REVISION, &seedhash, mutable_name);
+	char* tmpfile = ethash_io_create_filename(dirname, mutable_name, strlen(mutable_name));
+	if (!tmpfile) {
+		goto end;
+	}
 
-    *data = temp_data_ptr;
-	*data_size = (size_t)params->full_size;
-    return true;
+	FILE *f;
+	if (!force_create) {
+		// try to open the file
+		f = ethash_fopen(tmpfile, "rb+");
+		if (f) {
+			size_t found_size;
+			if (!ethash_file_size(f, &found_size)) {
+				fclose(f);
+				goto free_memo;
+			}
+			if (file_size != found_size - ETHASH_DAG_MAGIC_NUM_SIZE) {
+				fclose(f);
+				ret = ETHASH_IO_MEMO_SIZE_MISMATCH;
+				goto free_memo;
+			}
+			// compare the magic number, no need to care about endianess since it's local
+			uint64_t magic_num;
+			if (fread(&magic_num, ETHASH_DAG_MAGIC_NUM_SIZE, 1, f) != 1) {
+				// I/O error
+				fclose(f);
+				ret = ETHASH_IO_MEMO_SIZE_MISMATCH;
+				goto free_memo;
+			}
+			if (magic_num != ETHASH_DAG_MAGIC_NUM) {
+				fclose(f);
+				ret = ETHASH_IO_MEMO_SIZE_MISMATCH;
+				goto free_memo;
+			}
+			ret = ETHASH_IO_MEMO_MATCH;
+			goto set_file;
+		}
+	}
+	
+	// file does not exist, will need to be created
+	f = ethash_fopen(tmpfile, "wb+");
+	if (!f) {
+		goto free_memo;
+	}
+	// make sure it's of the proper size
+	if (fseek(f, (long int)(file_size + ETHASH_DAG_MAGIC_NUM_SIZE - 1), SEEK_SET) != 0) {
+		fclose(f);
+		goto free_memo;
+	}
+	fputc('\n', f);
+	fflush(f);
+	ret = ETHASH_IO_MEMO_MISMATCH;
+	goto set_file;
 
-fail_free:
-    free(temp_data_ptr);
+	ret = ETHASH_IO_MEMO_MATCH;
+set_file:
+	*output_file = f;
+free_memo:
+	free(tmpfile);
 end:
-    return false;
+	return ret;
 }
-
-#undef PASS_ARR

libethash/io.h

@@ -22,94 +22,163 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <stdbool.h>
+#include <stdio.h>
+#ifdef __cplusplus
+#define __STDC_FORMAT_MACROS 1
+#endif
+#include <inttypes.h>
+#include "endian.h"
 #include "ethash.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-
-typedef struct ethash_blockhash { uint8_t b[32]; } ethash_blockhash_t;
-
-static const char DAG_FILE_NAME[] = "full";
-static const char DAG_MEMO_NAME[] = "full.info";
-// MSVC thinks that "static const unsigned int" is not a compile time variable. Sorry for the #define :(
-#define DAG_MEMO_BYTESIZE 36
-
+// Maximum size for mutable part of DAG file name
+// 6 is for "full-R", the suffix of the filename
+// 10 is for maximum number of digits of a uint32_t (for REVISION)
+// 1 is for - and 16 is for the first 16 hex digits for first 8 bytes of
+// the seedhash and last 1 is for the null terminating character
+// Reference: https://github.com/ethereum/wiki/wiki/Ethash-DAG
+#define DAG_MUTABLE_NAME_MAX_SIZE (6 + 10 + 1 + 16 + 1)
 /// Possible return values of @see ethash_io_prepare
 enum ethash_io_rc {
-    ETHASH_IO_FAIL = 0,      ///< There has been an IO failure
-    ETHASH_IO_MEMO_MISMATCH, ///< Memo file either did not exist or there was content mismatch
-    ETHASH_IO_MEMO_MATCH,    ///< Memo file existed and contents matched. No need to do anything
+	ETHASH_IO_FAIL = 0,           ///< There has been an IO failure
+	ETHASH_IO_MEMO_SIZE_MISMATCH, ///< DAG with revision/hash match, but file size was wrong.
+	ETHASH_IO_MEMO_MISMATCH,      ///< The DAG file did not exist or there was revision/hash mismatch
+	ETHASH_IO_MEMO_MATCH,         ///< DAG file existed and revision/hash matched. No need to do anything
 };
 
+// small hack for windows. I don't feel I should use va_args and forward just
+// to have this one function properly cross-platform abstracted
+#if defined(_WIN32) && !defined(__GNUC__)
+#define snprintf(...) sprintf_s(__VA_ARGS__)
+#endif
+
 /**
  * Prepares io for ethash
  *
- * Create the DAG directory if it does not exist, and check if the memo file matches.
- * If it does not match then it's deleted to pave the way for @ref ethash_io_write()
+ * Create the DAG directory and the DAG file if they don't exist.
  *
- * @param dirname        A null terminated c-string of the path of the ethash
- *                       data directory. If it does not exist it's created.
- * @param seedhash       The seedhash of the current block number
- * @return               For possible return values @see enum ethash_io_rc
+ * @param[in] dirname        A null terminated c-string of the path of the ethash
+ *                           data directory. If it does not exist it's created.
+ * @param[in] seedhash       The seedhash of the current block number, used in the
+ *                           naming of the file as can be seen from the spec at:
+ *                           https://github.com/ethereum/wiki/wiki/Ethash-DAG
+ * @param[out] output_file   If there was no failure then this will point to an open
+ *                           file descriptor. User is responsible for closing it.
+ *                           In the case of memo match then the file is open on read
+ *                           mode, while on the case of mismatch a new file is created
+ *                           on write mode
+ * @param[in] file_size      The size that the DAG file should have on disk
+ * @param[out] force_create  If true then there is no check to see if the file
+ *                           already exists
+ * @return                   For possible return values @see enum ethash_io_rc
  */
-enum ethash_io_rc ethash_io_prepare(char const *dirname, ethash_blockhash_t seedhash);
+enum ethash_io_rc ethash_io_prepare(
+	char const* dirname,
+	ethash_h256_t const seedhash,
+	FILE** output_file,
+	uint64_t file_size,
+	bool force_create
+);
+
 /**
- * Fully computes data and writes it to the file on disk.
+ * An fopen wrapper for no-warnings crossplatform fopen.
  *
- * This function should be called after @see ethash_io_prepare() and only if
- * its return value is @c ETHASH_IO_MEMO_MISMATCH. Will write both the full data
- * and the memo file.
+ * Msvc compiler considers fopen to be insecure and suggests to use their
+ * alternative. This is a wrapper for this alternative. Another way is to
+ * #define _CRT_SECURE_NO_WARNINGS, but disabling all security warnings does
+ * not sound like a good idea.
  *
- * @param[in] dirname        A null terminated c-string of the path of the ethash
- *                           data directory. Has to exist.
- * @param[in] params         An ethash_params object containing the full size
- *                           and the cache size
- * @param[in] seedhash       The seedhash of the current block number
- * @param[in] cache          The cache data. Would have usually been calulated by
- *                           @see ethash_prep_light().
- * @param[out] data          Pass a pointer to uint8_t by reference here. If the
- *                           function is succesfull then this point to the allocated
- *                           data calculated by @see ethash_prep_full(). Memory
- *                           ownership is transfered to the callee. Remember that
- *                           you eventually need to free this with a call to free().
- * @param[out] data_size     Pass a size_t by value. If the function is succesfull
- *                           then this will contain the number of bytes allocated
- *                           for @a data.
- * @return                   True for success and false in case of failure.
+ * @param file_name        The path to the file to open
+ * @param mode             Opening mode. Check fopen()
+ * @return                 The FILE* or NULL in failure
  */
-bool ethash_io_write(char const *dirname,
-                     ethash_params const* params,
-                     ethash_blockhash_t seedhash,
-                     void const* cache,
-                     uint8_t **data,
-                     size_t *data_size);
+FILE* ethash_fopen(char const* file_name, char const* mode);
 
-static inline void ethash_io_serialize_info(uint32_t revision,
-                                            ethash_blockhash_t seed_hash,
-                                            char *output)
-{
-    // if .info is only consumed locally we don't really care about endianess
-    memcpy(output, &revision, 4);
-    memcpy(output + 4, &seed_hash, 32);
-}
+/**
+ * An strncat wrapper for no-warnings crossplatform strncat.
+ *
+ * Msvc compiler considers strncat to be insecure and suggests to use their
+ * alternative. This is a wrapper for this alternative. Another way is to
+ * #define _CRT_SECURE_NO_WARNINGS, but disabling all security warnings does
+ * not sound like a good idea.
+ *
+ * @param des              Destination buffer
+ * @param dest_size        Maximum size of the destination buffer. This is the
+ *                         extra argument for the MSVC secure strncat
+ * @param src              Souce buffer
+ * @param count            Number of bytes to copy from source
+ * @return                 If all is well returns the dest buffer. If there is an
+ *                         error returns NULL
+ */
+char* ethash_strncat(char* dest, size_t dest_size, char const* src, size_t count);
 
-static inline char *ethash_io_create_filename(char const *dirname,
-                                              char const* filename,
-                                              size_t filename_length)
-{
-    // in C the cast is not needed, but a C++ compiler will complain for invalid conversion
-    char *name = (char*)malloc(strlen(dirname) + filename_length);
-    if (!name) {
-        return NULL;
-    }
+/**
+ * A cross-platform mkdir wrapper to create a directory or assert it's there
+ *
+ * @param dirname        The full path of the directory to create
+ * @return               true if the directory was created or if it already
+ *                       existed
+ */
+bool ethash_mkdir(char const* dirname);
 
-    name[0] = '\0';
-    strcat(name, dirname);
-    strcat(name, filename);
-    return name;
-}
+/**
+ * Get a file's size
+ *
+ * @param[in] f        The open file stream whose size to get
+ * @param[out] size    Pass a size_t by reference to contain the file size
+ * @return             true in success and false if there was a failure
+ */
+bool ethash_file_size(FILE* f, size_t* ret_size);
+
+/**
+ * Get a file descriptor number from a FILE stream
+ *
+ * @param f            The file stream whose fd to get
+ * @return             Platform specific fd handler
+ */
+int ethash_fileno(FILE* f);
+
+/**
+ * Create the filename for the DAG.
+ *
+ * @param dirname            The directory name in which the DAG file should reside
+ *                           If it does not end with a directory separator it is appended.
+ * @param filename           The actual name of the file
+ * @param filename_length    The length of the filename in bytes
+ * @return                   A char* containing the full name. User must deallocate.
+ */
+char* ethash_io_create_filename(
+	char const* dirname,
+	char const* filename,
+	size_t filename_length
+);
+
+/**
+ * Gets the default directory name for the DAG depending on the system
+ *
+ * The spec defining this directory is here: https://github.com/ethereum/wiki/wiki/Ethash-DAG
+ *
+ * @param[out] strbuf          A string buffer of sufficient size to keep the
+ *                             null termninated string of the directory name
+ * @param[in]  buffsize        Size of @a strbuf in bytes
+ * @return                     true for success and false otherwise
+ */
+bool ethash_get_default_dirname(char* strbuf, size_t buffsize);
 
+static inline bool ethash_io_mutable_name(
+	uint32_t revision,
+	ethash_h256_t const* seed_hash,
+	char* output
+)
+{
+    uint64_t hash = *((uint64_t*)seed_hash);
+#if LITTLE_ENDIAN == BYTE_ORDER
+    hash = ethash_swap_u64(hash);
+#endif
+    return snprintf(output, DAG_MUTABLE_NAME_MAX_SIZE, "full-R%u-%016" PRIx64, revision, hash) >= 0;
+}
 
 #ifdef __cplusplus
 }

libethash/io_posix.c

@@ -27,50 +27,76 @@
 #include <stdio.h>
 #include <unistd.h>
 
-enum ethash_io_rc ethash_io_prepare(char const *dirname, ethash_blockhash_t seedhash)
+FILE* ethash_fopen(char const* file_name, char const* mode)
 {
-    char read_buffer[DAG_MEMO_BYTESIZE];
-    char expect_buffer[DAG_MEMO_BYTESIZE];
-    enum ethash_io_rc ret = ETHASH_IO_FAIL;
+	return fopen(file_name, mode);
+}
 
-    // assert directory exists, full owner permissions and read/search for others
-    int rc = mkdir(dirname, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-    if (rc == -1 && errno != EEXIST) {
-        goto end;
-    }
+char* ethash_strncat(char* dest, size_t dest_size, char const* src, size_t count)
+{
+	return strlen(dest) + count + 1 <= dest_size ? strncat(dest, src, count) : NULL;
+}
 
-    char *memofile = ethash_io_create_filename(dirname, DAG_MEMO_NAME, sizeof(DAG_MEMO_NAME));
-    if (!memofile) {
-        goto end;
-    }
+bool ethash_mkdir(char const* dirname)
+{
+	int rc = mkdir(dirname, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+	return rc != -1 || errno == EEXIST;
+}
 
-    // try to open memo file
-    FILE *f = fopen(memofile, "rb");
-    if (!f) {
-        // file does not exist, so no checking happens. All is fine.
-        ret = ETHASH_IO_MEMO_MISMATCH;
-        goto free_memo;
-    }
+int ethash_fileno(FILE *f)
+{
+	return fileno(f);
+}
 
-    if (fread(read_buffer, 1, DAG_MEMO_BYTESIZE, f) != DAG_MEMO_BYTESIZE) {
-        goto close;
-    }
+char* ethash_io_create_filename(
+	char const* dirname,
+	char const* filename,
+	size_t filename_length
+)
+{
+	size_t dirlen = strlen(dirname);
+	size_t dest_size = dirlen + filename_length + 1;
+	if (dirname[dirlen] != '/') {
+		dest_size += 1;
+	}
+	char* name = malloc(dest_size);
+	if (!name) {
+		return NULL;
+	}
 
-    ethash_io_serialize_info(ETHASH_REVISION, seedhash, expect_buffer);
-    if (memcmp(read_buffer, expect_buffer, DAG_MEMO_BYTESIZE) != 0) {
-        // we have different memo contents so delete the memo file
-        if (unlink(memofile) != 0) {
-            goto close;
-        }
-        ret = ETHASH_IO_MEMO_MISMATCH;
-    }
+	name[0] = '\0';
+	ethash_strncat(name, dest_size, dirname, dirlen);
+	if (dirname[dirlen] != '/') {
+		ethash_strncat(name, dest_size, "/", 1);
+	}
+	ethash_strncat(name, dest_size, filename, filename_length);
+	return name;
+}
 
-    ret = ETHASH_IO_MEMO_MATCH;
+bool ethash_file_size(FILE* f, size_t* ret_size)
+{
+	struct stat st;
+	int fd;
+	if ((fd = fileno(f)) == -1 || fstat(fd, &st) != 0) {
+		return false;
+	}
+	*ret_size = st.st_size;
+	return true;
+}
 
-close:
-    fclose(f);
-free_memo:
-    free(memofile);
-end:
-    return ret;
+bool ethash_get_default_dirname(char* strbuf, size_t buffsize)
+{
+	static const char dir_suffix[] = ".ethash/";
+	strbuf[0] = '\0';
+	char* home_dir = getenv("HOME");
+	size_t len = strlen(home_dir);
+	if (!ethash_strncat(strbuf, buffsize, home_dir, len)) {
+		return false;
+	}
+	if (home_dir[len] != '/') {
+		if (!ethash_strncat(strbuf, buffsize, "/", 1)) {
+			return false;
+		}
+	}
+	return ethash_strncat(strbuf, buffsize, dir_suffix, sizeof(dir_suffix));
 }

libethash/io_win32.c

@@ -23,51 +23,78 @@
 #include <direct.h>
 #include <errno.h>
 #include <stdio.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <shlobj.h>
 
-enum ethash_io_rc ethash_io_prepare(char const *dirname, ethash_blockhash_t seedhash)
+FILE* ethash_fopen(char const* file_name, char const* mode)
 {
-    char read_buffer[DAG_MEMO_BYTESIZE];
-    char expect_buffer[DAG_MEMO_BYTESIZE];
-    enum ethash_io_rc ret = ETHASH_IO_FAIL;
+	FILE* f;
+	return fopen_s(&f, file_name, mode) == 0 ? f : NULL;
+}
 
-    // assert directory exists
-    int rc = _mkdir(dirname);
-    if (rc == -1 && errno != EEXIST) {
-        goto end;
-    }
+char* ethash_strncat(char* dest, size_t dest_size, char const* src, size_t count)
+{
+	return strncat_s(dest, dest_size, src, count) == 0 ? dest : NULL;
+}
 
-    char *memofile = ethash_io_create_filename(dirname, DAG_MEMO_NAME, sizeof(DAG_MEMO_NAME));
-    if (!memofile) {
-        goto end;
-    }
+bool ethash_mkdir(char const* dirname)
+{
+	int rc = _mkdir(dirname);
+	return rc != -1 || errno == EEXIST;
+}
 
-    // try to open memo file
-    FILE *f = fopen(memofile, "rb");
-    if (!f) {
-        // file does not exist, so no checking happens. All is fine.
-        ret = ETHASH_IO_MEMO_MISMATCH;
-        goto free_memo;
-    }
+int ethash_fileno(FILE* f)
+{
+	return _fileno(f);
+}
 
-    if (fread(read_buffer, 1, DAG_MEMO_BYTESIZE, f) != DAG_MEMO_BYTESIZE) {
-        goto close;
-    }
+char* ethash_io_create_filename(
+	char const* dirname,
+	char const* filename,
+	size_t filename_length
+)
+{
+	size_t dirlen = strlen(dirname);
+	size_t dest_size = dirlen + filename_length + 1;
+	if (dirname[dirlen] != '\\' || dirname[dirlen] != '/') {
+		dest_size += 1;
+	}
+	char* name = malloc(dest_size);
+	if (!name) {
+		return NULL;
+	}
+
+	name[0] = '\0';
+	ethash_strncat(name, dest_size, dirname, dirlen);
+	if (dirname[dirlen] != '\\' || dirname[dirlen] != '/') {
+		ethash_strncat(name, dest_size, "\\", 1);
+	}
+	ethash_strncat(name, dest_size, filename, filename_length);
+	return name;
+}
 
-    ethash_io_serialize_info(ETHASH_REVISION, seedhash, expect_buffer);
-    if (memcmp(read_buffer, expect_buffer, DAG_MEMO_BYTESIZE) != 0) {
-        // we have different memo contents so delete the memo file
-        if (_unlink(memofile) != 0) {
-            goto close;
-        }
-        ret = ETHASH_IO_MEMO_MISMATCH;
-    }
+bool ethash_file_size(FILE* f, size_t* ret_size)
+{
+	struct _stat st;
+	int fd;
+	if ((fd = _fileno(f)) == -1 || _fstat(fd, &st) != 0) {
+		return false;
+	}
+	*ret_size = st.st_size;
+	return true;
+}
 
-    ret = ETHASH_IO_MEMO_MATCH;
+bool ethash_get_default_dirname(char* strbuf, size_t buffsize)
+{
+	static const char dir_suffix[] = "Appdata\\Ethash\\";
+	strbuf[0] = '\0';
+	if (!SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_PROFILE, NULL, 0, (WCHAR*)strbuf))) {
+		return false;
+	}
+	if (!ethash_strncat(strbuf, buffsize, "\\", 1)) {
+		return false;
+	}
 
-close:
-    fclose(f);
-free_memo:
-    free(memofile);
-end:
-    return ret;
+	return ethash_strncat(strbuf, buffsize, dir_suffix, sizeof(dir_suffix));
 }

libethash/mmap.h

@@ -0,0 +1,47 @@
+/*
+  This file is part of ethash.
+
+  ethash is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  ethash is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with ethash.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file mmap.h
+ * @author Lefteris Karapetsas <lefteris@ethdev.com>
+ * @date 2015
+ */
+#pragma once
+#if defined(__MINGW32__) || defined(_WIN32)
+#include <sys/types.h>
+
+#define PROT_READ     0x1
+#define PROT_WRITE    0x2
+/* This flag is only available in WinXP+ */
+#ifdef FILE_MAP_EXECUTE
+#define PROT_EXEC     0x4
+#else
+#define PROT_EXEC        0x0
+#define FILE_MAP_EXECUTE 0
+#endif
+
+#define MAP_SHARED    0x01
+#define MAP_PRIVATE   0x02
+#define MAP_ANONYMOUS 0x20
+#define MAP_ANON      MAP_ANONYMOUS
+#define MAP_FAILED    ((void *) -1)
+
+void* mmap(void* start, size_t length, int prot, int flags, int fd, off_t offset);
+void munmap(void* addr, size_t length);
+#else // posix, yay! ^_^
+#include <sys/mman.h>
+#endif
+
+

libethash/mmap_win32.c

@@ -0,0 +1,84 @@
+/* mmap() replacement for Windows
+ *
+ * Author: Mike Frysinger <vapier@gentoo.org>
+ * Placed into the public domain
+ */
+
+/* References:
+ * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
+ * CloseHandle:       http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
+ * MapViewOfFile:     http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
+ * UnmapViewOfFile:   http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
+ */
+
+#include <io.h>
+#include <windows.h>
+#include "mmap.h"
+
+#ifdef __USE_FILE_OFFSET64
+# define DWORD_HI(x) (x >> 32)
+# define DWORD_LO(x) ((x) & 0xffffffff)
+#else
+# define DWORD_HI(x) (0)
+# define DWORD_LO(x) (x)
+#endif
+
+void* mmap(void* start, size_t length, int prot, int flags, int fd, off_t offset)
+{
+	if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC))
+		return MAP_FAILED;
+	if (fd == -1) {
+		if (!(flags & MAP_ANON) || offset)
+			return MAP_FAILED;
+	} else if (flags & MAP_ANON)
+		return MAP_FAILED;
+
+	DWORD flProtect;
+	if (prot & PROT_WRITE) {
+		if (prot & PROT_EXEC)
+			flProtect = PAGE_EXECUTE_READWRITE;
+		else
+			flProtect = PAGE_READWRITE;
+	} else if (prot & PROT_EXEC) {
+		if (prot & PROT_READ)
+			flProtect = PAGE_EXECUTE_READ;
+		else if (prot & PROT_EXEC)
+			flProtect = PAGE_EXECUTE;
+	} else
+		flProtect = PAGE_READONLY;
+
+	off_t end = length + offset;
+	HANDLE mmap_fd, h;
+	if (fd == -1)
+		mmap_fd = INVALID_HANDLE_VALUE;
+	else
+		mmap_fd = (HANDLE)_get_osfhandle(fd);
+	h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL);
+	if (h == NULL)
+		return MAP_FAILED;
+
+	DWORD dwDesiredAccess;
+	if (prot & PROT_WRITE)
+		dwDesiredAccess = FILE_MAP_WRITE;
+	else
+		dwDesiredAccess = FILE_MAP_READ;
+	if (prot & PROT_EXEC)
+		dwDesiredAccess |= FILE_MAP_EXECUTE;
+	if (flags & MAP_PRIVATE)
+		dwDesiredAccess |= FILE_MAP_COPY;
+	void *ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length);
+	if (ret == NULL) {
+		ret = MAP_FAILED;
+	}
+	// since we are handling the file ourselves with fd, close the Windows Handle here
+	CloseHandle(h);
+	return ret;
+}
+
+void munmap(void* addr, size_t length)
+{
+	UnmapViewOfFile(addr);
+}
+
+#undef DWORD_HI
+#undef DWORD_LO

libethash/sha3.c

@@ -17,65 +17,65 @@
 
 /*** Constants. ***/
 static const uint8_t rho[24] = \
-  { 1,  3,   6, 10, 15, 21,
-        28, 36, 45, 55,  2, 14,
-        27, 41, 56,  8, 25, 43,
-        62, 18, 39, 61, 20, 44};
+	{ 1,  3,   6, 10, 15, 21,
+	  28, 36, 45, 55,  2, 14,
+	  27, 41, 56,  8, 25, 43,
+	  62, 18, 39, 61, 20, 44};
 static const uint8_t pi[24] = \
-  {10,  7, 11, 17, 18, 3,
-        5, 16,  8, 21, 24, 4,
-        15, 23, 19, 13, 12, 2,
-        20, 14, 22,  9, 6,  1};
+	{10,  7, 11, 17, 18, 3,
+	 5, 16,  8, 21, 24, 4,
+	 15, 23, 19, 13, 12, 2,
+	 20, 14, 22,  9, 6,  1};
 static const uint64_t RC[24] = \
-  {1ULL, 0x8082ULL, 0x800000000000808aULL, 0x8000000080008000ULL,
-        0x808bULL, 0x80000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL,
-        0x8aULL, 0x88ULL, 0x80008009ULL, 0x8000000aULL,
-        0x8000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL,
-        0x8000000000008002ULL, 0x8000000000000080ULL, 0x800aULL, 0x800000008000000aULL,
-        0x8000000080008081ULL, 0x8000000000008080ULL, 0x80000001ULL, 0x8000000080008008ULL};
+	{1ULL, 0x8082ULL, 0x800000000000808aULL, 0x8000000080008000ULL,
+	 0x808bULL, 0x80000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL,
+	 0x8aULL, 0x88ULL, 0x80008009ULL, 0x8000000aULL,
+	 0x8000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL,
+	 0x8000000000008002ULL, 0x8000000000000080ULL, 0x800aULL, 0x800000008000000aULL,
+	 0x8000000080008081ULL, 0x8000000000008080ULL, 0x80000001ULL, 0x8000000080008008ULL};
 
 /*** Helper macros to unroll the permutation. ***/
 #define rol(x, s) (((x) << s) | ((x) >> (64 - s)))
 #define REPEAT6(e) e e e e e e
 #define REPEAT24(e) REPEAT6(e e e e)
 #define REPEAT5(e) e e e e e
-#define FOR5(v, s, e) \
-  v = 0;            \
-  REPEAT5(e; v += s;)
+#define FOR5(v, s, e)							\
+	v = 0;										\
+	REPEAT5(e; v += s;)
 
 /*** Keccak-f[1600] ***/
 static inline void keccakf(void* state) {
-    uint64_t* a = (uint64_t*)state;
-    uint64_t b[5] = {0};
-    uint64_t t = 0;
-    uint8_t x, y;
+	uint64_t* a = (uint64_t*)state;
+	uint64_t b[5] = {0};
+	uint64_t t = 0;
+	uint8_t x, y;
 
-    for (int i = 0; i < 24; i++) {
-        // Theta
-        FOR5(x, 1,
-                b[x] = 0;
-                FOR5(y, 5,
-                        b[x] ^= a[x + y]; ))
-        FOR5(x, 1,
-                FOR5(y, 5,
-                        a[y + x] ^= b[(x + 4) % 5] ^ rol(b[(x + 1) % 5], 1); ))
-        // Rho and pi
-        t = a[1];
-        x = 0;
-        REPEAT24(b[0] = a[pi[x]];
-                a[pi[x]] = rol(t, rho[x]);
-                t = b[0];
-                x++; )
-        // Chi
-        FOR5(y,
-                5,
-                FOR5(x, 1,
-                        b[x] = a[y + x];)
-                FOR5(x, 1,
-                a[y + x] = b[x] ^ ((~b[(x + 1) % 5]) & b[(x + 2) % 5]); ))
-        // Iota
-        a[0] ^= RC[i];
-    }
+	for (int i = 0; i < 24; i++) {
+		// Theta
+		FOR5(x, 1,
+				b[x] = 0;
+				FOR5(y, 5,
+						b[x] ^= a[x + y]; ))
+		FOR5(x, 1,
+				FOR5(y, 5,
+						a[y + x] ^= b[(x + 4) % 5] ^ rol(b[(x + 1) % 5], 1); ))
+		// Rho and pi
+		t = a[1];
+		x = 0;
+		REPEAT24(b[0] = a[pi[x]];
+				a[pi[x]] = rol(t, rho[x]);
+				t = b[0];
+				x++; )
+		// Chi
+		FOR5(y,
+				5,
+				FOR5(x, 1,
+						b[x] = a[y + x];)
+				FOR5(x, 1,
+				a[y + x] = b[x] ^ ((~b[(x + 1) % 5]) & b[(x + 2) % 5]); ))
+		// Iota
+		a[0] ^= RC[i];
+	}
 }
 
 /******** The FIPS202-defined functions. ********/
@@ -83,20 +83,20 @@ static inline void keccakf(void* state) {
 /*** Some helper macros. ***/
 
 #define _(S) do { S } while (0)
-#define FOR(i, ST, L, S) \
-  _(for (size_t i = 0; i < L; i += ST) { S; })
-#define mkapply_ds(NAME, S)                                          \
-  static inline void NAME(uint8_t* dst,                              \
-                          const uint8_t* src,                        \
-                          size_t len) {                              \
-    FOR(i, 1, len, S);                                               \
-  }
-#define mkapply_sd(NAME, S)                                          \
-  static inline void NAME(const uint8_t* src,                        \
-                          uint8_t* dst,                              \
-                          size_t len) {                              \
-    FOR(i, 1, len, S);                                               \
-  }
+#define FOR(i, ST, L, S)							\
+	_(for (size_t i = 0; i < L; i += ST) { S; })
+#define mkapply_ds(NAME, S)						\
+	static inline void NAME(uint8_t* dst,			\
+		const uint8_t* src,						\
+		size_t len) {								\
+		FOR(i, 1, len, S);							\
+	}
+#define mkapply_sd(NAME, S)						\
+	static inline void NAME(const uint8_t* src,	\
+		uint8_t* dst,								\
+		size_t len) {								\
+		FOR(i, 1, len, S);							\
+	}
 
 mkapply_ds(xorin, dst[i] ^= src[i])  // xorin
 mkapply_sd(setout, dst[i] = src[i])  // setout
@@ -105,47 +105,47 @@ mkapply_sd(setout, dst[i] = src[i])  // setout
 #define Plen 200
 
 // Fold P*F over the full blocks of an input.
-#define foldP(I, L, F) \
-  while (L >= rate) {  \
-    F(a, I, rate);     \
-    P(a);              \
-    I += rate;         \
-    L -= rate;         \
-  }
+#define foldP(I, L, F)								\
+	while (L >= rate) {							\
+		F(a, I, rate);								\
+		P(a);										\
+		I += rate;									\
+		L -= rate;									\
+	}
 
 /** The sponge-based hash construction. **/
 static inline int hash(uint8_t* out, size_t outlen,
-        const uint8_t* in, size_t inlen,
-        size_t rate, uint8_t delim) {
-    if ((out == NULL) || ((in == NULL) && inlen != 0) || (rate >= Plen)) {
-        return -1;
-    }
-    uint8_t a[Plen] = {0};
-    // Absorb input.
-    foldP(in, inlen, xorin);
-    // Xor in the DS and pad frame.
-    a[inlen] ^= delim;
-    a[rate - 1] ^= 0x80;
-    // Xor in the last block.
-    xorin(a, in, inlen);
-    // Apply P
-    P(a);
-    // Squeeze output.
-    foldP(out, outlen, setout);
-    setout(a, out, outlen);
-    memset(a, 0, 200);
-    return 0;
+		const uint8_t* in, size_t inlen,
+		size_t rate, uint8_t delim) {
+	if ((out == NULL) || ((in == NULL) && inlen != 0) || (rate >= Plen)) {
+		return -1;
+	}
+	uint8_t a[Plen] = {0};
+	// Absorb input.
+	foldP(in, inlen, xorin);
+	// Xor in the DS and pad frame.
+	a[inlen] ^= delim;
+	a[rate - 1] ^= 0x80;
+	// Xor in the last block.
+	xorin(a, in, inlen);
+	// Apply P
+	P(a);
+	// Squeeze output.
+	foldP(out, outlen, setout);
+	setout(a, out, outlen);
+	memset(a, 0, 200);
+	return 0;
 }
 
-#define defsha3(bits)                                             \
-  int sha3_##bits(uint8_t* out, size_t outlen,                    \
-                  const uint8_t* in, size_t inlen) {              \
-    if (outlen > (bits/8)) {                                      \
-      return -1;                                                  \
-    }                                                             \
-    return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x01);  \
-  }
+#define defsha3(bits)													\
+	int sha3_##bits(uint8_t* out, size_t outlen,						\
+		const uint8_t* in, size_t inlen) {								\
+		if (outlen > (bits/8)) {										\
+			return -1;                                                  \
+		}																\
+		return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x01);	\
+	}
 
 /*** FIPS202 SHA3 FOFs ***/
 defsha3(256)
-defsha3(512)
+defsha3(512)

libethash/sha3.h

@@ -8,20 +8,24 @@ extern "C" {
 #include <stdint.h>
 #include <stdlib.h>
 
+struct ethash_h256;
+
 #define decsha3(bits) \
-  int sha3_##bits(uint8_t*, size_t, const uint8_t*, size_t);
+	int sha3_##bits(uint8_t*, size_t, uint8_t const*, size_t);
 
 decsha3(256)
 decsha3(512)
 
-static inline void SHA3_256(uint8_t * const ret, uint8_t const *data, const size_t size) {
-    sha3_256(ret, 32, data, size);
+static inline void SHA3_256(struct ethash_h256 const* ret, uint8_t const* data, size_t const size)
+{
+	sha3_256((uint8_t*)ret, 32, data, size);
 }
 
-static inline void SHA3_512(uint8_t * const ret, uint8_t const *data, const size_t size) {
-    sha3_512(ret, 64, data, size);
+static inline void SHA3_512(uint8_t* ret, uint8_t const* data, size_t const size)
+{
+	sha3_512(ret, 64, data, size);
 }
 
 #ifdef __cplusplus
 }
-#endif
+#endif

libethash/sha3_cryptopp.cpp

@@ -19,16 +19,19 @@
 * @author Tim Hughes <tim@twistedfury.com>
 * @date 2015
 */
-
 #include <stdint.h>
 #include <cryptopp/sha3.h>
 
 extern "C" {
-void SHA3_256(uint8_t *const ret, const uint8_t *data, size_t size) {
-  CryptoPP::SHA3_256().CalculateDigest(ret, data, size);
+struct ethash_h256;
+typedef struct ethash_h256 ethash_h256_t;
+void SHA3_256(ethash_h256_t const* ret, uint8_t const* data, size_t size)
+{
+	CryptoPP::SHA3_256().CalculateDigest((uint8_t*)ret, data, size);
 }
 
-void SHA3_512(uint8_t *const ret, const uint8_t *data, size_t size) {
-  CryptoPP::SHA3_512().CalculateDigest(ret, data, size);
+void SHA3_512(uint8_t* const ret, uint8_t const* data, size_t size)
+{
+	CryptoPP::SHA3_512().CalculateDigest(ret, data, size);
+}
 }
-}

libethash/sha3_cryptopp.h

@@ -2,14 +2,17 @@
 
 #include "compiler.h"
 #include <stdint.h>
+#include <stdlib.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-void SHA3_256(uint8_t *const ret, const uint8_t *data, size_t size);
-void SHA3_512(uint8_t *const ret, const uint8_t *data, size_t size);
+struct ethash_h256;
+
+void SHA3_256(struct ethash_h256 const* ret, uint8_t const* data, size_t size);
+void SHA3_512(uint8_t* const ret, uint8_t const* data, size_t size);
 
 #ifdef __cplusplus
 }
-#endif
+#endif

libethash/util.h

@@ -27,9 +27,9 @@ extern "C" {
 #endif
 
 #ifdef _MSC_VER
-void debugf(const char *str, ...);
+void debugf(char const* str, ...);
 #else
-#define debugf(...) fprintf(stderr, __VA_ARGS__)
+#define debugf printf
 #endif
 
 static inline uint32_t min_u32(uint32_t a, uint32_t b)

libethash/util_win32.c

@@ -0,0 +1,38 @@
+/*
+  This file is part of cpp-ethereum.
+
+  cpp-ethereum is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  cpp-ethereum is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file util.c
+ * @author Tim Hughes <tim@twistedfury.com>
+ * @date 2015
+ */
+#include <stdarg.h>
+#include <stdio.h>
+#include "util.h"
+
+
+// foward declare without all of Windows.h
+__declspec(dllimport) void __stdcall OutputDebugStringA(char const* lpOutputString);
+
+void debugf(char const* str, ...)
+{
+	va_list args;
+	va_start(args, str);
+
+	char buf[1<<16];
+	_vsnprintf_s(buf, sizeof(buf), sizeof(buf), str, args);
+	buf[sizeof(buf)-1] = '\0';
+	OutputDebugStringA(buf);
+}

libethcore/BlockInfo.cpp

@@ -228,7 +228,7 @@ u256 BlockInfo::selectGasLimit(BlockInfo const& _parent) const
 		return c_genesisGasLimit;
 	else
 		// target minimum of 3141592
-		return max<u256>(max<u256>(c_minGasLimit, 3141592), (_parent.gasLimit * (c_gasLimitBoundDivisor - 1) + (_parent.gasUsed * 6 / 5)) / c_gasLimitBoundDivisor);
+		return max<u256>(max<u256>(c_minGasLimit, 3141592), _parent.gasLimit - _parent.gasLimit / c_gasLimitBoundDivisor + 1 + (_parent.gasUsed * 6 / 5) / c_gasLimitBoundDivisor);
 }
 
 u256 BlockInfo::calculateDifficulty(BlockInfo const& _parent) const
@@ -246,9 +246,9 @@ void BlockInfo::verifyParent(BlockInfo const& _parent) const
 		BOOST_THROW_EXCEPTION(InvalidDifficulty() << RequirementError((bigint)calculateDifficulty(_parent), (bigint)difficulty));
 
 	if (gasLimit < c_minGasLimit ||
-		gasLimit < _parent.gasLimit * (c_gasLimitBoundDivisor - 1) / c_gasLimitBoundDivisor ||
-		gasLimit > _parent.gasLimit * (c_gasLimitBoundDivisor + 1) / c_gasLimitBoundDivisor)
-		BOOST_THROW_EXCEPTION(InvalidGasLimit() << errinfo_min((bigint)_parent.gasLimit * (c_gasLimitBoundDivisor - 1) / c_gasLimitBoundDivisor) << errinfo_got((bigint)gasLimit) << errinfo_max((bigint)_parent.gasLimit * (c_gasLimitBoundDivisor + 1) / c_gasLimitBoundDivisor));
+		gasLimit <= _parent.gasLimit - _parent.gasLimit / c_gasLimitBoundDivisor ||
+		gasLimit >= _parent.gasLimit + _parent.gasLimit / c_gasLimitBoundDivisor)
+		BOOST_THROW_EXCEPTION(InvalidGasLimit() << errinfo_min((bigint)_parent.gasLimit - _parent.gasLimit / c_gasLimitBoundDivisor) << errinfo_got((bigint)gasLimit) << errinfo_max((bigint)_parent.gasLimit + _parent.gasLimit / c_gasLimitBoundDivisor));
 
 	// Check timestamp is after previous timestamp.
 	if (parentHash)

libethcore/Common.h

@@ -136,5 +136,16 @@ private:
 
 using Handler = std::shared_ptr<Signal::HandlerAux>;
 
+struct TransactionSkeleton
+{
+	bool creation = false;
+	Address from;
+	Address to;
+	u256 value;
+	bytes data;
+	u256 gas = UndefinedU256;
+	u256 gasPrice = UndefinedU256;
+};
+
 }
 }

libethcore/CommonJS.cpp

@@ -26,8 +26,6 @@
 namespace dev
 {
 
-const u256 UndefinedU256 = ~(u256)0;
-
 Address toAddress(std::string const& _sn)
 {
 	if (_sn.size() == 40)

libethcore/CommonJS.h

@@ -48,8 +48,6 @@ inline Address jsToAddress(std::string const& _s) { return jsToFixed<sizeof(dev:
 /// Convert u256 into user-readable string. Returns int/hex value of 64 bits int, hex of 160 bits FixedHash. As a fallback try to handle input as h256.
 std::string prettyU256(u256 _n, bool _abridged = true);
 
-extern const u256 UndefinedU256;
-
 }
 
 
@@ -59,18 +57,6 @@ namespace dev
 namespace eth
 {
 
-
-struct TransactionSkeleton
-{
-	bool creation = false;
-	Address from;
-	Address to;
-	u256 value;
-	bytes data;
-	u256 gas = UndefinedU256;
-	u256 gasPrice = UndefinedU256;
-};
-
 /// Convert to a block number, a bit like jsToInt, except that it correctly recognises "pending" and "latest".
 BlockNumber jsToBlockNumber(std::string const& _js);
 

libethcore/Ethash.cpp

@@ -36,6 +36,7 @@
 #include <libdevcrypto/CryptoPP.h>
 #include <libdevcrypto/FileSystem.h>
 #include <libethash/ethash.h>
+#include <libethash/internal.h>
 #if ETH_ETHASHCL || !ETH_TRUE
 #include <libethash-cl/ethash_cl_miner.h>
 #endif
@@ -71,12 +72,13 @@ Ethash::WorkPackage Ethash::package(BlockInfo const& _bi)
 	ret.boundary = _bi.boundary();
 	ret.headerHash = _bi.headerHash(WithoutNonce);
 	ret.seedHash = _bi.seedHash();
+	ret.blockNumber = (uint64_t) _bi.number;
 	return ret;
 }
 
-void Ethash::prep(BlockInfo const& _header)
+void Ethash::prep(BlockInfo const& _header, std::function<int(unsigned)> const& _f)
 {
-	EthashAux::full(_header);
+	EthashAux::full((unsigned)_header.number, _f);
 }
 
 bool Ethash::preVerify(BlockInfo const& _header)
@@ -87,10 +89,10 @@ bool Ethash::preVerify(BlockInfo const& _header)
 	h256 boundary = u256((bigint(1) << 256) / _header.difficulty);
 
 	return !!ethash_quick_check_difficulty(
-		_header.headerHash(WithoutNonce).data(),
+		(ethash_h256_t const*)_header.headerHash(WithoutNonce).data(),
 		(uint64_t)(u64)_header.nonce,
-		_header.mixHash.data(),
-		boundary.data());
+		(ethash_h256_t const*)_header.mixHash.data(),
+		(ethash_h256_t const*)boundary.data());
 }
 
 bool Ethash::verify(BlockInfo const& _header)
@@ -133,17 +135,14 @@ void Ethash::CPUMiner::workLoop()
 
 	WorkPackage w = work();
 
-	auto p = EthashAux::params(w.seedHash);
-	auto dag = EthashAux::full(w.seedHash);
-	auto dagPointer = dag->data.data();
-	uint8_t const* headerHashPointer = w.headerHash.data();
+	auto dag = EthashAux::full(w.blockNumber);
 	h256 boundary = w.boundary;
 	unsigned hashCount = 1;
 	for (; !shouldStop(); tryNonce++, hashCount++)
 	{
-		ethash_compute_full(&ethashReturn, dagPointer, &p, headerHashPointer, tryNonce);
-		h256 value = h256(ethashReturn.result, h256::ConstructFromPointer);
-		if (value <= boundary && submitProof(Solution{(Nonce)(u64)tryNonce, h256(ethashReturn.mix_hash, h256::ConstructFromPointer)}))
+		ethashReturn = ethash_full_compute(dag->full, *(ethash_h256_t*)w.headerHash.data(), tryNonce);
+		h256 value = h256((uint8_t*)&ethashReturn.result, h256::ConstructFromPointer);
+		if (value <= boundary && submitProof(Solution{(Nonce)(u64)tryNonce, h256((uint8_t*)&ethashReturn.mix_hash, h256::ConstructFromPointer)}))
 			break;
 		if (!(hashCount % 1000))
 			accumulateHashes(1000);
@@ -285,7 +284,7 @@ Ethash::GPUMiner::~GPUMiner()
 bool Ethash::GPUMiner::report(uint64_t _nonce)
 {
 	Nonce n = (Nonce)(u64)_nonce;
-	Result r = EthashAux::eval(work().seedHash, work().headerHash, n);
+	Result r = EthashAux::eval(work().blockNumber, work().headerHash, n);
 	if (r.value < work().boundary)
 		return submitProof(Solution{n, r.mixHash});
 	return false;
@@ -311,8 +310,11 @@ void Ethash::GPUMiner::workLoop()
 
 			unsigned device = instances() > 1 ? index() : s_deviceId;
 
-			EthashAux::FullType dag = EthashAux::full(m_minerSeed);
-			m_miner->init(dag->data.data(), dag->data.size(), 32, s_platformId, device);
+			if (!EthashAux::computeFull(w.blockNumber))
+				return;
+			EthashAux::FullType dag = EthashAux::full(w.blockNumber);
+			bytesConstRef dagData = dag->data();
+			m_miner->init(dagData.data(), dagData.size(), 32, s_platformId, device);
 		}
 
 		uint64_t upper64OfBoundary = (uint64_t)(u64)((u256)w.boundary >> 192);

libethcore/Ethash.h

@@ -66,14 +66,15 @@ public:
 
 		h256 boundary;
 		h256 headerHash;	///< When h256() means "pause until notified a new work package is available".
-		h256 seedHash;
+		h256 seedHash; /// LTODO: IS this needed now that we use the block number instead?
+		uint64_t blockNumber;
 	};
 
 	static const WorkPackage NullWorkPackage;
 
 	static std::string name();
 	static unsigned revision();
-	static void prep(BlockInfo const& _header);
+	static void prep(BlockInfo const& _header, std::function<int(unsigned)> const& _f = std::function<int(unsigned)>());
 	static bool verify(BlockInfo const& _header);
 	static bool preVerify(BlockInfo const& _header);
 	static WorkPackage package(BlockInfo const& _header);

libethcore/EthashAux.cpp

@@ -33,29 +33,25 @@
 #include <libdevcrypto/CryptoPP.h>
 #include <libdevcrypto/SHA3.h>
 #include <libdevcrypto/FileSystem.h>
+#include <libethash/internal.h>
 #include "BlockInfo.h"
+#include "Exceptions.h"
 using namespace std;
 using namespace chrono;
 using namespace dev;
 using namespace eth;
 
+const char* DAGChannel::name() { return EthGreen "DAG"; }
+
 EthashAux* dev::eth::EthashAux::s_this = nullptr;
 
 EthashAux::~EthashAux()
 {
 }
 
-ethash_params EthashAux::params(BlockInfo const& _header)
-{
-	return params((unsigned)_header.number);
-}
-
-ethash_params EthashAux::params(unsigned _n)
+uint64_t EthashAux::cacheSize(BlockInfo const& _header)
 {
-	ethash_params p;
-	p.cache_size = ethash_get_cachesize(_n);
-	p.full_size = ethash_get_datasize(_n);
-	return p;
+	return ethash_get_cachesize((uint64_t)_header.number);
 }
 
 h256 EthashAux::seedHash(unsigned _number)
@@ -82,7 +78,7 @@ h256 EthashAux::seedHash(unsigned _number)
 	return get()->m_seedHashes[epoch];
 }
 
-ethash_params EthashAux::params(h256 const& _seedHash)
+uint64_t EthashAux::number(h256 const& _seedHash)
 {
 	Guard l(get()->x_epochs);
 	unsigned epoch = 0;
@@ -100,125 +96,136 @@ ethash_params EthashAux::params(h256 const& _seedHash)
 	}
 	else
 		epoch = epochIter->second;
-	return params(epoch * ETHASH_EPOCH_LENGTH);
+	return epoch * ETHASH_EPOCH_LENGTH;
 }
 
 void EthashAux::killCache(h256 const& _s)
 {
-	RecursiveGuard l(x_this);
+	RecursiveGuard l(x_lights);
 	m_lights.erase(_s);
 }
 
 EthashAux::LightType EthashAux::light(BlockInfo const& _header)
 {
-	return light(_header.seedHash());
+	return light((uint64_t)_header.number);
 }
 
-EthashAux::LightType EthashAux::light(h256 const& _seedHash)
+EthashAux::LightType EthashAux::light(uint64_t _blockNumber)
 {
-	RecursiveGuard l(get()->x_this);
-	LightType ret = get()->m_lights[_seedHash];
-	return ret ? ret : (get()->m_lights[_seedHash] = make_shared<LightAllocation>(_seedHash));
+	RecursiveGuard l(get()->x_lights);
+	h256 seedHash = EthashAux::seedHash(_blockNumber);
+	LightType ret = get()->m_lights[seedHash];
+	return ret ? ret : (get()->m_lights[seedHash] = make_shared<LightAllocation>(_blockNumber));
 }
 
-EthashAux::LightAllocation::LightAllocation(h256 const& _seed)
+EthashAux::LightAllocation::LightAllocation(uint64_t _blockNumber)
 {
-	auto p = params(_seed);
-	size = p.cache_size;
-	light = ethash_new_light(&p, _seed.data());
+	light = ethash_light_new(_blockNumber);
+	size = ethash_get_cachesize(_blockNumber);
 }
 
 EthashAux::LightAllocation::~LightAllocation()
 {
-	ethash_delete_light(light);
+	ethash_light_delete(light);
 }
 
-EthashAux::FullType EthashAux::full(BlockInfo const& _header, bytesRef _dest, bool _createIfMissing)
+bytesConstRef EthashAux::LightAllocation::data() const
 {
-	return full(_header.seedHash(), _dest, _createIfMissing);
+	return bytesConstRef((byte const*)light->cache, size);
 }
 
-EthashAux::FullType EthashAux::full(h256 const& _seedHash, bytesRef _dest, bool _createIfMissing)
+EthashAux::FullAllocation::FullAllocation(ethash_light_t _light, ethash_callback_t _cb)
 {
-	RecursiveGuard l(get()->x_this);
-	FullType ret = get()->m_fulls[_seedHash].lock();
-	if (ret && _dest)
-	{
-		assert(ret->data.size() <= _dest.size());
-		ret->data.copyTo(_dest);
-		return FullType();
-	}
-	if (!ret)
-	{
-		// drop our last used cache sine we're allocating another 1GB.
-		get()->m_lastUsedFull.reset();
+	full = ethash_full_new(_light, _cb);
+}
 
-		try {
-			boost::filesystem::create_directories(getDataDir("ethash"));
-		} catch (...) {}
+EthashAux::FullAllocation::~FullAllocation()
+{
+	ethash_full_delete(full);
+}
 
-		auto info = rlpList(Ethash::revision(), _seedHash);
-		std::string oldMemoFile = getDataDir("ethash") + "/full";
-		std::string memoFile = getDataDir("ethash") + "/full-R" + toString(ETHASH_REVISION) + "-" + toHex(_seedHash.ref().cropped(0, 8));
-		if (boost::filesystem::exists(oldMemoFile) && contents(oldMemoFile + ".info") == info)
-		{
-			// memofile valid - rename.
-			boost::filesystem::rename(oldMemoFile, memoFile);
-		}
+bytesConstRef EthashAux::FullAllocation::data() const
+{
+	return bytesConstRef((byte const*)ethash_full_dag(full), size());
+}
 
-		DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile));
-		DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile + ".info"));
+static std::function<int(unsigned)> s_dagCallback;
+static int dagCallbackShim(unsigned _p)
+{
+	clog(DAGChannel) << "Generating DAG file. Progress: " << toString(_p) << "%";
+	return s_dagCallback ? s_dagCallback(_p) : 0;
+}
 
-		ethash_params p = params(_seedHash);
-		assert(!_dest || _dest.size() >= p.full_size);	// must be big enough.
+EthashAux::FullType EthashAux::full(uint64_t _blockNumber, function<int(unsigned)> const& _f)
+{
+	auto l = light(_blockNumber);
+	h256 seedHash = EthashAux::seedHash(_blockNumber);
+	FullType ret;
 
-		bytesRef r = contentsNew(memoFile, _dest);
-		if (!r)
+	DEV_GUARDED(get()->x_fulls)
+		if ((ret = get()->m_fulls[seedHash].lock()))
 		{
-			if (!_createIfMissing)
-				return FullType();
-			// file didn't exist.
-			if (_dest)
-				// buffer was passed in - no insertion into cache nor need to allocate
-				r = _dest;
-			else
-				r = bytesRef(new byte[p.full_size], p.full_size);
-			ethash_prep_full(r.data(), &p, light(_seedHash)->light);
-			writeFile(memoFile, r);
+			get()->m_lastUsedFull = ret;
+			return ret;
 		}
-		if (_dest)
-			return FullType();
-		ret = make_shared<FullAllocation>(r);
-		get()->m_fulls[_seedHash] = ret;
-	}
-	get()->m_lastUsedFull = ret;
+
+	s_dagCallback = _f;
+	ret = make_shared<FullAllocation>(l->light, dagCallbackShim);
+
+	DEV_GUARDED(get()->x_fulls)
+		get()->m_fulls[seedHash] = get()->m_lastUsedFull = ret;
 	return ret;
 }
 
-Ethash::Result EthashAux::eval(BlockInfo const& _header, Nonce const& _nonce)
+unsigned EthashAux::computeFull(uint64_t _blockNumber)
 {
-	return eval(_header.seedHash(), _header.headerHash(WithoutNonce), _nonce);
+	Guard l(get()->x_fulls);
+	h256 seedHash = EthashAux::seedHash(_blockNumber);
+	if (FullType ret = get()->m_fulls[seedHash].lock())
+	{
+		get()->m_lastUsedFull = ret;
+		return 100;
+	}
+
+	if (!get()->m_fullGenerator || !get()->m_fullGenerator->joinable())
+	{
+		get()->m_fullProgress = 0;
+		get()->m_generatingFullNumber = _blockNumber / ETHASH_EPOCH_LENGTH * ETHASH_EPOCH_LENGTH;
+		get()->m_fullGenerator = unique_ptr<thread>(new thread([=](){
+			get()->full(_blockNumber, [](unsigned p){ get()->m_fullProgress = p; return 0; });
+			get()->m_fullProgress = 0;
+			get()->m_generatingFullNumber = NotGenerating;
+		}));
+	}
+
+	return (get()->m_generatingFullNumber == _blockNumber) ? get()->m_fullProgress : 0;
 }
 
-Ethash::Result EthashAux::FullAllocation::compute(h256 const& _seedHash, h256 const& _headerHash, Nonce const& _nonce) const
+Ethash::Result EthashAux::FullAllocation::compute(h256 const& _headerHash, Nonce const& _nonce) const
 {
-	ethash_return_value r;
-	auto p = EthashAux::params(_seedHash);
-	ethash_compute_full(&r, data.data(), &p, _headerHash.data(), (uint64_t)(u64)_nonce);
-	return Ethash::Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)};
+	ethash_return_value_t r = ethash_full_compute(full, *(ethash_h256_t*)_headerHash.data(), (uint64_t)(u64)_nonce);
+	if (!r.success)
+		BOOST_THROW_EXCEPTION(DAGCreationFailure());
+	return Ethash::Result{h256((uint8_t*)&r.result, h256::ConstructFromPointer), h256((uint8_t*)&r.mix_hash, h256::ConstructFromPointer)};
 }
 
-Ethash::Result EthashAux::LightAllocation::compute(h256 const& _seedHash, h256 const& _headerHash, Nonce const& _nonce) const
+Ethash::Result EthashAux::LightAllocation::compute(h256 const& _headerHash, Nonce const& _nonce) const
+{
+	ethash_return_value r = ethash_light_compute(light, *(ethash_h256_t*)_headerHash.data(), (uint64_t)(u64)_nonce);
+	if (!r.success)
+		BOOST_THROW_EXCEPTION(DAGCreationFailure());
+	return Ethash::Result{h256((uint8_t*)&r.result, h256::ConstructFromPointer), h256((uint8_t*)&r.mix_hash, h256::ConstructFromPointer)};
+}
+
+Ethash::Result EthashAux::eval(BlockInfo const& _header, Nonce const& _nonce)
 {
-	ethash_return_value r;
-	auto p = EthashAux::params(_seedHash);
-	ethash_compute_light(&r, light, &p, _headerHash.data(), (uint64_t)(u64)_nonce);
-	return Ethash::Result{h256(r.result, h256::ConstructFromPointer), h256(r.mix_hash, h256::ConstructFromPointer)};
+	return eval((uint64_t)_header.number, _header.headerHash(WithoutNonce), _nonce);
 }
 
-Ethash::Result EthashAux::eval(h256 const& _seedHash, h256 const& _headerHash, Nonce const& _nonce)
+Ethash::Result EthashAux::eval(uint64_t _blockNumber, h256 const& _headerHash, Nonce const& _nonce)
 {
-	if (auto dag = EthashAux::get()->full(_seedHash, bytesRef(), false))
-		return dag->compute(_seedHash, _headerHash, _nonce);
-	return EthashAux::get()->light(_seedHash)->compute(_seedHash, _headerHash, _nonce);
+	h256 seedHash = EthashAux::seedHash(_blockNumber);
+	if (FullType dag = get()->m_fulls[seedHash].lock())
+		return dag->compute(_headerHash, _nonce);
+	return EthashAux::get()->light(_blockNumber)->compute(_headerHash, _nonce);
 }

libethcore/EthashAux.h

@@ -19,12 +19,17 @@
  * @date 2014
  */
 
+#include <condition_variable>
 #include <libethash/ethash.h>
+#include <libdevcore/Worker.h>
 #include "Ethash.h"
 
 namespace dev
 {
-namespace eth{
+namespace eth
+{
+
+struct DAGChannel: public LogChannel { static const char* name(); static const int verbosity = 1; };
 
 class EthashAux
 {
@@ -33,39 +38,47 @@ public:
 
 	static EthashAux* get() { if (!s_this) s_this = new EthashAux(); return s_this; }
 
-	struct FullAllocation
-	{
-		FullAllocation(bytesConstRef _d): data(_d) {}
-		~FullAllocation() { delete [] data.data(); }
-		Ethash::Result compute(h256 const& _seedHash, h256 const& _headerHash, Nonce const& _nonce) const;
-		bytesConstRef const data;
-	};
-
 	struct LightAllocation
 	{
-		LightAllocation(h256 const& _seed);
+		LightAllocation(uint64_t _blockNumber);
 		~LightAllocation();
-		bytesConstRef data() const { return bytesConstRef((byte const*)light, size); }
-		Ethash::Result compute(h256 const& _seedHash, h256 const& _headerHash, Nonce const& _nonce) const;
+		bytesConstRef data() const;
+		Ethash::Result compute(h256 const& _headerHash, Nonce const& _nonce) const;
 		ethash_light_t light;
 		uint64_t size;
 	};
 
+	struct FullAllocation
+	{
+		FullAllocation(ethash_light_t _light, ethash_callback_t _cb);
+		~FullAllocation();
+		Ethash::Result compute(h256 const& _headerHash, Nonce const& _nonce) const;
+		bytesConstRef data() const;
+		uint64_t size() const { return ethash_full_dag_size(full); }
+		ethash_full_t full;
+	};
+
 	using LightType = std::shared_ptr<LightAllocation>;
 	using FullType = std::shared_ptr<FullAllocation>;
 
 	static h256 seedHash(unsigned _number);
-	static ethash_params params(BlockInfo const& _header);
-	static ethash_params params(h256 const& _seedHash);
-	static ethash_params params(unsigned _n);
+	static uint64_t number(h256 const& _seedHash);
+	static uint64_t cacheSize(BlockInfo const& _header);
+
 	static LightType light(BlockInfo const& _header);
-	static LightType light(h256 const& _seedHash);
-	static FullType full(BlockInfo const& _header, bytesRef _dest = bytesRef(), bool _createIfMissing = true);
-	static FullType full(h256 const& _header, bytesRef _dest = bytesRef(), bool _createIfMissing = true);
+	static LightType light(uint64_t _blockNumber);
+
+	static const uint64_t NotGenerating = (uint64_t)-1;
+	/// Kicks off generation of DAG for @a _blocknumber and @returns false or @returns true if ready.
+	static unsigned computeFull(uint64_t _blockNumber);
+	/// Information on the generation progress.
+	static std::pair<uint64_t, unsigned> fullGeneratingProgress() { return std::make_pair(get()->m_generatingFullNumber, get()->m_fullProgress); }
+	/// Kicks off generation of DAG for @a _blocknumber and blocks until ready; @returns result.
+	static FullType full(uint64_t _blockNumber, std::function<int(unsigned)> const& _f = std::function<int(unsigned)>());
 
 	static Ethash::Result eval(BlockInfo const& _header) { return eval(_header, _header.nonce); }
 	static Ethash::Result eval(BlockInfo const& _header, Nonce const& _nonce);
-	static Ethash::Result eval(h256 const& _seedHash, h256 const& _headerHash, Nonce const& _nonce);
+	static Ethash::Result eval(uint64_t _blockNumber, h256 const& _headerHash, Nonce const& _nonce);
 
 private:
 	EthashAux() {}
@@ -73,11 +86,17 @@ private:
 	void killCache(h256 const& _s);
 
 	static EthashAux* s_this;
-	RecursiveMutex x_this;
 
+	RecursiveMutex x_lights;
 	std::unordered_map<h256, std::shared_ptr<LightAllocation>> m_lights;
+
+	Mutex x_fulls;
+	std::condition_variable m_fullsChanged;
 	std::unordered_map<h256, std::weak_ptr<FullAllocation>> m_fulls;
 	FullType m_lastUsedFull;
+	std::unique_ptr<std::thread> m_fullGenerator;
+	uint64_t m_generatingFullNumber = NotGenerating;
+	unsigned m_fullProgress;
 
 	Mutex x_epochs;
 	std::unordered_map<h256, unsigned> m_epochs;

libethcore/Exceptions.h

@@ -73,6 +73,7 @@ class InvalidBlockNonce: virtual public dev::Exception {};
 struct InvalidParentHash: virtual dev::Exception {};
 struct InvalidNumber: virtual dev::Exception {};
 struct InvalidContractAddress: virtual public dev::Exception {};
-
+struct DAGCreationFailure: virtual public dev::Exception {};
+struct DAGComputeFailure: virtual public dev::Exception {};
 }
 }

libethereum/BlockChain.cpp

@@ -456,7 +456,7 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 	{
 		// Check transactions are valid and that they result in a state equivalent to our state_root.
 		// Get total difficulty increase and update state, checking it.
-		State s(_db);	//, bi.coinbaseAddress
+		State s(_db);
 		auto tdIncrease = s.enactOn(&_block, bi, *this, _ir);
 
 		BlockLogBlooms blb;
@@ -466,7 +466,15 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 			blb.blooms.push_back(s.receipt(i).bloom());
 			br.receipts.push_back(s.receipt(i));
 		}
-		s.cleanup(true);
+		try {
+			s.cleanup(true);
+		}
+		catch (BadRoot)
+		{
+			cwarn << "BadRoot error. Retrying import later.";
+			BOOST_THROW_EXCEPTION(FutureTime());
+		}
+
 		td = pd.totalDifficulty + tdIncrease;
 
 #if ETH_TIMED_IMPORTS
@@ -603,7 +611,6 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 		{
 			newLastBlockHash = bi.hash();
 			newLastBlockNumber = (unsigned)bi.number;
-			extrasBatch.Put(ldb::Slice("best"), ldb::Slice((char const*)&(bi.hash()), 32));
 		}
 
 		clog(BlockChainNote) << "   Imported and best" << td << " (#" << bi.number << "). Has" << (details(bi.parentHash).children.size() - 1) << "siblings. Route:" << route;
@@ -623,12 +630,33 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 	m_blocksDB->Write(m_writeOptions, &blocksBatch);
 	m_extrasDB->Write(m_writeOptions, &extrasBatch);
 
-	DEV_WRITE_GUARDED(x_lastBlockHash)
+	if (isKnown(bi.hash()) && !details(bi.hash()))
+	{
+		clog(BlockChainDebug) << "Known block just inserted has no details.";
+		clog(BlockChainDebug) << "Block:" << bi;
+		clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
+		exit(-1);
+	}
+
+	try {
+		State canary(_db, *this, bi.hash());
+	}
+	catch (...)
 	{
-		m_lastBlockHash = newLastBlockHash;
-		m_lastBlockNumber = newLastBlockNumber;
+		clog(BlockChainDebug) << "Failed to initialise State object form imported block.";
+		clog(BlockChainDebug) << "Block:" << bi;
+		clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
+		exit(-1);
 	}
 
+	if (m_lastBlockHash != newLastBlockHash)
+		DEV_WRITE_GUARDED(x_lastBlockHash)
+		{
+			m_lastBlockHash = newLastBlockHash;
+			m_lastBlockNumber = newLastBlockNumber;
+			m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&m_lastBlockHash, 32));
+		}
+
 #if ETH_PARANOIA || !ETH_TRUE
 	checkConsistency();
 #endif
@@ -646,14 +674,6 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import
 	if (!route.empty())
 		noteCanonChanged();
 
-	if (isKnown(bi.hash()) && !details(bi.hash()))
-	{
-		clog(BlockChainDebug) << "Known block just inserted has no details.";
-		clog(BlockChainDebug) << "Block:" << bi;
-		clog(BlockChainDebug) << "DATABASE CORRUPTION: CRITICAL FAILURE";
-		exit(-1);
-	}
-
 	h256s fresh;
 	h256s dead;
 	bool isOld = true;

libethereum/Client.cpp

@@ -164,7 +164,7 @@ const char* ClientDetail::name() { return EthTeal "⧫" EthCoal " ●"; }
 #endif
 
 Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId):
-	Worker("eth"),
+	Worker("eth", 0),
 	m_vc(_dbPath),
 	m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }),
 	m_gp(new TrivialGasPricer),

libethereum/ClientBase.h

@@ -81,13 +81,14 @@ public:
 	/// Submits a new contract-creation transaction.
 	/// @returns the new contract's address (assuming it all goes through).
 	virtual Address submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas = 10000, u256 _gasPrice = 10 * szabo) override;
+	using Interface::submitTransaction;
 
 	/// Makes the given call. Nothing is recorded into the state.
-	virtual ExecutionResult call(Address const& _secret, u256 _value, Address _dest, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = PendingBlock, FudgeFactor _ff = FudgeFactor::Strict) override;
+	virtual ExecutionResult call(Address const& _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber, FudgeFactor _ff = FudgeFactor::Strict) override;
 	using Interface::call;
 
 	/// Makes the given create. Nothing is recorded into the state.
-	virtual ExecutionResult create(Address const& _secret, u256 _value, bytes const& _data = bytes(), u256 _gas = 10000, u256 _gasPrice = 10 * szabo, BlockNumber _blockNumber = PendingBlock, FudgeFactor _ff = FudgeFactor::Strict) override;
+	virtual ExecutionResult create(Address const& _secret, u256 _value, bytes const& _data, u256 _gas, u256 _gasPrice, BlockNumber _blockNumber, FudgeFactor _ff = FudgeFactor::Strict) override;
 	using Interface::create;
 
 	using Interface::balanceAt;

libethereum/Interface.h

@@ -72,6 +72,10 @@ public:
 	/// @returns the new contract's address (assuming it all goes through).
 	virtual Address submitTransaction(Secret _secret, u256 _endowment, bytes const& _init, u256 _gas = 10000, u256 _gasPrice = 10 * szabo) = 0;
 
+	/// Submits a new contract-creation transaction.
+	/// @returns the new contract's address (assuming it all goes through).
+	Address submitTransaction(Secret const& _secret, TransactionSkeleton const& _t) { if (_t.creation) return submitTransaction(_secret, _t.value, _t.data, _t.gas, _t.gasPrice); submitTransaction(_secret, _t.value, _t.to, _t.data, _t.gas, _t.gasPrice); return Address(); }
+
 	/// Blocks until all pending transactions have been processed.
 	virtual void flushTransactions() = 0;
 

libethereum/KeyManager.cpp

@@ -28,6 +28,7 @@
 #include <libdevcore/RLP.h>
 using namespace std;
 using namespace dev;
+using namespace eth;
 namespace fs = boost::filesystem;
 
 KeyManager::KeyManager(std::string const& _keysFile):
@@ -151,9 +152,18 @@ void KeyManager::kill(Address const& _a)
 	m_store.kill(id);
 }
 
-std::map<Address, std::pair<std::string, std::string>> KeyManager::keys() const
+AddressHash KeyManager::accounts() const
 {
-	std::map<Address, std::pair<std::string, std::string>> ret;
+	AddressHash ret;
+	for (auto const& i: m_addrLookup)
+		if (m_keyInfo.count(i.second) > 0)
+			ret.insert(i.first);
+	return ret;
+}
+
+std::unordered_map<Address, std::pair<std::string, std::string>> KeyManager::accountDetails() const
+{
+	std::unordered_map<Address, std::pair<std::string, std::string>> ret;
 	for (auto const& i: m_addrLookup)
 		if (m_keyInfo.count(i.second) > 0)
 			ret[i.first] = make_pair(m_keyInfo.at(i.second).info, m_passwordInfo.at(m_keyInfo.at(i.second).passHash));

libethereum/KeyManager.h

@@ -28,7 +28,8 @@
 
 namespace dev
 {
-
+namespace eth
+{
 class UnknownPassword: public Exception {};
 
 struct KeyInfo
@@ -65,7 +66,10 @@ public:
 	bool load(std::string const& _pass);
 	void save(std::string const& _pass) const { write(_pass, m_keysFile); }
 
-	std::map<Address, std::pair<std::string, std::string>> keys() const;
+	void notePassword(std::string const& _pass) { m_cachedPasswords[hashPassword(_pass)] = _pass; }
+
+	AddressHash accounts() const;
+	std::unordered_map<Address, std::pair<std::string, std::string>> accountDetails() const;
 
 	h128 uuid(Address const& _a) const;
 	Address address(h128 const& _uuid) const;
@@ -92,12 +96,12 @@ private:
 	void write(h128 const& _key, std::string const& _keysFile) const;
 
 	// Ethereum keys.
-	std::map<Address, h128> m_addrLookup;
-	std::map<h128, KeyInfo> m_keyInfo;
-	std::map<h256, std::string> m_passwordInfo;
+	std::unordered_map<Address, h128> m_addrLookup;
+	std::unordered_map<h128, KeyInfo> m_keyInfo;
+	std::unordered_map<h256, std::string> m_passwordInfo;
 
 	// Passwords that we're storing.
-	mutable std::map<h256, std::string> m_cachedPasswords;
+	mutable std::unordered_map<h256, std::string> m_cachedPasswords;
 
 	// The default password for keys in the keystore - protected by the master password.
 	std::string m_password;
@@ -108,3 +112,4 @@ private:
 };
 
 }
+}

libethereum/State.cpp

@@ -515,10 +515,10 @@ pair<TransactionReceipts, bool> State::sync(BlockChain const& _bc, TransactionQu
 						cnote << i.first << "Dropping old transaction (nonce too low)";
 						_tq.drop(i.first);
 					}
-					else if (got > req + 5)
+					else if (got > req + 25)
 					{
 						// too new
-						cnote << i.first << "Dropping new transaction (> 5 nonces ahead)";
+						cnote << i.first << "Dropping new transaction (> 25 nonces ahead)";
 						_tq.drop(i.first);
 					}
 					else
@@ -711,6 +711,17 @@ void State::cleanup(bool _fullCommit)
 
 		// Commit the new trie to disk.
 		clog(StateTrace) << "Committing to disk: stateRoot" << m_currentBlock.stateRoot << "=" << rootHash() << "=" << toHex(asBytes(m_db.lookup(rootHash())));
+
+		try {
+			EnforceRefs er(m_db, true);
+			rootHash();
+		}
+		catch (BadRoot const&)
+		{
+			clog(StateChat) << "Trie corrupt! :-(";
+			throw;
+		}
+
 		m_db.commit();
 		clog(StateTrace) << "Committed: stateRoot" << m_currentBlock.stateRoot << "=" << rootHash() << "=" << toHex(asBytes(m_db.lookup(rootHash())));
 

libethereum/State.h

@@ -49,7 +49,7 @@ class BlockChain;
 class State;
 
 struct StateChat: public LogChannel { static const char* name(); static const int verbosity = 4; };
-struct StateTrace: public LogChannel { static const char* name(); static const int verbosity = 7; };
+struct StateTrace: public LogChannel { static const char* name(); static const int verbosity = 5; };
 struct StateDetail: public LogChannel { static const char* name(); static const int verbosity = 14; };
 struct StateSafeExceptions: public LogChannel { static const char* name(); static const int verbosity = 21; };
 

libevm/CMakeLists.txt

@@ -15,6 +15,9 @@ aux_source_directory(. SRC_LIST)
 # and windows is failing to build without that
 include_directories(BEFORE ..)
 include_directories(${Boost_INCLUDE_DIRS})
+if (EVMJIT)
+	include_directories(../evmjit/include)
+endif()
 
 set(EXECUTABLE evm)
 

libevm/SmartVM.cpp

@@ -0,0 +1,79 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// SmartVM is only available if EVM JIT is enabled
+#if ETH_EVMJIT
+
+#include "SmartVM.h"
+#include <unordered_map>
+#include <libdevcore/Log.h>
+#include <libdevcrypto/SHA3.h>
+#include <evmjit/JIT.h>
+#include <evmjit/libevmjit-cpp/Utils.h>
+#include "VMFactory.h"
+
+namespace dev
+{
+namespace eth
+{
+namespace
+{
+	using HitMap = std::unordered_map<h256, uint64_t>;
+
+	HitMap& getHitMap()
+	{
+		static HitMap s_hitMap;
+		return s_hitMap;
+	}
+}
+
+bytesConstRef SmartVM::go(ExtVMFace& _ext, OnOpFunc const& _onOp, uint64_t _steps)
+{
+	auto codeHash = sha3(_ext.code);
+	auto vmKind = VMKind::Interpreter; // default VM
+
+	// Jitted EVM code already in memory?
+	if (evmjit::JIT::isCodeReady(eth2llvm(codeHash)))
+	{
+		cnote << "Jitted";
+		vmKind = VMKind::JIT;
+	}
+	else
+	{
+		// Check EVM code hit count
+		static const uint64_t c_hitTreshold = 1;
+		auto& hits = getHitMap()[codeHash];
+		++hits;
+		if (hits > c_hitTreshold)
+		{
+			cnote << "JIT selected";
+			vmKind = VMKind::JIT;
+		}
+	}
+
+	// TODO: Selected VM must be kept only because it returns reference to its internal memory.
+	//       VM implementations should be stateless, without gas counter and escaping memory reference.
+	m_selectedVM = VMFactory::create(vmKind, gas());
+	auto out = m_selectedVM->go(_ext, _onOp, _steps);
+	m_gas = m_selectedVM->gas();
+	return out;
+}
+
+}
+}
+
+#endif

libevm/SmartVM.h

@@ -0,0 +1,43 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#pragma once
+
+#include "VMFace.h"
+
+namespace dev
+{
+namespace eth
+{
+
+/// Smart VM proxy.
+///
+/// This class is a strategy pattern implementation for VM. For every EVM code
+/// execution request it tries to select the best VM implementation (Interpreter or JIT)
+/// by analyzing available information like: code size, hit count, JIT status, etc.
+class SmartVM: public VMFace
+{
+public:
+	SmartVM(u256 _gas): VMFace(_gas) {}
+
+	virtual bytesConstRef go(ExtVMFace& _ext, OnOpFunc const& _onOp = {}, uint64_t _steps = (uint64_t)-1) override final;
+
+private:
+	std::unique_ptr<VMFace> m_selectedVM;
+};
+
+}
+}

libevm/VMFactory.cpp

@@ -18,6 +18,7 @@
 #include "VMFactory.h"
 #include <libdevcore/Assertions.h>
 #include "VM.h"
+#include "SmartVM.h"
 
 #if ETH_EVMJIT
 #include <evmjit/libevmjit-cpp/JitVM.h>
@@ -29,7 +30,7 @@ namespace eth
 {
 namespace
 {
-	VMKind g_kind = VMKind::Interpreter;
+	auto g_kind = VMKind::Interpreter;
 }
 
 void VMFactory::setKind(VMKind _kind)
@@ -38,11 +39,25 @@ void VMFactory::setKind(VMKind _kind)
 }
 
 std::unique_ptr<VMFace> VMFactory::create(u256 _gas)
+{
+	return create(g_kind, _gas);
+}
+
+std::unique_ptr<VMFace> VMFactory::create(VMKind _kind, u256 _gas)
 {
 #if ETH_EVMJIT
-	return std::unique_ptr<VMFace>(g_kind == VMKind::JIT ? static_cast<VMFace*>(new JitVM(_gas)) : static_cast<VMFace*>(new VM(_gas)));
+	switch (_kind)
+	{
+	default:
+	case VMKind::Interpreter:
+		return std::unique_ptr<VMFace>(new VM(_gas));
+	case VMKind::JIT:
+		return std::unique_ptr<VMFace>(new JitVM(_gas));
+	case VMKind::Smart:
+		return std::unique_ptr<VMFace>(new SmartVM(_gas));
+	}
 #else
-	asserts(g_kind == VMKind::Interpreter && "JIT disabled in build configuration");
+	asserts(_kind == VMKind::Interpreter && "JIT disabled in build configuration");
 	return std::unique_ptr<VMFace>(new VM(_gas));
 #endif
 }

libevm/VMFactory.h

@@ -23,10 +23,11 @@ namespace dev
 namespace eth
 {
 
-enum class VMKind: bool
+enum class VMKind
 {
 	Interpreter,
-	JIT
+	JIT,
+	Smart
 };
 
 class VMFactory
@@ -34,7 +35,13 @@ class VMFactory
 public:
 	VMFactory() = delete;
 
+	/// Creates a VM instance of global kind (controlled by setKind() function).
 	static std::unique_ptr<VMFace> create(u256 _gas);
+
+	/// Creates a VM instance of kind provided.
+	static std::unique_ptr<VMFace> create(VMKind _kind, u256 _gas);
+
+	/// Set global VM kind
 	static void setKind(VMKind _kind);
 };
 

libevmasm/Assembly.cpp

@@ -24,6 +24,7 @@
 #include <libdevcore/Log.h>
 #include <libevmasm/CommonSubexpressionEliminator.h>
 #include <libevmasm/ControlFlowGraph.h>
+#include <libevmasm/BlockDeduplicator.h>
 #include <json/json.h>
 using namespace std;
 using namespace dev;
@@ -311,54 +312,54 @@ Assembly& Assembly::optimise(bool _enable)
 		copt << toString(*this);
 		count = 0;
 
-		//@todo CFG interface should be a generator, that returns an item and a pointer to a
-		// knownstate, which has to replace the current state if it is not null.
-		// Feed these items to the CSE, but also store them and replace the stored version
-		// if the items generated by the CSE are shorter. (or even use less gas?)
-		copt << "Performing control flow analysis...";
+		copt << "Performing optimisation...";
 		{
 			ControlFlowGraph cfg(m_items);
-			AssemblyItems optItems;
+			AssemblyItems optimisedItems;
 			for (BasicBlock const& block: cfg.optimisedBlocks())
-				copy(m_items.begin() + block.begin, m_items.begin() + block.end,
-					 back_inserter(optItems));
-			if (optItems.size() < m_items.size())
 			{
-				copt << "Old size: " << m_items.size() << ", new size: " << optItems.size();
-				m_items = move(optItems);
-				count++;
+				assertThrow(!!block.startState, OptimizerException, "");
+				CommonSubexpressionEliminator eliminator(*block.startState);
+				auto iter = m_items.begin() + block.begin;
+				auto const end = m_items.begin() + block.end;
+				while (iter < end)
+				{
+					auto orig = iter;
+					iter = eliminator.feedItems(iter, end);
+					bool shouldReplace = false;
+					AssemblyItems optimisedChunk;
+					try
+					{
+						optimisedChunk = eliminator.getOptimizedItems();
+						shouldReplace = (optimisedChunk.size() < size_t(iter - orig));
+					}
+					catch (StackTooDeepException const&)
+					{
+						// This might happen if the opcode reconstruction is not as efficient
+						// as the hand-crafted code.
+					}
+
+					if (shouldReplace)
+					{
+						copt << "Old size: " << (iter - orig) << ", new size: " << optimisedChunk.size();
+						count++;
+						optimisedItems += optimisedChunk;
+					}
+					else
+						copy(orig, iter, back_inserter(optimisedItems));
+				}
 			}
-		}
 
-		copt << "Performing common subexpression elimination...";
-		for (auto iter = m_items.begin(); iter != m_items.end();)
-		{
-			//@todo use only a single state / expression classes instance.
-			KnownState state(make_shared<ExpressionClasses>());
-			CommonSubexpressionEliminator eliminator(state);
-			auto orig = iter;
-			iter = eliminator.feedItems(iter, m_items.end());
-			AssemblyItems optItems;
-			bool shouldReplace = false;
-			try
+			if (optimisedItems.size() < m_items.size())
 			{
-				optItems = eliminator.getOptimizedItems();
-				shouldReplace = (optItems.size() < size_t(iter - orig));
-			}
-			catch (StackTooDeepException const&)
-			{
-				// This might happen if the opcode reconstruction is not as efficient
-				// as the hand-crafted code.
+				m_items = move(optimisedItems);
+				count++;
 			}
 
-			if (shouldReplace)
-			{
-				copt << "Old size: " << (iter - orig) << ", new size: " << optItems.size();
+			// This only modifies PushTags, we have to run again to actually remove code.
+			BlockDeduplicator dedup(m_items);
+			if (dedup.deduplicate())
 				count++;
-				for (auto moveIter = optItems.begin(); moveIter != optItems.end(); ++orig, ++moveIter)
-					*orig = move(*moveIter);
-				iter = m_items.erase(orig, iter);
-			}
 		}
 	}
 
@@ -461,7 +462,8 @@ bytes Assembly::assemble() const
 	for (auto const& i: tagRef)
 	{
 		bytesRef r(ret.data() + i.first, bytesPerTag);
-		toBigEndian(tagPos[i.second], r);
+		//@todo in the failure case, we could use the position of the invalid jumpdest
+		toBigEndian(i.second < tagPos.size() ? tagPos[i.second] : (1 << (8 * bytesPerTag)) - 1, r);
 	}
 
 	if (!m_data.empty())

libevmasm/AssemblyItem.h

@@ -68,6 +68,8 @@ public:
 	/// @returns true iff the type and data of the items are equal.
 	bool operator==(AssemblyItem const& _other) const { return m_type == _other.m_type && m_data == _other.m_data; }
 	bool operator!=(AssemblyItem const& _other) const { return !operator==(_other); }
+	/// Less-than operator compatible with operator==.
+	bool operator<(AssemblyItem const& _other) const { return std::tie(m_type, m_data) < std::tie(_other.m_type, _other.m_data); }
 
 	/// @returns an upper bound for the number of bytes required by this item, assuming that
 	/// the value of a jump tag takes @a _addressLength bytes.

libevmasm/BlockDeduplicator.cpp

@@ -0,0 +1,91 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file BlockDeduplicator.cpp
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Unifies basic blocks that share content.
+ */
+
+#include <libevmasm/BlockDeduplicator.h>
+#include <functional>
+#include <libevmasm/AssemblyItem.h>
+#include <libevmasm/SemanticInformation.h>
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+
+bool BlockDeduplicator::deduplicate()
+{
+	// Compares indices based on the suffix that starts there, ignoring tags and stopping at
+	// opcodes that stop the control flow.
+	function<bool(size_t, size_t)> comparator = [&](size_t _i, size_t _j)
+	{
+		if (_i == _j)
+			return false;
+
+		BlockIterator first(m_items.begin() + _i, m_items.end());
+		BlockIterator second(m_items.begin() + _j, m_items.end());
+		BlockIterator end(m_items.end(), m_items.end());
+
+		if (first != end && (*first).type() == Tag)
+			++first;
+		if (second != end && (*second).type() == Tag)
+			++second;
+
+		return std::lexicographical_compare(first, end, second, end);
+	};
+
+	set<size_t, function<bool(size_t, size_t)>> blocksSeen(comparator);
+	map<u256, u256> tagReplacement;
+	for (size_t i = 0; i < m_items.size(); ++i)
+	{
+		if (m_items.at(i).type() != Tag)
+			continue;
+		auto it = blocksSeen.find(i);
+		if (it == blocksSeen.end())
+			blocksSeen.insert(i);
+		else
+			tagReplacement[m_items.at(i).data()] = m_items.at(*it).data();
+	}
+
+	bool ret = false;
+	for (AssemblyItem& item: m_items)
+		if (item.type() == PushTag && tagReplacement.count(item.data()))
+		{
+			ret = true;
+			item.setData(tagReplacement.at(item.data()));
+		}
+	return ret;
+}
+
+BlockDeduplicator::BlockIterator& BlockDeduplicator::BlockIterator::operator++()
+{
+	if (it == end)
+		return *this;
+	if (SemanticInformation::altersControlFlow(*it) && *it != AssemblyItem(eth::Instruction::JUMPI))
+		it = end;
+	else
+	{
+		++it;
+		while (it != end && it->type() == Tag)
+			++it;
+	}
+	return *this;
+}

libevmasm/BlockDeduplicator.h

@@ -0,0 +1,69 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @file BlockDeduplicator.h
+ * @author Christian <c@ethdev.com>
+ * @date 2015
+ * Unifies basic blocks that share content.
+ */
+
+#pragma once
+
+#include <cstddef>
+#include <vector>
+#include <functional>
+
+namespace dev
+{
+namespace eth
+{
+
+class AssemblyItem;
+using AssemblyItems = std::vector<AssemblyItem>;
+
+/**
+ * Optimizer class to be used to unify blocks that share content.
+ * Modifies the passed vector in place.
+ */
+class BlockDeduplicator
+{
+public:
+	BlockDeduplicator(AssemblyItems& _items): m_items(_items) {}
+	/// @returns true if something was changed
+	bool deduplicate();
+
+private:
+	/// Iterator that skips tags skips to the end if (all branches of) the control
+	/// flow does not continue to the next instruction.
+	struct BlockIterator: std::iterator<std::forward_iterator_tag, AssemblyItem const>
+	{
+	public:
+		BlockIterator(AssemblyItems::const_iterator _it, AssemblyItems::const_iterator _end):
+			it(_it), end(_end) { }
+		BlockIterator& operator++();
+		bool operator==(BlockIterator const& _other) const { return it == _other.it; }
+		bool operator!=(BlockIterator const& _other) const { return it != _other.it; }
+		AssemblyItem const& operator*() const { return *it; }
+		AssemblyItems::const_iterator it;
+		AssemblyItems::const_iterator end;
+	};
+
+	AssemblyItems& m_items;
+};
+
+}
+}

libevmasm/CMakeLists.txt

@@ -19,15 +19,10 @@ set(EXECUTABLE evmasm)
 
 file(GLOB HEADERS "*.h")
 
-if (ETH_STATIC)
-	add_library(${EXECUTABLE} STATIC ${SRC_LIST} ${HEADERS})
-else()
-	add_library(${EXECUTABLE} SHARED ${SRC_LIST} ${HEADERS})
-endif()
+add_library(${EXECUTABLE} ${SRC_LIST} ${HEADERS})
 
 target_link_libraries(${EXECUTABLE} evmcore)
 target_link_libraries(${EXECUTABLE} devcrypto)
 
 install( TARGETS ${EXECUTABLE} RUNTIME DESTINATION bin ARCHIVE DESTINATION lib LIBRARY DESTINATION lib )
 install( FILES ${HEADERS} DESTINATION include/${EXECUTABLE} )
-

libevmasm/CommonSubexpressionEliminator.cpp

@@ -45,16 +45,22 @@ vector<AssemblyItem> CommonSubexpressionEliminator::getOptimizedItems()
 	for (int height = minHeight; height <= m_state.stackHeight(); ++height)
 		targetStackContents[height] = m_state.stackElement(height, SourceLocation());
 
-	// Debug info:
-	//stream(cout, initialStackContents, targetStackContents);
-
 	AssemblyItems items = CSECodeGenerator(m_state.expressionClasses(), m_storeOperations).generateCode(
 		m_initialState.stackHeight(),
 		initialStackContents,
 		targetStackContents
 	);
 	if (m_breakingItem)
+	{
 		items.push_back(*m_breakingItem);
+		m_state.feedItem(*m_breakingItem);
+	}
+
+	// cleanup
+	m_initialState = m_state;
+	m_breakingItem = nullptr;
+	m_storeOperations.clear();
+
 	return items;
 }
 
@@ -113,16 +119,14 @@ AssemblyItems CSECodeGenerator::generateCode(
 {
 	m_stackHeight = _initialStackHeight;
 	m_stack = _initialStack;
+	m_targetStack = _targetStackContents;
 	for (auto const& item: m_stack)
-		if (!m_classPositions.count(item.second))
-			m_classPositions[item.second] = item.first;
-
-	// @todo: provide information about the positions of copies of class elements
+		m_classPositions[item.second].insert(item.first);
 
 	// generate the dependency graph starting from final storage and memory writes and target stack contents
 	for (auto const& p: m_storeOperations)
 		addDependencies(p.second.back().expression);
-	for (auto const& targetItem: _targetStackContents)
+	for (auto const& targetItem: m_targetStack)
 	{
 		m_finalClasses.insert(targetItem.second);
 		addDependencies(targetItem.second);
@@ -141,13 +145,16 @@ AssemblyItems CSECodeGenerator::generateCode(
 			generateClassElement(seqAndId.second, true);
 
 	// generate the target stack elements
-	for (auto const& targetItem: _targetStackContents)
+	for (auto const& targetItem: m_targetStack)
 	{
-		int position = generateClassElement(targetItem.second);
-		assertThrow(position != c_invalidPosition, OptimizerException, "");
-		if (position == targetItem.first)
+		if (m_stack.count(targetItem.first) && m_stack.at(targetItem.first) == targetItem.second)
+			continue; // already there
+		generateClassElement(targetItem.second);
+		assertThrow(!m_classPositions[targetItem.second].empty(), OptimizerException, "");
+		if (m_classPositions[targetItem.second].count(targetItem.first))
 			continue;
 		SourceLocation const& location = m_expressionClasses.representative(targetItem.second).item->getLocation();
+		int position = classElementPosition(targetItem.second);
 		if (position < targetItem.first)
 			// it is already at its target, we need another copy
 			appendDup(position, location);
@@ -164,21 +171,24 @@ AssemblyItems CSECodeGenerator::generateCode(
 
 	// check validity
 	int finalHeight = 0;
-	if (!_targetStackContents.empty())
+	if (!m_targetStack.empty())
 		// have target stack, so its height should be the final height
-		finalHeight = (--_targetStackContents.end())->first;
+		finalHeight = (--m_targetStack.end())->first;
 	else if (!_initialStack.empty())
 		// no target stack, only erase the initial stack
 		finalHeight = _initialStack.begin()->first - 1;
 	else
 		// neither initial no target stack, no change in height
-		finalHeight = 0;
+		finalHeight = _initialStackHeight;
 	assertThrow(finalHeight == m_stackHeight, OptimizerException, "Incorrect final stack height.");
+
 	return m_generatedItems;
 }
 
 void CSECodeGenerator::addDependencies(Id _c)
 {
+	if (m_classPositions.count(_c))
+		return; // it is already on the stack
 	if (m_neededBy.count(_c))
 		return; // we already computed the dependencies for _c
 	ExpressionClasses::Expression expr = m_expressionClasses.representative(_c);
@@ -254,19 +264,23 @@ void CSECodeGenerator::addDependencies(Id _c)
 	}
 }
 
-int CSECodeGenerator::generateClassElement(Id _c, bool _allowSequenced)
+void CSECodeGenerator::generateClassElement(Id _c, bool _allowSequenced)
 {
+	for (auto it: m_classPositions)
+		for (auto p: it.second)
+			if (p > m_stackHeight)
+				assertThrow(false, OptimizerException, "");
 	// do some cleanup
 	removeStackTopIfPossible();
 
 	if (m_classPositions.count(_c))
 	{
 		assertThrow(
-			m_classPositions[_c] != c_invalidPosition,
+			!m_classPositions[_c].empty(),
 			OptimizerException,
 			"Element already removed but still needed."
 		);
-		return m_classPositions[_c];
+		return;
 	}
 	ExpressionClasses::Expression const& expr = m_expressionClasses.representative(_c);
 	assertThrow(
@@ -339,16 +353,16 @@ int CSECodeGenerator::generateClassElement(Id _c, bool _allowSequenced)
 			m_generatedItems.back() == AssemblyItem(Instruction::SWAP1))
 		// this will not append a swap but remove the one that is already there
 		appendOrRemoveSwap(m_stackHeight - 1, location);
-	for (auto arg: arguments)
-		if (canBeRemoved(arg, _c))
-			m_classPositions[arg] = c_invalidPosition;
 	for (size_t i = 0; i < arguments.size(); ++i)
+	{
+		m_classPositions[m_stack[m_stackHeight - i]].erase(m_stackHeight - i);
 		m_stack.erase(m_stackHeight - i);
+	}
 	appendItem(*expr.item);
 	if (expr.item->type() != Operation || instructionInfo(expr.item->instruction()).ret == 1)
 	{
 		m_stack[m_stackHeight] = _c;
-		return m_classPositions[_c] = m_stackHeight;
+		m_classPositions[_c].insert(m_stackHeight);
 	}
 	else
 	{
@@ -357,31 +371,39 @@ int CSECodeGenerator::generateClassElement(Id _c, bool _allowSequenced)
 			OptimizerException,
 			"Invalid number of return values."
 		);
-		return m_classPositions[_c] = c_invalidPosition;
+		m_classPositions[_c]; // ensure it is created to mark the expression as generated
 	}
 }
 
 int CSECodeGenerator::classElementPosition(Id _id) const
 {
 	assertThrow(
-		m_classPositions.count(_id) && m_classPositions.at(_id) != c_invalidPosition,
+		m_classPositions.count(_id) && !m_classPositions.at(_id).empty(),
 		OptimizerException,
 		"Element requested but is not present."
 	);
-	return m_classPositions.at(_id);
+	return *max_element(m_classPositions.at(_id).begin(), m_classPositions.at(_id).end());
 }
 
-bool CSECodeGenerator::canBeRemoved(Id _element, Id _result)
+bool CSECodeGenerator::canBeRemoved(Id _element, Id _result, int _fromPosition)
 {
-	// Returns false if _element is finally needed or is needed by a class that has not been
-	// computed yet. Note that m_classPositions also includes classes that were deleted in the meantime.
-	if (m_finalClasses.count(_element))
-		return false;
+	// Default for _fromPosition is the canonical position of the element.
+	if (_fromPosition == c_invalidPosition)
+		_fromPosition = classElementPosition(_element);
 
-	auto range = m_neededBy.equal_range(_element);
-	for (auto it = range.first; it != range.second; ++it)
-		if (it->second != _result && !m_classPositions.count(it->second))
-			return false;
+	bool haveCopy = m_classPositions.at(_element).size() > 1;
+	if (m_finalClasses.count(_element))
+		// It is part of the target stack. It can be removed if it is a copy that is not in the target position.
+		return haveCopy && (!m_targetStack.count(_fromPosition) || m_targetStack[_fromPosition] != _element);
+	else if (!haveCopy)
+	{
+		// Can be removed unless it is needed by a class that has not been computed yet.
+		// Note that m_classPositions also includes classes that were deleted in the meantime.
+		auto range = m_neededBy.equal_range(_element);
+		for (auto it = range.first; it != range.second; ++it)
+			if (it->second != _result && !m_classPositions.count(it->second))
+				return false;
+	}
 	return true;
 }
 
@@ -391,11 +413,11 @@ bool CSECodeGenerator::removeStackTopIfPossible()
 		return false;
 	assertThrow(m_stack.count(m_stackHeight) > 0, OptimizerException, "");
 	Id top = m_stack[m_stackHeight];
-	if (!canBeRemoved(top))
+	if (!canBeRemoved(top, Id(-1), m_stackHeight))
 		return false;
-	m_generatedItems.push_back(AssemblyItem(Instruction::POP));
+	m_classPositions[m_stack[m_stackHeight]].erase(m_stackHeight);
 	m_stack.erase(m_stackHeight);
-	m_stackHeight--;
+	appendItem(AssemblyItem(Instruction::POP));
 	return true;
 }
 
@@ -407,6 +429,7 @@ void CSECodeGenerator::appendDup(int _fromPosition, SourceLocation const& _locat
 	assertThrow(1 <= instructionNum, OptimizerException, "Invalid stack access.");
 	appendItem(AssemblyItem(dupInstruction(instructionNum), _location));
 	m_stack[m_stackHeight] = m_stack[_fromPosition];
+	m_classPositions[m_stack[m_stackHeight]].insert(m_stackHeight);
 }
 
 void CSECodeGenerator::appendOrRemoveSwap(int _fromPosition, SourceLocation const& _location)
@@ -418,13 +441,15 @@ void CSECodeGenerator::appendOrRemoveSwap(int _fromPosition, SourceLocation cons
 	assertThrow(instructionNum <= 16, StackTooDeepException, "Stack too deep.");
 	assertThrow(1 <= instructionNum, OptimizerException, "Invalid stack access.");
 	appendItem(AssemblyItem(swapInstruction(instructionNum), _location));
-	// The value of a class can be present in multiple locations on the stack. We only update the
-	// "canonical" one that is tracked by m_classPositions
-	if (m_classPositions[m_stack[m_stackHeight]] == m_stackHeight)
-		m_classPositions[m_stack[m_stackHeight]] = _fromPosition;
-	if (m_classPositions[m_stack[_fromPosition]] == _fromPosition)
-		m_classPositions[m_stack[_fromPosition]] = m_stackHeight;
-	swap(m_stack[m_stackHeight], m_stack[_fromPosition]);
+
+	if (m_stack[m_stackHeight] != m_stack[_fromPosition])
+	{
+		m_classPositions[m_stack[m_stackHeight]].erase(m_stackHeight);
+		m_classPositions[m_stack[m_stackHeight]].insert(_fromPosition);
+		m_classPositions[m_stack[_fromPosition]].erase(_fromPosition);
+		m_classPositions[m_stack[_fromPosition]].insert(m_stackHeight);
+		swap(m_stack[m_stackHeight], m_stack[_fromPosition]);
+	}
 	if (m_generatedItems.size() >= 2 &&
 		SemanticInformation::isSwapInstruction(m_generatedItems.back()) &&
 		*(m_generatedItems.end() - 2) == m_generatedItems.back())

libevmasm/CommonSubexpressionEliminator.h

@@ -71,13 +71,6 @@ public:
 	/// @returns the resulting items after optimization.
 	AssemblyItems getOptimizedItems();
 
-	/// Streams debugging information to @a _out.
-	std::ostream& stream(
-		std::ostream& _out,
-		std::map<int, Id> _initialStack = std::map<int, Id>(),
-		std::map<int, Id> _targetStack = std::map<int, Id>()
-	) const;
-
 private:
 	/// Feeds the item into the system for analysis.
 	void feedItem(AssemblyItem const& _item, bool _copyItem = false);
@@ -126,16 +119,15 @@ private:
 	void addDependencies(Id _c);
 
 	/// Produce code that generates the given element if it is not yet present.
-	/// @returns the stack position of the element or c_invalidPosition if it does not actually
-	/// generate a value on the stack.
 	/// @param _allowSequenced indicates that sequence-constrained operations are allowed
-	int generateClassElement(Id _c, bool _allowSequenced = false);
+	void generateClassElement(Id _c, bool _allowSequenced = false);
 	/// @returns the position of the representative of the given id on the stack.
 	/// @note throws an exception if it is not on the stack.
 	int classElementPosition(Id _id) const;
 
-	/// @returns true if @a _element can be removed - in general or, if given, while computing @a _result.
-	bool canBeRemoved(Id _element, Id _result = Id(-1));
+	/// @returns true if the copy of @a _element can be removed from stack position _fromPosition
+	/// - in general or, if given, while computing @a _result.
+	bool canBeRemoved(Id _element, Id _result = Id(-1), int _fromPosition = c_invalidPosition);
 
 	/// Appends code to remove the topmost stack element if it can be removed.
 	bool removeStackTopIfPossible();
@@ -157,8 +149,8 @@ private:
 	std::multimap<Id, Id> m_neededBy;
 	/// Current content of the stack.
 	std::map<int, Id> m_stack;
-	/// Current positions of equivalence classes, equal to c_invalidPosition if already deleted.
-	std::map<Id, int> m_classPositions;
+	/// Current positions of equivalence classes, equal to the empty set if already deleted.
+	std::map<Id, std::set<int>> m_classPositions;
 
 	/// The actual eqivalence class items and how to compute them.
 	ExpressionClasses& m_expressionClasses;
@@ -167,6 +159,7 @@ private:
 	std::map<std::pair<StoreOperation::Target, Id>, StoreOperations> m_storeOperations;
 	/// The set of equivalence classes that should be present on the stack at the end.
 	std::set<Id> m_finalClasses;
+	std::map<int, Id> m_targetStack;
 };
 
 template <class _AssemblyItemIterator>
@@ -175,6 +168,7 @@ _AssemblyItemIterator CommonSubexpressionEliminator::feedItems(
 	_AssemblyItemIterator _end
 )
 {
+	assertThrow(!m_breakingItem, OptimizerException, "Invalid use of CommonSubexpressionEliminator.");
 	for (; _iterator != _end && !SemanticInformation::breaksCSEAnalysisBlock(*_iterator); ++_iterator)
 		feedItem(*_iterator);
 	if (_iterator != _end)

libevmasm/ControlFlowGraph.cpp

@@ -142,7 +142,7 @@ void ControlFlowGraph::removeUnusedBlocks()
 		BasicBlock const& block = m_blocks.at(blocksToProcess.back());
 		blocksToProcess.pop_back();
 		for (BlockId tag: block.pushedTags)
-			if (!neededBlocks.count(tag))
+			if (!neededBlocks.count(tag) && m_blocks.count(tag))
 			{
 				neededBlocks.insert(tag);
 				blocksToProcess.push_back(tag);
@@ -191,12 +191,12 @@ void ControlFlowGraph::setPrevLinks()
 		if (push.type() != PushTag)
 			continue;
 		BlockId nextId(push.data());
-		if (m_blocks.at(nextId).prev)
+		if (m_blocks.count(nextId) && m_blocks.at(nextId).prev)
 			continue;
 		bool hasLoop = false;
-		for (BlockId id = nextId; id && !hasLoop; id = m_blocks.at(id).next)
+		for (BlockId id = nextId; id && m_blocks.count(id) && !hasLoop; id = m_blocks.at(id).next)
 			hasLoop = (id == blockId);
-		if (hasLoop)
+		if (hasLoop || !m_blocks.count(nextId))
 			continue;
 
 		m_blocks[nextId].prev = blockId;
@@ -225,6 +225,8 @@ void ControlFlowGraph::gatherKnowledge()
 	{
 		//@todo we might have to do something like incrementing the sequence number for each JUMPDEST
 		assertThrow(!!workQueue.back().first, OptimizerException, "");
+		if (!m_blocks.count(workQueue.back().first))
+			continue; // too bad, we do not know the tag, probably an invalid jump
 		BasicBlock& block = m_blocks.at(workQueue.back().first);
 		KnownStatePointer state = workQueue.back().second;
 		workQueue.pop_back();
@@ -281,6 +283,15 @@ void ControlFlowGraph::gatherKnowledge()
 		)
 			workQueue.push_back(make_pair(block.next, state->copy()));
 	}
+
+	// Remove all blocks we never visited here. This might happen because a tag is pushed but
+	// never used for a JUMP.
+	// Note that this invalidates some contents of pushedTags
+	for (auto it = m_blocks.begin(); it != m_blocks.end();)
+		if (!it->second.startState)
+			it = m_blocks.erase(it);
+		else
+			it++;
 }
 
 BasicBlocks ControlFlowGraph::rebuildCode()
@@ -288,7 +299,8 @@ BasicBlocks ControlFlowGraph::rebuildCode()
 	map<BlockId, unsigned> pushes;
 	for (auto& idAndBlock: m_blocks)
 		for (BlockId ref: idAndBlock.second.pushedTags)
-			pushes[ref]++;
+			if (m_blocks.count(ref))
+				pushes[ref]++;
 
 	set<BlockId> blocksToAdd;
 	for (auto it: m_blocks)

libevmasm/KnownState.cpp

@@ -160,23 +160,51 @@ KnownState::StoreOperation KnownState::feedItem(AssemblyItem const& _item, bool
 	return op;
 }
 
-void KnownState::reduceToCommonKnowledge(KnownState const& /*_other*/)
+/// Helper function for KnownState::reduceToCommonKnowledge, removes everything from
+/// _this which is not in or not equal to the value in _other.
+template <class _Mapping, class _KeyType> void intersect(
+	_Mapping& _this,
+	_Mapping const& _other,
+	function<_KeyType(_KeyType)> const& _keyTrans = [](_KeyType _k) { return _k; }
+)
+{
+	for (auto it = _this.begin(); it != _this.end();)
+		if (_other.count(_keyTrans(it->first)) && _other.at(_keyTrans(it->first)) == it->second)
+			++it;
+		else
+			it = _this.erase(it);
+}
+
+template <class _Mapping> void intersect(_Mapping& _this, _Mapping const& _other)
 {
-	//@todo
-	*this = KnownState(m_expressionClasses);
+	intersect<_Mapping, ExpressionClasses::Id>(_this, _other, [](ExpressionClasses::Id _k) { return _k; });
+}
+
+void KnownState::reduceToCommonKnowledge(KnownState const& _other)
+{
+	int stackDiff = m_stackHeight - _other.m_stackHeight;
+	function<int(int)> stackKeyTransform = [=](int _key) -> int { return _key - stackDiff; };
+	intersect(m_stackElements, _other.m_stackElements, stackKeyTransform);
+	// Use the smaller stack height. Essential to terminate in case of loops.
+	if (m_stackHeight > _other.m_stackHeight)
+	{
+		map<int, Id> shiftedStack;
+		for (auto const& stackElement: m_stackElements)
+			shiftedStack[stackElement.first - stackDiff] = stackElement.second;
+		m_stackElements = move(shiftedStack);
+		m_stackHeight = _other.m_stackHeight;
+	}
+
+	intersect(m_storageContent, _other.m_storageContent);
+	intersect(m_memoryContent, _other.m_memoryContent);
 }
 
 bool KnownState::operator==(const KnownState& _other) const
 {
-	//@todo
-	return (
-		m_stackElements.empty() &&
-		_other.m_stackElements.empty() &&
-		m_storageContent.empty() &&
-		_other.m_storageContent.empty() &&
-		m_memoryContent.empty() &&
-		_other.m_memoryContent.empty()
-	);
+	return m_storageContent == _other.m_storageContent &&
+		m_memoryContent == _other.m_memoryContent &&
+		m_stackHeight == _other.m_stackHeight &&
+		m_stackElements == _other.m_stackElements;
 }
 
 ExpressionClasses::Id KnownState::stackElement(int _stackHeight, SourceLocation const& _location)

libjsqrc/ethereumjs/bower.json

@@ -1,7 +1,7 @@
 {
   "name": "web3",
   "namespace": "ethereum",
-  "version": "0.3.3",
+  "version": "0.3.6",
   "description": "Ethereum Compatible JavaScript API",
   "main": [
     "./dist/web3.js",

libjsqrc/ethereumjs/dist/web3-light.js

@@ -22,118 +22,29 @@ require=(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof requ
  * @date 2014
  */
 
-var utils = require('../utils/utils');
 var coder = require('./coder');
-var solUtils = require('./utils');
-
-/**
- * Formats input params to bytes
- *
- * @method formatInput
- * @param {Array} abi inputs of method
- * @param {Array} params that will be formatted to bytes
- * @returns bytes representation of input params
- */
-var formatInput = function (inputs, params) {
-    var i = inputs.map(function (input) {
-        return input.type;
-    });
-    return coder.encodeParams(i, params);
-};
-
-/** 
- * Formats output bytes back to param list
- *
- * @method formatOutput
- * @param {Array} abi outputs of method
- * @param {String} bytes represention of output
- * @returns {Array} output params
- */
-var formatOutput = function (outs, bytes) {
-    var o = outs.map(function (out) {
-        return out.type;
-    });
-    
-    return coder.decodeParams(o, bytes); 
-};
-
-/**
- * Should be called to create input parser for contract with given abi
- *
- * @method inputParser
- * @param {Array} contract abi
- * @returns {Object} input parser object for given json abi
- * TODO: refactor creating the parser, do not double logic from contract
- */
-var inputParser = function (json) {
-    var parser = {};
-    json.forEach(function (method) {
-        var displayName = utils.extractDisplayName(method.name);
-        var typeName = utils.extractTypeName(method.name);
-
-        var impl = function () {
-            var params = Array.prototype.slice.call(arguments);
-            return formatInput(method.inputs, params);
-        };
-
-        if (parser[displayName] === undefined) {
-            parser[displayName] = impl;
-        }
-
-        parser[displayName][typeName] = impl;
-    });
-
-    return parser;
-};
-
-/**
- * Should be called to create output parser for contract with given abi
- *
- * @method outputParser
- * @param {Array} contract abi
- * @returns {Object} output parser for given json abi
- */
-var outputParser = function (json) {
-    var parser = {};
-    json.forEach(function (method) {
-
-        var displayName = utils.extractDisplayName(method.name);
-        var typeName = utils.extractTypeName(method.name);
-
-        var impl = function (output) {
-            return formatOutput(method.outputs, output);
-        };
-
-        if (parser[displayName] === undefined) {
-            parser[displayName] = impl;
-        }
-
-        parser[displayName][typeName] = impl;
-    });
-
-    return parser;
-};
+var utils = require('./utils');
 
 var formatConstructorParams = function (abi, params) {
-    var constructor = solUtils.getConstructor(abi, params.length);
+    var constructor = utils.getConstructor(abi, params.length);
     if (!constructor) {
         if (params.length > 0) {
             console.warn("didn't found matching constructor, using default one");
         }
         return '';
     }
-    return formatInput(constructor.inputs, params);
+
+    return coder.encodeParams(constructor.inputs.map(function (input) {
+        return input.type;
+    }), params);
 };
 
 module.exports = {
-    inputParser: inputParser,
-    outputParser: outputParser,
-    formatInput: formatInput,
-    formatOutput: formatOutput,
     formatConstructorParams: formatConstructorParams
 };
 
-},{"../utils/utils":8,"./coder":2,"./utils":5}],2:[function(require,module,exports){
+
+},{"./coder":2,"./utils":5}],2:[function(require,module,exports){
 /*
     This file is part of ethereum.js.
 
@@ -213,9 +124,8 @@ SolidityType.prototype.formatInput = function (param, arrayType) {
         return param.map(function (p) {
             return self._inputFormatter(p);
         }).reduce(function (acc, current) {
-            acc.appendArrayElement(current);
-            return acc;
-        }, new SolidityParam('', f.formatInputInt(param.length).value));
+            return acc.combine(current);
+        }, f.formatInputInt(param.length)).withOffset(32);
     } 
     return this._inputFormatter(param);
 };
@@ -232,9 +142,9 @@ SolidityType.prototype.formatOutput = function (param, arrayType) {
     if (arrayType) {
         // let's assume, that we solidity will never return long arrays :P 
         var result = [];
-        var length = new BigNumber(param.prefix, 16);
+        var length = new BigNumber(param.dynamicPart().slice(0, 64), 16);
         for (var i = 0; i < length * 64; i += 64) {
-            result.push(this._outputFormatter(new SolidityParam(param.suffix.slice(i, i + 64))));
+            result.push(this._outputFormatter(new SolidityParam(param.dynamicPart().substr(i + 64, 64))));
         }
         return result;
     }
@@ -242,31 +152,21 @@ SolidityType.prototype.formatOutput = function (param, arrayType) {
 };
 
 /**
- * Should be used to check if a type is variadic
+ * Should be used to slice single param from bytes
  *
- * @method isVariadicType
- * @param {String} type
- * @returns {Bool} true if the type is variadic
- */
-SolidityType.prototype.isVariadicType = function (type) {
-    return isArrayType(type) || this._mode === 'bytes';
-};
-
-/**
- * Should be used to shift param from params group
- *
- * @method shiftParam
+ * @method sliceParam
+ * @param {String} bytes
+ * @param {Number} index of param to slice
  * @param {String} type
- * @returns {SolidityParam} shifted param
+ * @returns {SolidityParam} param
  */
-SolidityType.prototype.shiftParam = function (type, param) {
+SolidityType.prototype.sliceParam = function (bytes, index, type) {
     if (this._mode === 'bytes') {
-        return param.shiftBytes();
+        return SolidityParam.decodeBytes(bytes, index);
     } else if (isArrayType(type)) {
-        var length = new BigNumber(param.prefix.slice(0, 64), 16);
-        return param.shiftArray(length);
+        return SolidityParam.decodeArray(bytes, index);
     }
-    return param.shiftValue();
+    return SolidityParam.decodeParam(bytes, index);
 };
 
 /**
@@ -296,28 +196,6 @@ SolidityCoder.prototype._requireType = function (type) {
     return solidityType;
 };
 
-/**
- * Should be used to transform plain bytes to SolidityParam object
- *
- * @method _bytesToParam
- * @param {Array} types of params
- * @param {String} bytes to be transformed to SolidityParam
- * @return {SolidityParam} SolidityParam for this group of params
- */
-SolidityCoder.prototype._bytesToParam = function (types, bytes) {
-    var self = this;
-    var prefixTypes = types.reduce(function (acc, type) {
-        return self._requireType(type).isVariadicType(type) ? acc + 1 : acc;
-    }, 0);
-    var valueTypes = types.length - prefixTypes;
-
-    var prefix = bytes.slice(0, prefixTypes * 64);
-    bytes = bytes.slice(prefixTypes * 64);
-    var value = bytes.slice(0, valueTypes * 64);
-    var suffix = bytes.slice(valueTypes * 64);
-    return new SolidityParam(value, prefix, suffix); 
-};
-
 /**
  * Should be used to transform plain param of given type to SolidityParam
  *
@@ -352,24 +230,11 @@ SolidityCoder.prototype.encodeParam = function (type, param) {
  */
 SolidityCoder.prototype.encodeParams = function (types, params) {
     var self = this;
-    return types.map(function (type, index) {
+    var solidityParams = types.map(function (type, index) {
         return self._formatInput(type, params[index]);
-    }).reduce(function (acc, solidityParam) {
-        acc.append(solidityParam);
-        return acc;
-    }, new SolidityParam()).encode();
-};
+    });
 
-/**
- * Should be used to transform SolidityParam to plain param
- *
- * @method _formatOutput
- * @param {String} type
- * @param {SolidityParam} param
- * @return {Object} plain param
- */
-SolidityCoder.prototype._formatOutput = function (type, param) {
-    return this._requireType(type).formatOutput(param, isArrayType(type));
+    return SolidityParam.encodeList(solidityParams);
 };
 
 /**
@@ -381,7 +246,7 @@ SolidityCoder.prototype._formatOutput = function (type, param) {
  * @return {Object} plain param
  */
 SolidityCoder.prototype.decodeParam = function (type, bytes) {
-    return this._formatOutput(type, this._bytesToParam([type], bytes));
+    return this.decodeParams([type], bytes)[0];
 };
 
 /**
@@ -394,10 +259,9 @@ SolidityCoder.prototype.decodeParam = function (type, bytes) {
  */
 SolidityCoder.prototype.decodeParams = function (types, bytes) {
     var self = this;
-    var param = this._bytesToParam(types, bytes);
-    return types.map(function (type) {
+    return types.map(function (type, index) {
         var solidityType = self._requireType(type);
-        var p = solidityType.shiftParam(type, param);
+        var p = solidityType.sliceParam(bytes, index, type);
         return solidityType.formatOutput(p, isArrayType(type));
     });
 };
@@ -530,7 +394,7 @@ var formatInputBytes = function (value) {
  */
 var formatInputDynamicBytes = function (value) {
     var result = utils.fromAscii(value, c.ETH_PADDING).substr(2);
-    return new SolidityParam('', formatInputInt(value.length).value, result);
+    return new SolidityParam(formatInputInt(value.length).value + result, 32);
 };
 
 /**
@@ -576,7 +440,7 @@ var signedIsNegative = function (value) {
  * @returns {BigNumber} right-aligned output bytes formatted to big number
  */
 var formatOutputInt = function (param) {
-    var value = param.value || "0";
+    var value = param.staticPart() || "0";
 
     // check if it's negative number
     // it it is, return two's complement
@@ -594,7 +458,7 @@ var formatOutputInt = function (param) {
  * @returns {BigNumeber} right-aligned output bytes formatted to uint
  */
 var formatOutputUInt = function (param) {
-    var value = param.value || "0";
+    var value = param.staticPart() || "0";
     return new BigNumber(value, 16);
 };
 
@@ -628,7 +492,7 @@ var formatOutputUReal = function (param) {
  * @returns {Boolean} right-aligned input bytes formatted to bool
  */
 var formatOutputBool = function (param) {
-    return param.value === '0000000000000000000000000000000000000000000000000000000000000001' ? true : false;
+    return param.staticPart() === '0000000000000000000000000000000000000000000000000000000000000001' ? true : false;
 };
 
 /**
@@ -640,7 +504,7 @@ var formatOutputBool = function (param) {
  */
 var formatOutputBytes = function (param) {
     // length might also be important!
-    return utils.toAscii(param.value);
+    return utils.toAscii(param.staticPart());
 };
 
 /**
@@ -652,7 +516,7 @@ var formatOutputBytes = function (param) {
  */
 var formatOutputDynamicBytes = function (param) {
     // length might also be important!
-    return utils.toAscii(param.suffix);
+    return utils.toAscii(param.dynamicPart().slice(64));
 };
 
 /**
@@ -663,7 +527,7 @@ var formatOutputDynamicBytes = function (param) {
  * @returns {String} address
  */
 var formatOutputAddress = function (param) {
-    var value = param.value;
+    var value = param.staticPart();
     return "0x" + value.slice(value.length - 40, value.length);
 };
 
@@ -707,91 +571,196 @@ module.exports = {
  * @date 2015
  */
 
+var utils = require('../utils/utils');
+
 /**
  * SolidityParam object prototype.
  * Should be used when encoding, decoding solidity bytes
  */
-var SolidityParam = function (value, prefix, suffix) {
-    this.prefix = prefix || '';
+var SolidityParam = function (value, offset) {
     this.value = value || '';
-    this.suffix = suffix || '';
+    this.offset = offset; // offset in bytes
+};
+
+/**
+ * This method should be used to get length of params's dynamic part
+ * 
+ * @method dynamicPartLength
+ * @returns {Number} length of dynamic part (in bytes)
+ */
+SolidityParam.prototype.dynamicPartLength = function () {
+    return this.dynamicPart().length / 2;
+};
+
+/**
+ * This method should be used to create copy of solidity param with different offset
+ *
+ * @method withOffset
+ * @param {Number} offset length in bytes
+ * @returns {SolidityParam} new solidity param with applied offset
+ */
+SolidityParam.prototype.withOffset = function (offset) {
+    return new SolidityParam(this.value, offset);
 };
 
 /**
- * This method should be used to encode two params one after another
+ * This method should be used to combine solidity params together
+ * eg. when appending an array
  *
- * @method append
- * @param {SolidityParam} param that it appended after this
+ * @method combine
+ * @param {SolidityParam} param with which we should combine
+ * @param {SolidityParam} result of combination
  */
-SolidityParam.prototype.append = function (param) {
-    this.prefix += param.prefix;
-    this.value += param.value;
-    this.suffix += param.suffix;
+SolidityParam.prototype.combine = function (param) {
+    return new SolidityParam(this.value + param.value); 
 };
 
 /**
- * This method should be used to encode next param in an array
+ * This method should be called to check if param has dynamic size.
+ * If it has, it returns true, otherwise false
  *
- * @method appendArrayElement
- * @param {SolidityParam} param that is appended to an array
+ * @method isDynamic
+ * @returns {Boolean}
  */
-SolidityParam.prototype.appendArrayElement = function (param) {
-    this.suffix += param.value;
-    this.prefix += param.prefix;
-    // TODO: suffix not supported = it's required for nested arrays;
+SolidityParam.prototype.isDynamic = function () {
+    return this.value.length > 64;
 };
 
 /**
- * This method should be used to create bytearrays from param
+ * This method should be called to transform offset to bytes
+ *
+ * @method offsetAsBytes
+ * @returns {String} bytes representation of offset
+ */
+SolidityParam.prototype.offsetAsBytes = function () {
+    return !this.isDynamic() ? '' : utils.padLeft(utils.toTwosComplement(this.offset).toString(16), 64);
+};
+
+/**
+ * This method should be called to get static part of param
+ *
+ * @method staticPart
+ * @returns {String} offset if it is a dynamic param, otherwise value
+ */
+SolidityParam.prototype.staticPart = function () {
+    if (!this.isDynamic()) {
+        return this.value; 
+    } 
+    return this.offsetAsBytes();
+};
+
+/**
+ * This method should be called to get dynamic part of param
+ *
+ * @method dynamicPart
+ * @returns {String} returns a value if it is a dynamic param, otherwise empty string
+ */
+SolidityParam.prototype.dynamicPart = function () {
+    return this.isDynamic() ? this.value : '';
+};
+
+/**
+ * This method should be called to encode param
  *
  * @method encode
- * @return {String} encoded param(s)
+ * @returns {String}
  */
 SolidityParam.prototype.encode = function () {
-    return this.prefix + this.value + this.suffix;
+    return this.staticPart() + this.dynamicPart();
 };
 
 /**
- * This method should be used to shift first param from group of params
+ * This method should be called to encode array of params
  *
- * @method shiftValue
- * @return {SolidityParam} first value param
+ * @method encodeList
+ * @param {Array[SolidityParam]} params
+ * @returns {String}
  */
-SolidityParam.prototype.shiftValue = function () {
-    var value = this.value.slice(0, 64);
-    this.value = this.value.slice(64);
-    return new SolidityParam(value);
+SolidityParam.encodeList = function (params) {
+    
+    // updating offsets
+    var totalOffset = params.length * 32;
+    var offsetParams = params.map(function (param) {
+        if (!param.isDynamic()) {
+            return param;
+        }
+        var offset = totalOffset;
+        totalOffset += param.dynamicPartLength();
+        return param.withOffset(offset);
+    });
+
+    // encode everything!
+    return offsetParams.reduce(function (result, param) {
+        return result + param.dynamicPart();
+    }, offsetParams.reduce(function (result, param) {
+        return result + param.staticPart();
+    }, ''));
 };
 
 /**
- * This method should be used to first bytes param from group of params
+ * This method should be used to decode plain (static) solidity param at given index
  *
- * @method shiftBytes
- * @return {SolidityParam} first bytes param
+ * @method decodeParam
+ * @param {String} bytes
+ * @param {Number} index
+ * @returns {SolidityParam}
  */
-SolidityParam.prototype.shiftBytes = function () {
-    return this.shiftArray(1);   
+SolidityParam.decodeParam = function (bytes, index) {
+    index = index || 0;
+    return new SolidityParam(bytes.substr(index * 64, 64)); 
 };
 
 /**
- * This method should be used to shift an array from group of params 
- * 
- * @method shiftArray
- * @param {Number} size of an array to shift
- * @return {SolidityParam} first array param
+ * This method should be called to get offset value from bytes at given index
+ *
+ * @method getOffset
+ * @param {String} bytes
+ * @param {Number} index
+ * @returns {Number} offset as number
+ */
+var getOffset = function (bytes, index) {
+    // we can do this cause offset is rather small
+    return parseInt('0x' + bytes.substr(index * 64, 64));
+};
+
+/**
+ * This method should be called to decode solidity bytes param at given index
+ *
+ * @method decodeBytes
+ * @param {String} bytes
+ * @param {Number} index
+ * @returns {SolidityParam}
+ */
+SolidityParam.decodeBytes = function (bytes, index) {
+    index = index || 0;
+    //TODO add support for strings longer than 32 bytes
+    //var length = parseInt('0x' + bytes.substr(offset * 64, 64));
+
+    var offset = getOffset(bytes, index);
+
+    // 2 * , cause we also parse length
+    return new SolidityParam(bytes.substr(offset * 2, 2 * 64));
+};
+
+/**
+ * This method should be used to decode solidity array at given index
+ *
+ * @method decodeArray
+ * @param {String} bytes
+ * @param {Number} index
+ * @returns {SolidityParam}
  */
-SolidityParam.prototype.shiftArray = function (length) {
-    var prefix = this.prefix.slice(0, 64);
-    this.prefix = this.value.slice(64);
-    var suffix = this.suffix.slice(0, 64 * length);
-    this.suffix = this.suffix.slice(64 * length);
-    return new SolidityParam('', prefix, suffix);
+SolidityParam.decodeArray = function (bytes, index) {
+    index = index || 0;
+    var offset = getOffset(bytes, index);
+    var length = parseInt('0x' + bytes.substr(offset * 2, 64));
+    return new SolidityParam(bytes.substr(offset * 2, (length + 1) * 64));
 };
 
 module.exports = SolidityParam;
 
 
-},{}],5:[function(require,module,exports){
+},{"../utils/utils":8}],5:[function(require,module,exports){
 /*
     This file is part of ethereum.js.
 
@@ -828,6 +797,11 @@ var getConstructor = function (abi, numberOfArgs) {
     })[0];
 };
 
+//var getSupremeType = function (type) {
+    //return type.substr(0, type.indexOf('[')) + ']';
+//};
+
+
 module.exports = {
     getConstructor: getConstructor
 };
@@ -1394,7 +1368,7 @@ module.exports = {
 
 },{"bignumber.js":"bignumber.js"}],9:[function(require,module,exports){
 module.exports={
-    "version": "0.3.3"
+    "version": "0.3.6"
 }
 
 },{}],10:[function(require,module,exports){
@@ -1796,7 +1770,7 @@ module.exports = {
 
 /**
  * Web3
- * 
+ *
  * @module web3
  */
 
@@ -1850,16 +1824,16 @@ var uncleCountCall = function (args) {
 /// @returns an array of objects describing web3.eth api methods
 
 var getBalance = new Method({
-    name: 'getBalance', 
-    call: 'eth_getBalance', 
+    name: 'getBalance',
+    call: 'eth_getBalance',
     params: 2,
     inputFormatter: [utils.toAddress, formatters.inputDefaultBlockNumberFormatter],
     outputFormatter: formatters.outputBigNumberFormatter
 });
 
 var getStorageAt = new Method({
-    name: 'getStorageAt', 
-    call: 'eth_getStorageAt', 
+    name: 'getStorageAt',
+    call: 'eth_getStorageAt',
     params: 3,
     inputFormatter: [null, utils.toHex, formatters.inputDefaultBlockNumberFormatter]
 });
@@ -1872,7 +1846,7 @@ var getCode = new Method({
 });
 
 var getBlock = new Method({
-    name: 'getBlock', 
+    name: 'getBlock',
     call: blockCall,
     params: 2,
     inputFormatter: [formatters.inputBlockNumberFormatter, function (val) { return !!val; }],
@@ -1997,6 +1971,11 @@ var properties = [
         name: 'mining',
         getter: 'eth_mining'
     }),
+    new Property({
+        name: 'hashrate',
+        getter: 'eth_hashrate',
+        outputFormatter: utils.toDecimal
+    }),
     new Property({
         name: 'gasPrice',
         getter: 'eth_gasPrice',
@@ -2118,7 +2097,7 @@ SolidityEvent.prototype.encode = function (indexed, options) {
     ['fromBlock', 'toBlock'].filter(function (f) {
         return options[f] !== undefined;
     }).forEach(function (f) {
-        result[f] = utils.toHex(options[f]);
+        result[f] = formatters.inputBlockNumberFormatter(options[f]);
     });
 
     result.topics = [];
@@ -2447,7 +2426,7 @@ var inputTransactionFormatter = function (options){
         delete options.code;
     }
 
-    ['gasPrice', 'gas', 'value'].filter(function (key) {
+    ['gasPrice', 'gas', 'value', 'nonce'].filter(function (key) {
         return options[key] !== undefined;
     }).forEach(function(key){
         options[key] = utils.fromDecimal(options[key]);
@@ -2796,15 +2775,32 @@ HttpProvider.prototype.send = function (payload) {
     //if (request.status !== 200) {
         //return;
     //}
-    return JSON.parse(request.responseText);
+
+    var result = request.responseText;
+
+    try {
+        result = JSON.parse(result);
+    } catch(e) {
+        throw errors.InvalidResponse(result);                
+    }
+
+    return result;
 };
 
 HttpProvider.prototype.sendAsync = function (payload, callback) {
     var request = new XMLHttpRequest();
     request.onreadystatechange = function() {
         if (request.readyState === 4) {
-            // TODO: handle the error properly here!!!
-            callback(null, JSON.parse(request.responseText));
+            var result = request.responseText;
+            var error = null;
+
+            try {
+                result = JSON.parse(result);
+            } catch(e) {
+                error = errors.InvalidResponse(result);                
+            }
+
+            callback(error, result);
         }
     };