Merge pull request #2814 from chfast/evmjit-resolve-jumps

EVMJIT: better jumps resolving

evmjit/libevmjit/Array.h

@@ -58,7 +58,7 @@ private:
 	llvm::Function* createExtendFunc();
 	llvm::Function* getReallocFunc();
 
-	LazyFunction m_pushFunc = {[this](){ return createArrayPushFunc(); }}; // TODO: If works on MSVC, remove form initialization list
+	LazyFunction m_pushFunc = {[this](){ return createArrayPushFunc(); }};
 	LazyFunction m_setFunc = {[this](){ return createArraySetFunc(); }};
 	LazyFunction m_getPtrFunc = {[this](){ return createGetPtrFunc(); }};
 	LazyFunction m_getFunc = {[this](){ return createArrayGetFunc(); }};

evmjit/libevmjit/BasicBlock.cpp

@@ -110,11 +110,13 @@ void LocalStack::set(size_t _index, llvm::Value* _word)
 void LocalStack::finalize(llvm::IRBuilder<>& _builder, llvm::BasicBlock& _bb)
 {
 	auto blockTerminator = _bb.getTerminator();
-	assert(blockTerminator);
+	if (!blockTerminator || blockTerminator->getOpcode() != llvm::Instruction::Ret)
-	if (blockTerminator->getOpcode() != llvm::Instruction::Ret)
 	{
 		// Not needed in case of ret instruction. Ret invalidates the stack.
-		_builder.SetInsertPoint(blockTerminator);
+		if (blockTerminator)
+			_builder.SetInsertPoint(blockTerminator);
+		else
+			_builder.SetInsertPoint(&_bb);
 
 		// Update items fetched from global stack ignoring the poped ones
 		assert(m_globalPops <= m_input.size()); // pop() always does get()

evmjit/libevmjit/Compiler.cpp

@@ -29,6 +29,8 @@ namespace eth
 namespace jit
 {
 
+static const auto c_destIdxLabel = "destIdx";
+
 Compiler::Compiler(Options const& _options):
 	m_options(_options),
 	m_builder(llvm::getGlobalContext())
@@ -93,21 +95,34 @@ std::vector<BasicBlock> Compiler::createBasicBlocks(code_iterator _codeBegin, co
 	return blocks;
 }
 
-void Compiler::fillJumpTable()
+void Compiler::resolveJumps()
 {
-	assert(m_jumpTableBB);
+	// Iterate through all EVM instructions blocks (skip first 4 - special blocks).
-	if (llvm::pred_empty(m_jumpTableBB))
+	for (auto it = std::next(m_mainFunc->begin(), 4); it != m_mainFunc->end(); ++it)
-	{
-		m_jumpTableBB->eraseFromParent(); // remove if unused
-		return;
-	}
-
-	// TODO: Extend this function as `resolveJumps()` and fill gaps in branch instructions.
-	auto target = llvm::cast<llvm::PHINode>(m_jumpTableBB->begin());
-	for (auto pred: llvm::predecessors(m_jumpTableBB))
 	{
-		auto targetMd = llvm::cast<llvm::LocalAsMetadata>(pred->getTerminator()->getMetadata("target")->getOperand(0));
+		auto jumpTable = llvm::cast<llvm::SwitchInst>(m_jumpTableBB->getTerminator());
-		target->addIncoming(targetMd->getValue(), pred);
+		auto jumpTableInput = llvm::cast<llvm::PHINode>(m_jumpTableBB->begin());
+		auto nextBlock = it->getNextNode() != m_mainFunc->end() ? it->getNextNode() : m_stopBB;
+		auto term = it->getTerminator();
+
+		if (!term) // Block may have no terminator if the next instruction is a jump destination.
+			llvm::IRBuilder<>{it}.CreateBr(nextBlock);
+		else if (auto jump = llvm::dyn_cast<llvm::BranchInst>(term))
+			if (jump->getSuccessor(0) == m_jumpTableBB)
+			{
+				auto destIdx = llvm::cast<llvm::ValueAsMetadata>(jump->getMetadata(c_destIdxLabel)->getOperand(0))->getValue();
+				if (auto constant = llvm::dyn_cast<llvm::ConstantInt>(destIdx))
+				{
+					// If destination index is a constant do direct jump to the destination block.
+					auto bb = jumpTable->findCaseValue(constant).getCaseSuccessor();
+					jump->setSuccessor(0, bb);
+				}
+				else
+					jumpTableInput->addIncoming(destIdx, it); // Fill up PHI node
+
+				if (jump->isConditional())
+					jump->setSuccessor(1, nextBlock); // Set next block for conditional jumps
+			}
 	}
 }
 
@@ -160,13 +175,8 @@ std::unique_ptr<llvm::Module> Compiler::compile(code_iterator _begin, code_itera
 	auto firstBB = blocks.empty() ? m_stopBB : blocks.front().llvm();
 	m_builder.CreateCondBr(normalFlow, firstBB, m_abortBB, Type::expectTrue);
 
-	for (auto it = blocks.begin(); it != blocks.end(); ++it)
+	for (auto& block: blocks)
-	{
+		compileBasicBlock(block, runtimeManager, arith, memory, ext, gasMeter, stack);
-		// TODO: Rewrite
-		auto nextIt = it + 1;
-		auto nextBasicBlock = (nextIt != blocks.end()) ? nextIt->llvm() : nullptr; // TODO: What with Stop block?
-		compileBasicBlock(*it, runtimeManager, arith, memory, ext, gasMeter, nextBasicBlock, stack, jumpTable);
-	}
 
 	// Code for special blocks:
 	m_builder.SetInsertPoint(m_stopBB);
@@ -175,19 +185,15 @@ std::unique_ptr<llvm::Module> Compiler::compile(code_iterator _begin, code_itera
 	m_builder.SetInsertPoint(m_abortBB);
 	runtimeManager.exit(ReturnCode::OutOfGas);
 
-	fillJumpTable();
+	resolveJumps();
 
 	return module;
 }
 
 
 void Compiler::compileBasicBlock(BasicBlock& _basicBlock, RuntimeManager& _runtimeManager,
-								 Arith256& _arith, Memory& _memory, Ext& _ext, GasMeter& _gasMeter, llvm::BasicBlock* _nextBasicBlock, Stack& _globalStack,
+								 Arith256& _arith, Memory& _memory, Ext& _ext, GasMeter& _gasMeter, Stack& _globalStack)
-							 	 llvm::SwitchInst& jumpTable)
 {
-	if (!_nextBasicBlock) // this is the last block in the code
-		_nextBasicBlock = m_stopBB;
-
 	m_builder.SetInsertPoint(_basicBlock.llvm());
 	LocalStack stack{_globalStack};
 
@@ -556,24 +562,16 @@ void Compiler::compileBasicBlock(BasicBlock& _basicBlock, RuntimeManager& _runti
 		case Instruction::JUMP:
 		case Instruction::JUMPI:
 		{
-			auto jumpBlock = m_jumpTableBB;
+			auto destIdx = llvm::MDNode::get(m_builder.getContext(), llvm::ValueAsMetadata::get(stack.pop()));
-			auto target = stack.pop();
+
+			// Create branch instruction, initially to jump table.
+			// Destination will be optimized with direct jump during jump resolving if destination index is a constant.
 			auto jumpInst = (inst == Instruction::JUMP) ?
-					m_builder.CreateBr(jumpBlock) :
+					m_builder.CreateBr(m_jumpTableBB) :
-					m_builder.CreateCondBr(m_builder.CreateICmpNE(stack.pop(), Constant::get(0), "jump.check"), jumpBlock, _nextBasicBlock);
+					m_builder.CreateCondBr(m_builder.CreateICmpNE(stack.pop(), Constant::get(0), "jump.check"), m_jumpTableBB, nullptr);
 
-			if (auto constant = llvm::dyn_cast<llvm::ConstantInt>(target))
+			// Attach medatada to branch instruction with information about destination index.
-			{
+			jumpInst->setMetadata(c_destIdxLabel, destIdx);
-				// If target index is a constant do direct jump to the target block.
-				auto bb = jumpTable.findCaseValue(constant).getCaseSuccessor();
-				jumpInst->setSuccessor(0, bb);
-			}
-			else
-			{
-				// Attach medatada to branch instruction with information about target index.
-				auto targetMd = llvm::MDNode::get(jumpInst->getContext(), llvm::LocalAsMetadata::get(target));
-				jumpInst->setMetadata("target", targetMd);
-			}
 			break;
 		}
 
@@ -790,10 +788,6 @@ void Compiler::compileBasicBlock(BasicBlock& _basicBlock, RuntimeManager& _runti
 
 	_gasMeter.commitCostBlock();
 
-	// Block may have no terminator if the next instruction is a jump destination.
-	if (!_basicBlock.llvm()->getTerminator())
-		m_builder.CreateBr(_nextBasicBlock);
-
 	stack.finalize(m_builder, *_basicBlock.llvm()); // TODO: Use references
 
 	m_builder.SetInsertPoint(_basicBlock.llvm()->getFirstNonPHI()); // TODO: Move to LocalStack::finalize

evmjit/libevmjit/Compiler.h

@@ -31,9 +31,9 @@ private:
 	std::vector<BasicBlock> createBasicBlocks(code_iterator _begin, code_iterator _end, llvm::SwitchInst& _switchInst);
 
 	void compileBasicBlock(BasicBlock& _basicBlock, class RuntimeManager& _runtimeManager, class Arith256& _arith, class Memory& _memory, class Ext& _ext, class GasMeter& _gasMeter,
-			llvm::BasicBlock* _nextBasicBlock, class Stack& _globalStack, llvm::SwitchInst& _jumpTable);
+			class Stack& _globalStack);
 
-	void fillJumpTable();
+	void resolveJumps();
 
 	/// Compiler options
 	Options const& m_options;

evmjit/libevmjit/Stack.cpp

@@ -7,8 +7,6 @@
 #include "RuntimeManager.h"
 #include "Utils.h"
 
-#include <set> // DEBUG only
-
 namespace dev
 {
 namespace eth