|
|
@ -19,6 +19,7 @@ namespace jit |
|
|
|
|
|
|
|
|
Memory::Memory(RuntimeManager& _runtimeManager, GasMeter& _gasMeter): |
|
|
Memory::Memory(RuntimeManager& _runtimeManager, GasMeter& _gasMeter): |
|
|
RuntimeHelper(_runtimeManager), // TODO: RuntimeHelper not needed
|
|
|
RuntimeHelper(_runtimeManager), // TODO: RuntimeHelper not needed
|
|
|
|
|
|
m_memory{getBuilder(), _runtimeManager.getMem()}, |
|
|
m_gasMeter(_gasMeter) |
|
|
m_gasMeter(_gasMeter) |
|
|
{} |
|
|
{} |
|
|
|
|
|
|
|
|
@ -27,20 +28,20 @@ llvm::Function* Memory::getRequireFunc() |
|
|
auto& func = m_require; |
|
|
auto& func = m_require; |
|
|
if (!func) |
|
|
if (!func) |
|
|
{ |
|
|
{ |
|
|
llvm::Type* argTypes[] = {Type::RuntimePtr, Type::Word, Type::Word}; |
|
|
llvm::Type* argTypes[] = {Array::getType()->getPointerTo(), Type::Word, Type::Word, Type::BytePtr, Type::GasPtr}; |
|
|
func = llvm::Function::Create(llvm::FunctionType::get(Type::Void, argTypes, false), llvm::Function::PrivateLinkage, "mem.require", getModule()); |
|
|
func = llvm::Function::Create(llvm::FunctionType::get(Type::Void, argTypes, false), llvm::Function::PrivateLinkage, "mem.require", getModule()); |
|
|
auto rt = func->arg_begin(); |
|
|
func->setDoesNotThrow(); |
|
|
rt->setName("rt"); |
|
|
|
|
|
auto offset = rt->getNextNode(); |
|
|
auto mem = &func->getArgumentList().front(); |
|
|
offset->setName("offset"); |
|
|
mem->setName("mem"); |
|
|
auto size = offset->getNextNode(); |
|
|
auto blkOffset = mem->getNextNode(); |
|
|
size->setName("size"); |
|
|
blkOffset->setName("blkOffset"); |
|
|
|
|
|
auto blkSize = blkOffset->getNextNode(); |
|
|
llvm::Type* resizeArgs[] = {Type::RuntimePtr, Type::WordPtr}; |
|
|
blkSize->setName("blkSize"); |
|
|
auto resize = llvm::Function::Create(llvm::FunctionType::get(Type::BytePtr, resizeArgs, false), llvm::Function::ExternalLinkage, "mem_resize", getModule()); |
|
|
auto jmpBuf = blkSize->getNextNode(); |
|
|
llvm::AttrBuilder attrBuilder; |
|
|
jmpBuf->setName("jmpBuf"); |
|
|
attrBuilder.addAttribute(llvm::Attribute::NoAlias).addAttribute(llvm::Attribute::NoCapture).addAttribute(llvm::Attribute::NonNull).addAttribute(llvm::Attribute::ReadOnly); |
|
|
auto gas = jmpBuf->getNextNode(); |
|
|
resize->setAttributes(llvm::AttributeSet::get(resize->getContext(), 1, attrBuilder)); |
|
|
gas->setName("gas"); |
|
|
|
|
|
|
|
|
auto preBB = llvm::BasicBlock::Create(func->getContext(), "Pre", func); |
|
|
auto preBB = llvm::BasicBlock::Create(func->getContext(), "Pre", func); |
|
|
auto checkBB = llvm::BasicBlock::Create(func->getContext(), "Check", func); |
|
|
auto checkBB = llvm::BasicBlock::Create(func->getContext(), "Check", func); |
|
|
@ -51,40 +52,38 @@ llvm::Function* Memory::getRequireFunc() |
|
|
|
|
|
|
|
|
// BB "Pre": Ignore checks with size 0
|
|
|
// BB "Pre": Ignore checks with size 0
|
|
|
m_builder.SetInsertPoint(preBB); |
|
|
m_builder.SetInsertPoint(preBB); |
|
|
auto sizeIsZero = m_builder.CreateICmpEQ(size, Constant::get(0)); |
|
|
m_builder.CreateCondBr(m_builder.CreateICmpNE(blkSize, Constant::get(0)), checkBB, returnBB, Type::expectTrue); |
|
|
m_builder.CreateCondBr(sizeIsZero, returnBB, checkBB); |
|
|
|
|
|
|
|
|
|
|
|
// BB "Check"
|
|
|
// BB "Check"
|
|
|
m_builder.SetInsertPoint(checkBB); |
|
|
m_builder.SetInsertPoint(checkBB); |
|
|
auto uaddWO = llvm::Intrinsic::getDeclaration(getModule(), llvm::Intrinsic::uadd_with_overflow, Type::Word); |
|
|
static const auto c_inputMax = uint64_t(1) << 33; // max value of blkSize and blkOffset that will not result in integer overflow in calculations below
|
|
|
auto uaddRes = m_builder.CreateCall2(uaddWO, offset, size, "res"); |
|
|
auto blkOffsetOk = m_builder.CreateICmpULE(blkOffset, Constant::get(c_inputMax), "blkOffsetOk"); |
|
|
auto sizeRequired = m_builder.CreateExtractValue(uaddRes, 0, "sizeReq"); |
|
|
auto blkO = m_builder.CreateSelect(blkOffsetOk, m_builder.CreateTrunc(blkOffset, Type::Size), m_builder.getInt64(c_inputMax), "bklO"); |
|
|
auto overflow1 = m_builder.CreateExtractValue(uaddRes, 1, "overflow1"); |
|
|
auto blkSizeOk = m_builder.CreateICmpULE(blkSize, Constant::get(c_inputMax), "blkSizeOk"); |
|
|
auto rtPtr = getRuntimeManager().getRuntimePtr(); |
|
|
auto blkS = m_builder.CreateSelect(blkSizeOk, m_builder.CreateTrunc(blkSize, Type::Size), m_builder.getInt64(c_inputMax), "bklS"); |
|
|
auto sizePtr = m_builder.CreateStructGEP(rtPtr, 4); |
|
|
|
|
|
auto currSize = m_builder.CreateLoad(sizePtr, "currSize"); |
|
|
auto sizeReq0 = m_builder.CreateNUWAdd(blkO, blkS, "sizeReq0"); |
|
|
auto tooSmall = m_builder.CreateICmpULE(currSize, sizeRequired, "tooSmall"); |
|
|
auto sizeReq = m_builder.CreateAnd(m_builder.CreateNUWAdd(sizeReq0, m_builder.getInt64(31)), uint64_t(-1) << 5, "sizeReq"); // s' = ((s0 + 31) / 32) * 32
|
|
|
auto resizeNeeded = m_builder.CreateOr(tooSmall, overflow1, "resizeNeeded"); |
|
|
auto sizeCur = m_memory.size(mem); |
|
|
m_builder.CreateCondBr(resizeNeeded, resizeBB, returnBB); // OPT branch weights?
|
|
|
auto sizeOk = m_builder.CreateICmpULE(sizeReq, sizeCur, "sizeOk"); |
|
|
|
|
|
|
|
|
|
|
|
m_builder.CreateCondBr(sizeOk, returnBB, resizeBB, Type::expectTrue); |
|
|
|
|
|
|
|
|
// BB "Resize"
|
|
|
// BB "Resize"
|
|
|
m_builder.SetInsertPoint(resizeBB); |
|
|
m_builder.SetInsertPoint(resizeBB); |
|
|
// Check gas first
|
|
|
// Check gas first
|
|
|
uaddRes = m_builder.CreateCall2(uaddWO, sizeRequired, Constant::get(31), "res"); |
|
|
auto w1 = m_builder.CreateLShr(sizeReq, 5); |
|
|
auto wordsRequired = m_builder.CreateExtractValue(uaddRes, 0); |
|
|
auto w1s = m_builder.CreateNUWMul(w1, w1); |
|
|
auto overflow2 = m_builder.CreateExtractValue(uaddRes, 1, "overflow2"); |
|
|
auto c1 = m_builder.CreateAdd(m_builder.CreateNUWMul(w1, m_builder.getInt64(3)), m_builder.CreateLShr(w1s, 9)); |
|
|
auto overflow = m_builder.CreateOr(overflow1, overflow2, "overflow"); |
|
|
auto w0 = m_builder.CreateLShr(sizeCur, 5); |
|
|
wordsRequired = m_builder.CreateSelect(overflow, Constant::get(-1), wordsRequired); |
|
|
auto w0s = m_builder.CreateNUWMul(w0, w0); |
|
|
wordsRequired = m_builder.CreateUDiv(wordsRequired, Constant::get(32), "wordsReq"); |
|
|
auto c0 = m_builder.CreateAdd(m_builder.CreateNUWMul(w0, m_builder.getInt64(3)), m_builder.CreateLShr(w0s, 9)); |
|
|
sizeRequired = m_builder.CreateMul(wordsRequired, Constant::get(32), "roundedSizeReq"); |
|
|
auto cc = m_builder.CreateNUWSub(c1, c0); |
|
|
auto words = m_builder.CreateUDiv(currSize, Constant::get(32), "words"); // size is always 32*k
|
|
|
auto costOk = m_builder.CreateAnd(blkOffsetOk, blkSizeOk, "costOk"); |
|
|
auto newWords = m_builder.CreateSub(wordsRequired, words, "addtionalWords"); |
|
|
auto c = m_builder.CreateSelect(costOk, cc, m_builder.getInt64(std::numeric_limits<int64_t>::max()), "c"); |
|
|
m_gasMeter.countMemory(newWords); |
|
|
m_gasMeter.count(c, jmpBuf, gas); |
|
|
// Resize
|
|
|
// Resize
|
|
|
m_builder.CreateStore(sizeRequired, sizePtr); |
|
|
m_memory.extend(mem, sizeReq); |
|
|
auto newData = m_builder.CreateCall2(resize, rt, sizePtr, "newData"); |
|
|
|
|
|
auto dataPtr = m_builder.CreateStructGEP(rtPtr, 3); |
|
|
|
|
|
m_builder.CreateStore(newData, dataPtr); |
|
|
|
|
|
m_builder.CreateBr(returnBB); |
|
|
m_builder.CreateBr(returnBB); |
|
|
|
|
|
|
|
|
// BB "Return"
|
|
|
// BB "Return"
|
|
|
@ -94,12 +93,12 @@ llvm::Function* Memory::getRequireFunc() |
|
|
return func; |
|
|
return func; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
llvm::Function* Memory::createFunc(bool _isStore, llvm::Type* _valueType, GasMeter&) |
|
|
llvm::Function* Memory::createFunc(bool _isStore, llvm::Type* _valueType) |
|
|
{ |
|
|
{ |
|
|
auto isWord = _valueType == Type::Word; |
|
|
auto isWord = _valueType == Type::Word; |
|
|
|
|
|
|
|
|
llvm::Type* storeArgs[] = {Type::RuntimePtr, Type::Word, _valueType}; |
|
|
llvm::Type* storeArgs[] = {Array::getType()->getPointerTo(), Type::Word, _valueType}; |
|
|
llvm::Type* loadArgs[] = {Type::RuntimePtr, Type::Word}; |
|
|
llvm::Type* loadArgs[] = {Array::getType()->getPointerTo(), Type::Word}; |
|
|
auto name = _isStore ? isWord ? "mstore" : "mstore8" : "mload"; |
|
|
auto name = _isStore ? isWord ? "mstore" : "mstore8" : "mload"; |
|
|
auto funcType = _isStore ? llvm::FunctionType::get(Type::Void, storeArgs, false) : llvm::FunctionType::get(Type::Word, loadArgs, false); |
|
|
auto funcType = _isStore ? llvm::FunctionType::get(Type::Void, storeArgs, false) : llvm::FunctionType::get(Type::Word, loadArgs, false); |
|
|
auto func = llvm::Function::Create(funcType, llvm::Function::PrivateLinkage, name, getModule()); |
|
|
auto func = llvm::Function::Create(funcType, llvm::Function::PrivateLinkage, name, getModule()); |
|
|
@ -107,28 +106,25 @@ llvm::Function* Memory::createFunc(bool _isStore, llvm::Type* _valueType, GasMet |
|
|
InsertPointGuard guard(m_builder); // Restores insert point at function exit
|
|
|
InsertPointGuard guard(m_builder); // Restores insert point at function exit
|
|
|
|
|
|
|
|
|
m_builder.SetInsertPoint(llvm::BasicBlock::Create(func->getContext(), {}, func)); |
|
|
m_builder.SetInsertPoint(llvm::BasicBlock::Create(func->getContext(), {}, func)); |
|
|
auto rt = func->arg_begin(); |
|
|
auto mem = &func->getArgumentList().front(); |
|
|
rt->setName("rt"); |
|
|
mem->setName("mem"); |
|
|
auto index = rt->getNextNode(); |
|
|
auto index = mem->getNextNode(); |
|
|
index->setName("index"); |
|
|
index->setName("index"); |
|
|
|
|
|
|
|
|
auto valueSize = _valueType->getPrimitiveSizeInBits() / 8; |
|
|
|
|
|
this->require(index, Constant::get(valueSize)); |
|
|
|
|
|
auto ptr = getBytePtr(index); |
|
|
|
|
|
if (isWord) |
|
|
|
|
|
ptr = m_builder.CreateBitCast(ptr, Type::WordPtr, "wordPtr"); |
|
|
|
|
|
if (_isStore) |
|
|
if (_isStore) |
|
|
{ |
|
|
{ |
|
|
llvm::Value* value = index->getNextNode(); |
|
|
auto valueArg = index->getNextNode(); |
|
|
value->setName("value"); |
|
|
valueArg->setName("value"); |
|
|
if (isWord) |
|
|
auto value = isWord ? Endianness::toBE(m_builder, valueArg) : valueArg; |
|
|
value = Endianness::toBE(m_builder, value); |
|
|
auto memPtr = m_memory.getPtr(mem, m_builder.CreateTrunc(index, Type::Size)); |
|
|
m_builder.CreateStore(value, ptr); |
|
|
auto valuePtr = m_builder.CreateBitCast(memPtr, _valueType->getPointerTo(), "valuePtr"); |
|
|
|
|
|
m_builder.CreateStore(value, valuePtr); |
|
|
m_builder.CreateRetVoid(); |
|
|
m_builder.CreateRetVoid(); |
|
|
} |
|
|
} |
|
|
else |
|
|
else |
|
|
{ |
|
|
{ |
|
|
llvm::Value* ret = m_builder.CreateLoad(ptr); |
|
|
auto memPtr = m_memory.getPtr(mem, m_builder.CreateTrunc(index, Type::Size)); |
|
|
|
|
|
llvm::Value* ret = m_builder.CreateLoad(memPtr); |
|
|
ret = Endianness::toNative(m_builder, ret); |
|
|
ret = Endianness::toNative(m_builder, ret); |
|
|
m_builder.CreateRet(ret); |
|
|
m_builder.CreateRet(ret); |
|
|
} |
|
|
} |
|
|
@ -140,7 +136,7 @@ llvm::Function* Memory::getLoadWordFunc() |
|
|
{ |
|
|
{ |
|
|
auto& func = m_loadWord; |
|
|
auto& func = m_loadWord; |
|
|
if (!func) |
|
|
if (!func) |
|
|
func = createFunc(false, Type::Word, m_gasMeter); |
|
|
func = createFunc(false, Type::Word); |
|
|
return func; |
|
|
return func; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
@ -148,7 +144,7 @@ llvm::Function* Memory::getStoreWordFunc() |
|
|
{ |
|
|
{ |
|
|
auto& func = m_storeWord; |
|
|
auto& func = m_storeWord; |
|
|
if (!func) |
|
|
if (!func) |
|
|
func = createFunc(true, Type::Word, m_gasMeter); |
|
|
func = createFunc(true, Type::Word); |
|
|
return func; |
|
|
return func; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
@ -156,39 +152,41 @@ llvm::Function* Memory::getStoreByteFunc() |
|
|
{ |
|
|
{ |
|
|
auto& func = m_storeByte; |
|
|
auto& func = m_storeByte; |
|
|
if (!func) |
|
|
if (!func) |
|
|
func = createFunc(true, Type::Byte, m_gasMeter); |
|
|
func = createFunc(true, Type::Byte); |
|
|
return func; |
|
|
return func; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
llvm::Value* Memory::loadWord(llvm::Value* _addr) |
|
|
llvm::Value* Memory::loadWord(llvm::Value* _addr) |
|
|
{ |
|
|
{ |
|
|
return createCall(getLoadWordFunc(), {getRuntimeManager().getRuntimePtr(), _addr}); |
|
|
require(_addr, Constant::get(Type::Word->getPrimitiveSizeInBits() / 8)); |
|
|
|
|
|
return createCall(getLoadWordFunc(), {getRuntimeManager().getMem(), _addr}); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
void Memory::storeWord(llvm::Value* _addr, llvm::Value* _word) |
|
|
void Memory::storeWord(llvm::Value* _addr, llvm::Value* _word) |
|
|
{ |
|
|
{ |
|
|
createCall(getStoreWordFunc(), {getRuntimeManager().getRuntimePtr(), _addr, _word}); |
|
|
require(_addr, Constant::get(Type::Word->getPrimitiveSizeInBits() / 8)); |
|
|
|
|
|
createCall(getStoreWordFunc(), {getRuntimeManager().getMem(), _addr, _word}); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
void Memory::storeByte(llvm::Value* _addr, llvm::Value* _word) |
|
|
void Memory::storeByte(llvm::Value* _addr, llvm::Value* _word) |
|
|
{ |
|
|
{ |
|
|
|
|
|
require(_addr, Constant::get(Type::Byte->getPrimitiveSizeInBits() / 8)); |
|
|
auto byte = m_builder.CreateTrunc(_word, Type::Byte, "byte"); |
|
|
auto byte = m_builder.CreateTrunc(_word, Type::Byte, "byte"); |
|
|
createCall(getStoreByteFunc(), {getRuntimeManager().getRuntimePtr(), _addr, byte}); |
|
|
createCall(getStoreByteFunc(), {getRuntimeManager().getMem(), _addr, byte}); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
llvm::Value* Memory::getData() |
|
|
llvm::Value* Memory::getData() |
|
|
{ |
|
|
{ |
|
|
auto rtPtr = getRuntimeManager().getRuntimePtr(); |
|
|
auto memPtr = m_builder.CreateBitCast(getRuntimeManager().getMem(), Type::BytePtr->getPointerTo()); |
|
|
auto dataPtr = m_builder.CreateStructGEP(rtPtr, 3); |
|
|
auto data = m_builder.CreateLoad(memPtr, "data"); |
|
|
return m_builder.CreateLoad(dataPtr, "data"); |
|
|
assert(data->getType() == Type::BytePtr); |
|
|
|
|
|
return data; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
llvm::Value* Memory::getSize() |
|
|
llvm::Value* Memory::getSize() |
|
|
{ |
|
|
{ |
|
|
auto rtPtr = getRuntimeManager().getRuntimePtr(); |
|
|
return m_builder.CreateZExt(m_memory.size(), Type::Word, "msize"); // TODO: Allow placing i64 on stack
|
|
|
auto sizePtr = m_builder.CreateStructGEP(rtPtr, 4); |
|
|
|
|
|
return m_builder.CreateLoad(sizePtr, "size"); |
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
llvm::Value* Memory::getBytePtr(llvm::Value* _index) |
|
|
llvm::Value* Memory::getBytePtr(llvm::Value* _index) |
|
|
@ -204,7 +202,7 @@ void Memory::require(llvm::Value* _offset, llvm::Value* _size) |
|
|
if (!constant->getValue()) |
|
|
if (!constant->getValue()) |
|
|
return; |
|
|
return; |
|
|
} |
|
|
} |
|
|
createCall(getRequireFunc(), {getRuntimeManager().getRuntimePtr(), _offset, _size}); |
|
|
createCall(getRequireFunc(), {getRuntimeManager().getMem(), _offset, _size, getRuntimeManager().getJmpBuf(), getRuntimeManager().getGasPtr()}); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
void Memory::copyBytes(llvm::Value* _srcPtr, llvm::Value* _srcSize, llvm::Value* _srcIdx, |
|
|
void Memory::copyBytes(llvm::Value* _srcPtr, llvm::Value* _srcSize, llvm::Value* _srcIdx, |
|
|
@ -233,28 +231,18 @@ void Memory::copyBytes(llvm::Value* _srcPtr, llvm::Value* _srcSize, llvm::Value* |
|
|
auto dataLeftSize = m_builder.CreateNUWSub(size64, idx64); |
|
|
auto dataLeftSize = m_builder.CreateNUWSub(size64, idx64); |
|
|
auto outOfBound = m_builder.CreateICmpUGT(reqBytes, dataLeftSize); |
|
|
auto outOfBound = m_builder.CreateICmpUGT(reqBytes, dataLeftSize); |
|
|
auto bytesToCopyInner = m_builder.CreateSelect(outOfBound, dataLeftSize, reqBytes); |
|
|
auto bytesToCopyInner = m_builder.CreateSelect(outOfBound, dataLeftSize, reqBytes); |
|
|
auto bytesToCopy = m_builder.CreateSelect(isOutsideData, m_builder.getInt64(0), bytesToCopyInner); |
|
|
auto bytesToCopy = m_builder.CreateSelect(isOutsideData, m_builder.getInt64(0), bytesToCopyInner, "bytesToCopy"); |
|
|
|
|
|
auto bytesToZero = m_builder.CreateNUWSub(reqBytes, bytesToCopy, "bytesToZero"); |
|
|
|
|
|
|
|
|
auto src = m_builder.CreateGEP(_srcPtr, idx64, "src"); |
|
|
auto src = m_builder.CreateGEP(_srcPtr, idx64, "src"); |
|
|
auto dstIdx = m_builder.CreateTrunc(_destMemIdx, Type::Size, "dstIdx"); // Never allow memory index be a type bigger than i64
|
|
|
auto dstIdx = m_builder.CreateTrunc(_destMemIdx, Type::Size, "dstIdx"); // Never allow memory index be a type bigger than i64
|
|
|
auto dst = m_builder.CreateGEP(getData(), dstIdx, "dst"); |
|
|
auto padIdx = m_builder.CreateNUWAdd(dstIdx, bytesToCopy, "padIdx"); |
|
|
|
|
|
auto dst = m_memory.getPtr(getRuntimeManager().getMem(), dstIdx); |
|
|
|
|
|
auto pad = m_memory.getPtr(getRuntimeManager().getMem(), padIdx); |
|
|
m_builder.CreateMemCpy(dst, src, bytesToCopy, 0); |
|
|
m_builder.CreateMemCpy(dst, src, bytesToCopy, 0); |
|
|
|
|
|
m_builder.CreateMemSet(pad, m_builder.getInt8(0), bytesToZero, 0); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
extern "C" |
|
|
|
|
|
{ |
|
|
|
|
|
using namespace dev::eth::jit; |
|
|
|
|
|
|
|
|
|
|
|
EXPORT byte* mem_resize(Runtime* _rt, i256* _size) // TODO: Use uint64 as size OR use realloc in LLVM IR
|
|
|
|
|
|
{ |
|
|
|
|
|
auto size = _size->a; // Trunc to 64-bit
|
|
|
|
|
|
auto& memory = _rt->getMemory(); |
|
|
|
|
|
memory.resize(size); |
|
|
|
|
|
return memory.data(); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
Gav Wood
10 years ago