You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
406 lines
15 KiB
406 lines
15 KiB
#include "Arith256.h"
|
|
|
|
#include <iostream>
|
|
#include <iomanip>
|
|
|
|
#include "preprocessor/llvm_includes_start.h"
|
|
#include <llvm/IR/IntrinsicInst.h>
|
|
#include "preprocessor/llvm_includes_end.h"
|
|
|
|
#include "Type.h"
|
|
#include "Endianness.h"
|
|
|
|
namespace dev
|
|
{
|
|
namespace eth
|
|
{
|
|
namespace jit
|
|
{
|
|
|
|
Arith256::Arith256(llvm::IRBuilder<>& _builder) :
|
|
CompilerHelper(_builder)
|
|
{}
|
|
|
|
void Arith256::debug(llvm::Value* _value, char _c)
|
|
{
|
|
if (!m_debug)
|
|
{
|
|
llvm::Type* argTypes[] = {Type::Word, m_builder.getInt8Ty()};
|
|
m_debug = llvm::Function::Create(llvm::FunctionType::get(Type::Void, argTypes, false), llvm::Function::ExternalLinkage, "debug", getModule());
|
|
}
|
|
createCall(m_debug, {m_builder.CreateZExtOrTrunc(_value, Type::Word), m_builder.getInt8(_c)});
|
|
}
|
|
|
|
llvm::Function* Arith256::getMulFunc()
|
|
{
|
|
auto& func = m_mul;
|
|
if (!func)
|
|
{
|
|
llvm::Type* argTypes[] = {Type::Word, Type::Word};
|
|
func = llvm::Function::Create(llvm::FunctionType::get(Type::Word, argTypes, false), llvm::Function::PrivateLinkage, "mul", getModule());
|
|
|
|
auto x = &func->getArgumentList().front();
|
|
x->setName("x");
|
|
auto y = x->getNextNode();
|
|
y->setName("y");
|
|
|
|
InsertPointGuard guard{m_builder};
|
|
auto bb = llvm::BasicBlock::Create(m_builder.getContext(), {}, func);
|
|
m_builder.SetInsertPoint(bb);
|
|
auto i64 = Type::Size;
|
|
auto i128 = m_builder.getIntNTy(128);
|
|
auto i256 = Type::Word;
|
|
auto x_lo = m_builder.CreateTrunc(x, i64, "x.lo");
|
|
auto y_lo = m_builder.CreateTrunc(y, i64, "y.lo");
|
|
auto x_mi = m_builder.CreateTrunc(m_builder.CreateLShr(x, Constant::get(64)), i64);
|
|
auto y_mi = m_builder.CreateTrunc(m_builder.CreateLShr(y, Constant::get(64)), i64);
|
|
auto x_hi = m_builder.CreateTrunc(m_builder.CreateLShr(x, Constant::get(128)), i128);
|
|
auto y_hi = m_builder.CreateTrunc(m_builder.CreateLShr(y, Constant::get(128)), i128);
|
|
|
|
auto t1 = m_builder.CreateMul(m_builder.CreateZExt(x_lo, i128), m_builder.CreateZExt(y_lo, i128));
|
|
auto t2 = m_builder.CreateMul(m_builder.CreateZExt(x_lo, i128), m_builder.CreateZExt(y_mi, i128));
|
|
auto t3 = m_builder.CreateMul(m_builder.CreateZExt(x_lo, i128), y_hi);
|
|
auto t4 = m_builder.CreateMul(m_builder.CreateZExt(x_mi, i128), m_builder.CreateZExt(y_lo, i128));
|
|
auto t5 = m_builder.CreateMul(m_builder.CreateZExt(x_mi, i128), m_builder.CreateZExt(y_mi, i128));
|
|
auto t6 = m_builder.CreateMul(m_builder.CreateZExt(x_mi, i128), y_hi);
|
|
auto t7 = m_builder.CreateMul(x_hi, m_builder.CreateZExt(y_lo, i128));
|
|
auto t8 = m_builder.CreateMul(x_hi, m_builder.CreateZExt(y_mi, i128));
|
|
|
|
auto p = m_builder.CreateZExt(t1, i256);
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t2, i256), Constant::get(64)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t3, i256), Constant::get(128)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t4, i256), Constant::get(64)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t5, i256), Constant::get(128)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t6, i256), Constant::get(192)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t7, i256), Constant::get(128)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t8, i256), Constant::get(192)));
|
|
m_builder.CreateRet(p);
|
|
}
|
|
return func;
|
|
}
|
|
|
|
llvm::Function* Arith256::getMul512Func()
|
|
{
|
|
auto& func = m_mul512;
|
|
if (!func)
|
|
{
|
|
auto i512 = m_builder.getIntNTy(512);
|
|
llvm::Type* argTypes[] = {Type::Word, Type::Word};
|
|
func = llvm::Function::Create(llvm::FunctionType::get(i512, argTypes, false), llvm::Function::PrivateLinkage, "mul512", getModule());
|
|
|
|
auto x = &func->getArgumentList().front();
|
|
x->setName("x");
|
|
auto y = x->getNextNode();
|
|
y->setName("y");
|
|
|
|
InsertPointGuard guard{m_builder};
|
|
auto bb = llvm::BasicBlock::Create(m_builder.getContext(), {}, func);
|
|
m_builder.SetInsertPoint(bb);
|
|
auto i128 = m_builder.getIntNTy(128);
|
|
auto i256 = Type::Word;
|
|
auto x_lo = m_builder.CreateZExt(m_builder.CreateTrunc(x, i128, "x.lo"), i256);
|
|
auto y_lo = m_builder.CreateZExt(m_builder.CreateTrunc(y, i128, "y.lo"), i256);
|
|
auto x_hi = m_builder.CreateZExt(m_builder.CreateTrunc(m_builder.CreateLShr(x, Constant::get(128)), i128, "x.hi"), i256);
|
|
auto y_hi = m_builder.CreateZExt(m_builder.CreateTrunc(m_builder.CreateLShr(y, Constant::get(128)), i128, "y.hi"), i256);
|
|
|
|
auto t1 = createCall(getMulFunc(), {x_lo, y_lo});
|
|
auto t2 = createCall(getMulFunc(), {x_lo, y_hi});
|
|
auto t3 = createCall(getMulFunc(), {x_hi, y_lo});
|
|
auto t4 = createCall(getMulFunc(), {x_hi, y_hi});
|
|
|
|
auto p = m_builder.CreateZExt(t1, i512);
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t2, i512), m_builder.getIntN(512, 128)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t3, i512), m_builder.getIntN(512, 128)));
|
|
p = m_builder.CreateAdd(p, m_builder.CreateShl(m_builder.CreateZExt(t4, i512), m_builder.getIntN(512, 256)));
|
|
m_builder.CreateRet(p);
|
|
}
|
|
return func;
|
|
}
|
|
|
|
llvm::Function* Arith256::getDivFunc(llvm::Type* _type)
|
|
{
|
|
auto& func = _type == Type::Word ? m_div : m_div512;
|
|
|
|
if (!func)
|
|
{
|
|
// Based of "Improved shift divisor algorithm" from "Software Integer Division" by Microsoft Research
|
|
// The following algorithm also handles divisor of value 0 returning 0 for both quotient and reminder
|
|
|
|
llvm::Type* argTypes[] = {_type, _type};
|
|
auto retType = llvm::StructType::get(m_builder.getContext(), llvm::ArrayRef<llvm::Type*>{argTypes});
|
|
auto funcName = _type == Type::Word ? "div" : "div512";
|
|
func = llvm::Function::Create(llvm::FunctionType::get(retType, argTypes, false), llvm::Function::PrivateLinkage, funcName, getModule());
|
|
|
|
auto zero = llvm::ConstantInt::get(_type, 0);
|
|
auto one = llvm::ConstantInt::get(_type, 1);
|
|
|
|
auto x = &func->getArgumentList().front();
|
|
x->setName("x");
|
|
auto yArg = x->getNextNode();
|
|
yArg->setName("y");
|
|
|
|
InsertPointGuard guard{m_builder};
|
|
|
|
auto entryBB = llvm::BasicBlock::Create(m_builder.getContext(), "Entry", func);
|
|
auto mainBB = llvm::BasicBlock::Create(m_builder.getContext(), "Main", func);
|
|
auto loopBB = llvm::BasicBlock::Create(m_builder.getContext(), "Loop", func);
|
|
auto continueBB = llvm::BasicBlock::Create(m_builder.getContext(), "Continue", func);
|
|
auto returnBB = llvm::BasicBlock::Create(m_builder.getContext(), "Return", func);
|
|
|
|
m_builder.SetInsertPoint(entryBB);
|
|
auto yNonZero = m_builder.CreateICmpNE(yArg, zero);
|
|
auto yLEx = m_builder.CreateICmpULE(yArg, x);
|
|
auto r0 = m_builder.CreateSelect(yNonZero, x, zero, "r0");
|
|
m_builder.CreateCondBr(m_builder.CreateAnd(yLEx, yNonZero), mainBB, returnBB);
|
|
|
|
m_builder.SetInsertPoint(mainBB);
|
|
auto ctlzIntr = llvm::Intrinsic::getDeclaration(getModule(), llvm::Intrinsic::ctlz, _type);
|
|
// both y and r are non-zero
|
|
auto yLz = m_builder.CreateCall2(ctlzIntr, yArg, m_builder.getInt1(true), "y.lz");
|
|
auto rLz = m_builder.CreateCall2(ctlzIntr, r0, m_builder.getInt1(true), "r.lz");
|
|
auto i0 = m_builder.CreateNUWSub(yLz, rLz, "i0");
|
|
auto shlBy0 = m_builder.CreateICmpEQ(i0, zero);
|
|
auto y0 = m_builder.CreateShl(yArg, i0);
|
|
if (_type == m_builder.getIntNTy(512)) // Workaround for shl bug for long shifts
|
|
{
|
|
const auto treshold = m_builder.getIntN(512, 128);
|
|
auto highShift = m_builder.CreateICmpUGT(i0, treshold);
|
|
auto s = m_builder.CreateNUWSub(i0, treshold);
|
|
auto yhs = m_builder.CreateShl(yArg, treshold);
|
|
yhs = m_builder.CreateShl(yhs, s);
|
|
y0 = m_builder.CreateSelect(highShift, yhs, y0);
|
|
}
|
|
y0 = m_builder.CreateSelect(shlBy0, yArg, y0, "y0"); // Workaround for LLVM bug: shl by 0 produces wrong result
|
|
m_builder.CreateBr(loopBB);
|
|
|
|
m_builder.SetInsertPoint(loopBB);
|
|
auto yPhi = m_builder.CreatePHI(_type, 2, "y.phi");
|
|
auto rPhi = m_builder.CreatePHI(_type, 2, "r.phi");
|
|
auto iPhi = m_builder.CreatePHI(_type, 2, "i.phi");
|
|
auto qPhi = m_builder.CreatePHI(_type, 2, "q.phi");
|
|
auto rUpdate = m_builder.CreateNUWSub(rPhi, yPhi);
|
|
auto qUpdate = m_builder.CreateOr(qPhi, one); // q += 1, q lowest bit is 0
|
|
auto rGEy = m_builder.CreateICmpUGE(rPhi, yPhi);
|
|
auto r1 = m_builder.CreateSelect(rGEy, rUpdate, rPhi, "r1");
|
|
auto q1 = m_builder.CreateSelect(rGEy, qUpdate, qPhi, "q");
|
|
auto iZero = m_builder.CreateICmpEQ(iPhi, zero);
|
|
m_builder.CreateCondBr(iZero, returnBB, continueBB);
|
|
|
|
m_builder.SetInsertPoint(continueBB);
|
|
auto i2 = m_builder.CreateNUWSub(iPhi, one);
|
|
auto q2 = m_builder.CreateShl(q1, one);
|
|
auto y2 = m_builder.CreateLShr(yPhi, one);
|
|
m_builder.CreateBr(loopBB);
|
|
|
|
yPhi->addIncoming(y0, mainBB);
|
|
yPhi->addIncoming(y2, continueBB);
|
|
rPhi->addIncoming(r0, mainBB);
|
|
rPhi->addIncoming(r1, continueBB);
|
|
iPhi->addIncoming(i0, mainBB);
|
|
iPhi->addIncoming(i2, continueBB);
|
|
qPhi->addIncoming(zero, mainBB);
|
|
qPhi->addIncoming(q2, continueBB);
|
|
|
|
m_builder.SetInsertPoint(returnBB);
|
|
auto qRet = m_builder.CreatePHI(_type, 2, "q.ret");
|
|
qRet->addIncoming(zero, entryBB);
|
|
qRet->addIncoming(q1, loopBB);
|
|
auto rRet = m_builder.CreatePHI(_type, 2, "r.ret");
|
|
rRet->addIncoming(r0, entryBB);
|
|
rRet->addIncoming(r1, loopBB);
|
|
auto ret = m_builder.CreateInsertValue(llvm::UndefValue::get(retType), qRet, 0, "ret0");
|
|
ret = m_builder.CreateInsertValue(ret, rRet, 1, "ret");
|
|
m_builder.CreateRet(ret);
|
|
}
|
|
return func;
|
|
}
|
|
|
|
llvm::Function* Arith256::getExpFunc()
|
|
{
|
|
if (!m_exp)
|
|
{
|
|
llvm::Type* argTypes[] = {Type::Word, Type::Word};
|
|
m_exp = llvm::Function::Create(llvm::FunctionType::get(Type::Word, argTypes, false), llvm::Function::PrivateLinkage, "exp", getModule());
|
|
|
|
auto base = &m_exp->getArgumentList().front();
|
|
base->setName("base");
|
|
auto exponent = base->getNextNode();
|
|
exponent->setName("exponent");
|
|
|
|
InsertPointGuard guard{m_builder};
|
|
|
|
// while (e != 0) {
|
|
// if (e % 2 == 1)
|
|
// r *= b;
|
|
// b *= b;
|
|
// e /= 2;
|
|
// }
|
|
|
|
auto entryBB = llvm::BasicBlock::Create(m_builder.getContext(), "Entry", m_exp);
|
|
auto headerBB = llvm::BasicBlock::Create(m_builder.getContext(), "LoopHeader", m_exp);
|
|
auto bodyBB = llvm::BasicBlock::Create(m_builder.getContext(), "LoopBody", m_exp);
|
|
auto updateBB = llvm::BasicBlock::Create(m_builder.getContext(), "ResultUpdate", m_exp);
|
|
auto continueBB = llvm::BasicBlock::Create(m_builder.getContext(), "Continue", m_exp);
|
|
auto returnBB = llvm::BasicBlock::Create(m_builder.getContext(), "Return", m_exp);
|
|
|
|
m_builder.SetInsertPoint(entryBB);
|
|
m_builder.CreateBr(headerBB);
|
|
|
|
m_builder.SetInsertPoint(headerBB);
|
|
auto r = m_builder.CreatePHI(Type::Word, 2, "r");
|
|
auto b = m_builder.CreatePHI(Type::Word, 2, "b");
|
|
auto e = m_builder.CreatePHI(Type::Word, 2, "e");
|
|
auto eNonZero = m_builder.CreateICmpNE(e, Constant::get(0), "e.nonzero");
|
|
m_builder.CreateCondBr(eNonZero, bodyBB, returnBB);
|
|
|
|
m_builder.SetInsertPoint(bodyBB);
|
|
auto eOdd = m_builder.CreateICmpNE(m_builder.CreateAnd(e, Constant::get(1)), Constant::get(0), "e.isodd");
|
|
m_builder.CreateCondBr(eOdd, updateBB, continueBB);
|
|
|
|
m_builder.SetInsertPoint(updateBB);
|
|
auto r0 = createCall(getMulFunc(), {r, b});
|
|
m_builder.CreateBr(continueBB);
|
|
|
|
m_builder.SetInsertPoint(continueBB);
|
|
auto r1 = m_builder.CreatePHI(Type::Word, 2, "r1");
|
|
r1->addIncoming(r, bodyBB);
|
|
r1->addIncoming(r0, updateBB);
|
|
auto b1 = createCall(getMulFunc(), {b, b});
|
|
auto e1 = m_builder.CreateLShr(e, Constant::get(1), "e1");
|
|
m_builder.CreateBr(headerBB);
|
|
|
|
r->addIncoming(Constant::get(1), entryBB);
|
|
r->addIncoming(r1, continueBB);
|
|
b->addIncoming(base, entryBB);
|
|
b->addIncoming(b1, continueBB);
|
|
e->addIncoming(exponent, entryBB);
|
|
e->addIncoming(e1, continueBB);
|
|
|
|
m_builder.SetInsertPoint(returnBB);
|
|
m_builder.CreateRet(r);
|
|
}
|
|
return m_exp;
|
|
}
|
|
|
|
llvm::Function* Arith256::getAddModFunc()
|
|
{
|
|
if (!m_addmod)
|
|
{
|
|
auto i512Ty = m_builder.getIntNTy(512);
|
|
llvm::Type* argTypes[] = {Type::Word, Type::Word, Type::Word};
|
|
m_addmod = llvm::Function::Create(llvm::FunctionType::get(Type::Word, argTypes, false), llvm::Function::PrivateLinkage, "addmod", getModule());
|
|
|
|
auto x = &m_addmod->getArgumentList().front();
|
|
x->setName("x");
|
|
auto y = x->getNextNode();
|
|
y->setName("y");
|
|
auto mod = y->getNextNode();
|
|
mod->setName("m");
|
|
|
|
InsertPointGuard guard{m_builder};
|
|
|
|
auto entryBB = llvm::BasicBlock::Create(m_builder.getContext(), {}, m_addmod);
|
|
m_builder.SetInsertPoint(entryBB);
|
|
auto x512 = m_builder.CreateZExt(x, i512Ty, "x512");
|
|
auto y512 = m_builder.CreateZExt(y, i512Ty, "y512");
|
|
auto m512 = m_builder.CreateZExt(mod, i512Ty, "m512");
|
|
auto s = m_builder.CreateAdd(x512, y512, "s");
|
|
auto d = createCall(getDivFunc(i512Ty), {s, m512});
|
|
auto r = m_builder.CreateExtractValue(d, 1, "r");
|
|
m_builder.CreateRet(m_builder.CreateTrunc(r, Type::Word));
|
|
}
|
|
return m_addmod;
|
|
}
|
|
|
|
llvm::Function* Arith256::getMulModFunc()
|
|
{
|
|
if (!m_mulmod)
|
|
{
|
|
llvm::Type* argTypes[] = {Type::Word, Type::Word, Type::Word};
|
|
m_mulmod = llvm::Function::Create(llvm::FunctionType::get(Type::Word, argTypes, false), llvm::Function::PrivateLinkage, "mulmod", getModule());
|
|
|
|
auto i512Ty = m_builder.getIntNTy(512);
|
|
auto x = &m_mulmod->getArgumentList().front();
|
|
x->setName("x");
|
|
auto y = x->getNextNode();
|
|
y->setName("y");
|
|
auto mod = y->getNextNode();
|
|
mod->setName("mod");
|
|
|
|
InsertPointGuard guard{m_builder};
|
|
|
|
auto entryBB = llvm::BasicBlock::Create(m_builder.getContext(), {}, m_mulmod);
|
|
m_builder.SetInsertPoint(entryBB);
|
|
auto p = createCall(getMul512Func(), {x, y});
|
|
auto m = m_builder.CreateZExt(mod, i512Ty, "m");
|
|
auto d = createCall(getDivFunc(i512Ty), {p, m});
|
|
auto r = m_builder.CreateExtractValue(d, 1, "r");
|
|
r = m_builder.CreateTrunc(r, Type::Word);
|
|
m_builder.CreateRet(r);
|
|
}
|
|
return m_mulmod;
|
|
}
|
|
|
|
llvm::Value* Arith256::mul(llvm::Value* _arg1, llvm::Value* _arg2)
|
|
{
|
|
return createCall(getMulFunc(), {_arg1, _arg2});
|
|
}
|
|
|
|
std::pair<llvm::Value*, llvm::Value*> Arith256::div(llvm::Value* _arg1, llvm::Value* _arg2)
|
|
{
|
|
auto div = m_builder.CreateExtractValue(createCall(getDivFunc(Type::Word), {_arg1, _arg2}), 0, "div");
|
|
auto mod = m_builder.CreateExtractValue(createCall(getDivFunc(Type::Word), {_arg1, _arg2}), 1, "mod");
|
|
return std::make_pair(div, mod);
|
|
}
|
|
|
|
std::pair<llvm::Value*, llvm::Value*> Arith256::sdiv(llvm::Value* _x, llvm::Value* _y)
|
|
{
|
|
auto xIsNeg = m_builder.CreateICmpSLT(_x, Constant::get(0));
|
|
auto xNeg = m_builder.CreateSub(Constant::get(0), _x);
|
|
auto xAbs = m_builder.CreateSelect(xIsNeg, xNeg, _x);
|
|
|
|
auto yIsNeg = m_builder.CreateICmpSLT(_y, Constant::get(0));
|
|
auto yNeg = m_builder.CreateSub(Constant::get(0), _y);
|
|
auto yAbs = m_builder.CreateSelect(yIsNeg, yNeg, _y);
|
|
|
|
auto res = div(xAbs, yAbs);
|
|
|
|
// the reminder has the same sign as dividend
|
|
auto rNeg = m_builder.CreateSub(Constant::get(0), res.second);
|
|
res.second = m_builder.CreateSelect(xIsNeg, rNeg, res.second);
|
|
|
|
auto qNeg = m_builder.CreateSub(Constant::get(0), res.first);
|
|
auto xyOpposite = m_builder.CreateXor(xIsNeg, yIsNeg);
|
|
res.first = m_builder.CreateSelect(xyOpposite, qNeg, res.first);
|
|
|
|
return res;
|
|
}
|
|
|
|
llvm::Value* Arith256::exp(llvm::Value* _arg1, llvm::Value* _arg2)
|
|
{
|
|
return createCall(getExpFunc(), {_arg1, _arg2});
|
|
}
|
|
|
|
llvm::Value* Arith256::addmod(llvm::Value* _arg1, llvm::Value* _arg2, llvm::Value* _arg3)
|
|
{
|
|
return createCall(getAddModFunc(), {_arg1, _arg2, _arg3});
|
|
}
|
|
|
|
llvm::Value* Arith256::mulmod(llvm::Value* _arg1, llvm::Value* _arg2, llvm::Value* _arg3)
|
|
{
|
|
return createCall(getMulModFunc(), {_arg1, _arg2, _arg3});
|
|
}
|
|
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
extern "C"
|
|
{
|
|
EXPORT void debug(uint64_t a, uint64_t b, uint64_t c, uint64_t d, char z)
|
|
{
|
|
std::cerr << "DEBUG " << std::dec << z << ": " //<< d << c << b << a
|
|
<< " [" << std::hex << std::setfill('0') << std::setw(16) << d << std::setw(16) << c << std::setw(16) << b << std::setw(16) << a << "]\n";
|
|
}
|
|
}
|
|
|