RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-21 18:22:53 +01:00
Code Issues Projects Releases Wiki Activity

[VP] Implementation of intrinsic and SDNode definitions for VP load, store, gather, scatter.

Browse Source 
This patch adds intrinsic definitions and SDNodes for predicated
load/store/gather/scatter, based on the work done in D57504.

Reviewed By: simoll, craig.topper

Differential Revision: https://reviews.llvm.org/D99355
This commit is contained in:
Hussain Kadhem 2021-07-01 11:30:49 +02:00 committed by Simon Moll
parent e07552eb3f
commit 7eddb43fa0
6 changed files with 198 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
include/llvm/CodeGen/SelectionDAGNodes.h
View File

@ -1367,33 +1367,36 @@ public:
static bool classof(const SDNode *N) {
// For some targets, we lower some target intrinsics to a MemIntrinsicNode
// with either an intrinsic or a target opcode.
return N->getOpcode() == ISD::LOAD ||
N->getOpcode() == ISD::STORE ||
N->getOpcode() == ISD::PREFETCH ||
N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
N->getOpcode() == ISD::ATOMIC_SWAP ||
N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
N->getOpcode() == ISD::ATOMIC_LOAD ||
N->getOpcode() == ISD::ATOMIC_STORE ||
N->getOpcode() == ISD::MLOAD ||
N->getOpcode() == ISD::MSTORE ||
N->getOpcode() == ISD::MGATHER ||
N->getOpcode() == ISD::MSCATTER ||
N->isMemIntrinsic() ||
N->isTargetMemoryOpcode();
switch (N->getOpcode()) {
case ISD::LOAD:
case ISD::STORE:
case ISD::PREFETCH:
case ISD::ATOMIC_CMP_SWAP:
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
case ISD::ATOMIC_SWAP:
case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
case ISD::ATOMIC_LOAD_MIN:
case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX:
case ISD::ATOMIC_LOAD_FADD:
case ISD::ATOMIC_LOAD_FSUB:
case ISD::ATOMIC_LOAD:
case ISD::ATOMIC_STORE:
case ISD::MLOAD:
case ISD::MSTORE:
case ISD::MGATHER:
case ISD::MSCATTER:
return true;
default:
return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
}
}
};

21
include/llvm/IR/IntrinsicInst.h
View File

@ -403,21 +403,34 @@ public:
// Whether \p ID is a VP intrinsic ID.
static bool isVPIntrinsic(Intrinsic::ID);
/// \return the mask parameter or nullptr.
/// \return The mask parameter or nullptr.
Value *getMaskParam() const;
void setMaskParam(Value *);
/// \return the vector length parameter or nullptr.
/// \return The vector length parameter or nullptr.
Value *getVectorLengthParam() const;
void setVectorLengthParam(Value *);
/// \return whether the vector length param can be ignored.
/// \return Whether the vector length param can be ignored.
bool canIgnoreVectorLengthParam() const;
/// \return the static element count (vector number of elements) the vector
/// \return The static element count (vector number of elements) the vector
/// length parameter applies to.
ElementCount getStaticVectorLength() const;
/// \return The alignment of the pointer used by this load/store/gather or
/// scatter.
MaybeAlign getPointerAlignment() const;
// MaybeAlign setPointerAlignment(Align NewAlign); // TODO
/// \return The pointer operand of this load,store, gather or scatter.
Value *getMemoryPointerParam() const;
static Optional<unsigned> getMemoryPointerParamPos(Intrinsic::ID);
/// \return The data (payload) operand of this store or scatter.
Value *getMemoryDataParam() const;
static Optional<unsigned> getMemoryDataParamPos(Intrinsic::ID);
// Methods for support type inquiry through isa, cast, and dyn_cast:
static bool classof(const IntrinsicInst *I) {
return isVPIntrinsic(I->getIntrinsicID());

26
include/llvm/IR/Intrinsics.td
View File

@ -1368,6 +1368,32 @@ def int_experimental_stepvector : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
[], [IntrNoMem]>;
//===---------------- Vector Predication Intrinsics --------------===//
// Memory Intrinsics
def int_vp_store : DefaultAttrsIntrinsic<[],
[ llvm_anyvector_ty,
LLVMAnyPointerType<LLVMMatchType<0>>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_i32_ty],
[ NoCapture<ArgIndex<1>>, IntrNoSync, IntrWriteMem, IntrArgMemOnly, IntrWillReturn ]>;
def int_vp_load : DefaultAttrsIntrinsic<[ llvm_anyvector_ty],
[ LLVMAnyPointerType<LLVMMatchType<0>>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_i32_ty],
[ NoCapture<ArgIndex<0>>, IntrNoSync, IntrReadMem, IntrWillReturn, IntrArgMemOnly ]>;
def int_vp_gather: DefaultAttrsIntrinsic<[ llvm_anyvector_ty],
[ LLVMVectorOfAnyPointersToElt<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_i32_ty],
[ IntrReadMem, IntrNoSync, IntrWillReturn, IntrArgMemOnly ]>;
def int_vp_scatter: DefaultAttrsIntrinsic<[],
[ llvm_anyvector_ty,
LLVMVectorOfAnyPointersToElt<0>,
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
llvm_i32_ty],
[ IntrArgMemOnly, IntrNoSync, IntrWillReturn ]>; // TODO allow IntrNoCapture for vectors of pointers
// Speculatable Binary operators
let IntrProperties = [IntrSpeculatable, IntrNoMem, IntrNoSync, IntrWillReturn] in {

43
include/llvm/IR/VPIntrinsics.def
View File

@ -100,6 +100,17 @@ END_REGISTER_VP_SDNODE(SDOPC)
#define HANDLE_VP_TO_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID)
#endif
// Map this VP intrinsic to its canonical functional intrinsic.
#ifndef HANDLE_VP_TO_INTRIN
#define HANDLE_VP_TO_INTRIN(ID)
#endif
// This VP Intrinsic is a memory operation
// The pointer arg is at POINTERPOS and the data arg is at DATAPOS.
#ifndef HANDLE_VP_IS_MEMOP
#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS)
#endif
/// } Property Macros
///// Integer Arithmetic {
@ -191,6 +202,36 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem)
///// } Floating-Point Arithmetic
///// Memory Operations {
// llvm.vp.store(ptr,val,mask,vlen)
BEGIN_REGISTER_VP(vp_store, 2, 3, VP_STORE, 0)
HANDLE_VP_TO_OPC(Store)
HANDLE_VP_TO_INTRIN(masked_store)
HANDLE_VP_IS_MEMOP(vp_store, 1, 0)
END_REGISTER_VP(vp_store, VP_STORE)
// llvm.vp.scatter(ptr,val,mask,vlen)
BEGIN_REGISTER_VP(vp_scatter, 2, 3, VP_SCATTER, 0)
HANDLE_VP_TO_INTRIN(masked_scatter)
HANDLE_VP_IS_MEMOP(vp_scatter, 1, 0)
END_REGISTER_VP(vp_scatter, VP_SCATTER)
// llvm.vp.load(ptr,mask,vlen)
BEGIN_REGISTER_VP(vp_load, 1, 2, VP_LOAD, -1)
HANDLE_VP_TO_OPC(Load)
HANDLE_VP_TO_INTRIN(masked_load)
HANDLE_VP_IS_MEMOP(vp_load, 0, None)
END_REGISTER_VP(vp_load, VP_LOAD)
// llvm.vp.gather(ptr,mask,vlen)
BEGIN_REGISTER_VP(vp_gather, 1, 2, VP_GATHER, -1)
HANDLE_VP_TO_INTRIN(masked_gather)
HANDLE_VP_IS_MEMOP(vp_gather, 0, None)
END_REGISTER_VP(vp_gather, VP_GATHER)
///// } Memory Operations
#undef BEGIN_REGISTER_VP
#undef BEGIN_REGISTER_VP_INTRINSIC
#undef BEGIN_REGISTER_VP_SDNODE
@ -199,3 +240,5 @@ HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem)
#undef END_REGISTER_VP_SDNODE
#undef HANDLE_VP_TO_OPC
#undef HANDLE_VP_TO_CONSTRAINEDFP
#undef HANDLE_VP_TO_INTRIN
#undef HANDLE_VP_IS_MEMOP

82
lib/IR/IntrinsicInst.cpp
View File

@ -1,4 +1,4 @@
//===-- InstrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
//===-- IntrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@ -340,6 +340,53 @@ VPIntrinsic::getVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
}
}
/// \return the alignment of the pointer used by this load/store/gather or
/// scatter.
MaybeAlign VPIntrinsic::getPointerAlignment() const {
Optional<unsigned> PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID());
assert(PtrParamOpt.hasValue() && "no pointer argument!");
return getParamAlign(PtrParamOpt.getValue());
}
/// \return The pointer operand of this load,store, gather or scatter.
Value *VPIntrinsic::getMemoryPointerParam() const {
if (auto PtrParamOpt = getMemoryPointerParamPos(getIntrinsicID()))
return getArgOperand(PtrParamOpt.getValue());
return nullptr;
}
Optional<unsigned> VPIntrinsic::getMemoryPointerParamPos(Intrinsic::ID VPID) {
switch (VPID) {
default:
return None;
#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \
case Intrinsic::VPID: \
return POINTERPOS;
#include "llvm/IR/VPIntrinsics.def"
}
}
/// \return The data (payload) operand of this store or scatter.
Value *VPIntrinsic::getMemoryDataParam() const {
auto DataParamOpt = getMemoryDataParamPos(getIntrinsicID());
if (!DataParamOpt.hasValue())
return nullptr;
return getArgOperand(DataParamOpt.getValue());
}
Optional<unsigned> VPIntrinsic::getMemoryDataParamPos(Intrinsic::ID VPID) {
switch (VPID) {
default:
return None;
#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS) \
case Intrinsic::VPID: \
return DATAPOS;
#include "llvm/IR/VPIntrinsics.def"
}
}
bool VPIntrinsic::isVPIntrinsic(Intrinsic::ID ID) {
switch (ID) {
default:
@ -424,10 +471,35 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const {
Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
ArrayRef<Value *> Params) {
assert(isVPIntrinsic(VPID) && "not a VP intrinsic");
// TODO: Extend this for other VP intrinsics as they are upstreamed. This
// works for binary arithmetic VP intrinsics.
auto *VPFunc = Intrinsic::getDeclaration(M, VPID, Params[0]->getType());
Function *VPFunc;
switch (VPID) {
default:
VPFunc = Intrinsic::getDeclaration(M, VPID, Params[0]->getType());
break;
case Intrinsic::vp_load:
VPFunc = Intrinsic::getDeclaration(
M, VPID,
{Params[0]->getType()->getPointerElementType(), Params[0]->getType()});
break;
case Intrinsic::vp_gather:
VPFunc = Intrinsic::getDeclaration(
M, VPID,
{VectorType::get(cast<VectorType>(Params[0]->getType())
->getElementType()
->getPointerElementType(),
cast<VectorType>(Params[0]->getType())),
Params[0]->getType()});
break;
case Intrinsic::vp_store:
VPFunc = Intrinsic::getDeclaration(
M, VPID,
{Params[1]->getType()->getPointerElementType(), Params[1]->getType()});
break;
case Intrinsic::vp_scatter:
VPFunc = Intrinsic::getDeclaration(
M, VPID, {Params[0]->getType(), Params[1]->getType()});
break;
}
assert(VPFunc && "Could not declare VP intrinsic");
return VPFunc;
}

5
unittests/IR/VPIntrinsicTest.cpp
View File

@ -46,6 +46,11 @@ protected:
Str << " declare <8 x float> @llvm.vp." << BinaryFPOpcode
<< ".v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) ";
Str << " declare void @llvm.vp.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, <8 x i1>, i32) ";
Str << " declare void @llvm.vp.scatter.v8i32.v8p0i32(<8 x i32>, <8 x i32*>, <8 x i1>, i32) ";
Str << " declare <8 x i32> @llvm.vp.load.v8i32.p0v8i32(<8 x i32>*, <8 x i1>, i32) ";
Str << " declare <8 x i32> @llvm.vp.gather.v8i32.v8p0i32(<8 x i32*>, <8 x i1>, i32) ";
return parseAssemblyString(Str.str(), Err, C);
}
};