From ecd2dc975e080e477f3ec974f977fca6baad2b35 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sat, 3 Jul 2021 14:57:41 +0200 Subject: [PATCH] [IRBuilder] Add type argument to CreateMaskedLoad/Gather Same as other CreateLoad-style APIs, these need an explicit type argument to support opaque pointers. Differential Revision: https://reviews.llvm.org/D105395 --- include/llvm/IR/IRBuilder.h | 4 +-- lib/IR/AutoUpgrade.cpp | 5 ++- lib/IR/IRBuilder.cpp | 31 ++++++++++++------- lib/Target/Hexagon/HexagonVectorCombine.cpp | 2 +- lib/Target/X86/X86InstCombineIntrinsic.cpp | 4 +-- .../Instrumentation/MemorySanitizer.cpp | 2 +- lib/Transforms/Vectorize/LoopVectorize.cpp | 8 ++--- lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +- 8 files changed, 32 insertions(+), 26 deletions(-) diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index 9dae6adaebe..9bd5ae1908c 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -752,7 +752,7 @@ public: CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr); /// Create a call to Masked Load intrinsic - CallInst *CreateMaskedLoad(Value *Ptr, Align Alignment, Value *Mask, + CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru = nullptr, const Twine &Name = ""); /// Create a call to Masked Store intrinsic @@ -760,7 +760,7 @@ public: Value *Mask); /// Create a call to Masked Gather intrinsic - CallInst *CreateMaskedGather(Value *Ptrs, Align Alignment, + CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask = nullptr, Value *PassThru = nullptr, const Twine &Name = ""); diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp index 851e8e45f47..6271385183e 100644 --- a/lib/IR/AutoUpgrade.cpp +++ b/lib/IR/AutoUpgrade.cpp @@ -1421,10 +1421,9 @@ static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment); // Convert the mask from an integer type to a vector of i1. - unsigned NumElts = - cast(Passthru->getType())->getNumElements(); + unsigned NumElts = cast(ValTy)->getNumElements(); Mask = getX86MaskVec(Builder, Mask, NumElts); - return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru); + return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru); } static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) { diff --git a/lib/IR/IRBuilder.cpp b/lib/IR/IRBuilder.cpp index 47b154c3cdc..5ea9867a246 100644 --- a/lib/IR/IRBuilder.cpp +++ b/lib/IR/IRBuilder.cpp @@ -493,6 +493,7 @@ Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) { } /// Create a call to a Masked Load intrinsic. +/// \p Ty - vector type to load /// \p Ptr - base pointer for the load /// \p Alignment - alignment of the source location /// \p Mask - vector of booleans which indicates what vector lanes should @@ -500,16 +501,16 @@ Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) { /// \p PassThru - pass-through value that is used to fill the masked-off lanes /// of the result /// \p Name - name of the result variable -CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment, +CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru, const Twine &Name) { auto *PtrTy = cast(Ptr->getType()); - Type *DataTy = PtrTy->getElementType(); - assert(DataTy->isVectorTy() && "Ptr should point to a vector"); + assert(Ty->isVectorTy() && "Type should be vector"); + assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type"); assert(Mask && "Mask should not be all-ones (null)"); if (!PassThru) - PassThru = UndefValue::get(DataTy); - Type *OverloadedTypes[] = { DataTy, PtrTy }; + PassThru = UndefValue::get(Ty); + Type *OverloadedTypes[] = { Ty, PtrTy }; Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru}; return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name); @@ -546,6 +547,7 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, } /// Create a call to a Masked Gather intrinsic. +/// \p Ty - vector type to gather /// \p Ptrs - vector of pointers for loading /// \p Align - alignment for one element /// \p Mask - vector of booleans which indicates what vector lanes should @@ -553,22 +555,27 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, /// \p PassThru - pass-through value that is used to fill the masked-off lanes /// of the result /// \p Name - name of the result variable -CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment, - Value *Mask, Value *PassThru, +CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, + Align Alignment, Value *Mask, + Value *PassThru, const Twine &Name) { + auto *VecTy = cast(Ty); + ElementCount NumElts = VecTy->getElementCount(); auto *PtrsTy = cast(Ptrs->getType()); - auto *PtrTy = cast(PtrsTy->getElementType()); - ElementCount NumElts = PtrsTy->getElementCount(); - auto *DataTy = VectorType::get(PtrTy->getElementType(), NumElts); + assert(cast(PtrsTy->getElementType()) + ->isOpaqueOrPointeeTypeMatches( + cast(Ty)->getElementType()) && + "Element type mismatch"); + assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch"); if (!Mask) Mask = Constant::getAllOnesValue( VectorType::get(Type::getInt1Ty(Context), NumElts)); if (!PassThru) - PassThru = UndefValue::get(DataTy); + PassThru = UndefValue::get(Ty); - Type *OverloadedTypes[] = {DataTy, PtrsTy}; + Type *OverloadedTypes[] = {Ty, PtrsTy}; Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru}; // We specify only one type when we create this intrinsic. Types of other diff --git a/lib/Target/Hexagon/HexagonVectorCombine.cpp b/lib/Target/Hexagon/HexagonVectorCombine.cpp index 9973822bf37..f949a9327f7 100644 --- a/lib/Target/Hexagon/HexagonVectorCombine.cpp +++ b/lib/Target/Hexagon/HexagonVectorCombine.cpp @@ -475,7 +475,7 @@ auto AlignVectors::createAlignedLoad(IRBuilder<> &Builder, Type *ValTy, return PassThru; if (Mask == ConstantInt::getTrue(Mask->getType())) return Builder.CreateAlignedLoad(ValTy, Ptr, Align(Alignment)); - return Builder.CreateMaskedLoad(Ptr, Align(Alignment), Mask, PassThru); + return Builder.CreateMaskedLoad(ValTy, Ptr, Align(Alignment), Mask, PassThru); } auto AlignVectors::createAlignedStore(IRBuilder<> &Builder, Value *Val, diff --git a/lib/Target/X86/X86InstCombineIntrinsic.cpp b/lib/Target/X86/X86InstCombineIntrinsic.cpp index c4150ed5285..243cf7c9e6a 100644 --- a/lib/Target/X86/X86InstCombineIntrinsic.cpp +++ b/lib/Target/X86/X86InstCombineIntrinsic.cpp @@ -72,8 +72,8 @@ static Instruction *simplifyX86MaskedLoad(IntrinsicInst &II, InstCombiner &IC) { Value *PtrCast = IC.Builder.CreateBitCast(Ptr, VecPtrTy, "castvec"); // The pass-through vector for an x86 masked load is a zero vector. - CallInst *NewMaskedLoad = - IC.Builder.CreateMaskedLoad(PtrCast, Align(1), BoolMask, ZeroVec); + CallInst *NewMaskedLoad = IC.Builder.CreateMaskedLoad( + II.getType(), PtrCast, Align(1), BoolMask, ZeroVec); return IC.replaceInstUsesWith(II, NewMaskedLoad); } diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp index fb779e8c555..28bccd38206 100644 --- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -3117,7 +3117,7 @@ struct MemorySanitizerVisitor : public InstVisitor { if (PropagateShadow) { std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); - setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Alignment, Mask, + setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask, getShadow(PassThru), "_msmaskedld")); } else { setShadow(&I, getCleanShadow(&I)); diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index d923d200290..617300f07db 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2778,7 +2778,7 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup( : ShuffledMask; } NewLoad = - Builder.CreateMaskedLoad(AddrParts[Part], Group->getAlign(), + Builder.CreateMaskedLoad(VecTy, AddrParts[Part], Group->getAlign(), GroupMask, PoisonVec, "wide.masked.vec"); } else @@ -2990,15 +2990,15 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction( if (CreateGatherScatter) { Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; Value *VectorGep = State.get(Addr, Part); - NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart, + NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart, nullptr, "wide.masked.gather"); addMetadata(NewLI, LI); } else { auto *VecPtr = CreateVecPtr(Part, State.get(Addr, VPIteration(0, 0))); if (isMaskRequired) NewLI = Builder.CreateMaskedLoad( - VecPtr, Alignment, BlockInMaskParts[Part], PoisonValue::get(DataTy), - "wide.masked.load"); + DataTy, VecPtr, Alignment, BlockInMaskParts[Part], + PoisonValue::get(DataTy), "wide.masked.load"); else NewLI = Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load"); diff --git a/lib/Transforms/Vectorize/SLPVectorizer.cpp b/lib/Transforms/Vectorize/SLPVectorizer.cpp index 830d730c9ff..e655227c2de 100644 --- a/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -5403,7 +5403,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { for (Value *V : E->Scalars) CommonAlignment = commonAlignment(CommonAlignment, cast(V)->getAlign()); - NewLI = Builder.CreateMaskedGather(VecPtr, CommonAlignment); + NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment); } Value *V = propagateMetadata(NewLI, E->Scalars);