mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-22 10:42:39 +01:00
Revert "Revert "[NFC][llvm] Make the contructors of ElementCount
private.""
Was reverted because MLIR/Flang builds were broken, these APIs have been fixed in the meantime.
This commit is contained in:
parent
4386b1823a
commit
db235b2187
@ -99,7 +99,8 @@ struct VFShape {
|
|||||||
// Retrieve the VFShape that can be used to map a (scalar) function to itself,
|
// Retrieve the VFShape that can be used to map a (scalar) function to itself,
|
||||||
// with VF = 1.
|
// with VF = 1.
|
||||||
static VFShape getScalarShape(const CallInst &CI) {
|
static VFShape getScalarShape(const CallInst &CI) {
|
||||||
return VFShape::get(CI, /*EC*/ {1, false}, /*HasGlobalPredicate*/ false);
|
return VFShape::get(CI, ElementCount::getFixed(1),
|
||||||
|
/*HasGlobalPredicate*/ false);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Retrieve the basic vectorization shape of the function, where all
|
// Retrieve the basic vectorization shape of the function, where all
|
||||||
@ -305,7 +306,7 @@ typedef unsigned ID;
|
|||||||
inline Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) {
|
inline Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) {
|
||||||
if (Scalar->isVoidTy() || VF == 1)
|
if (Scalar->isVoidTy() || VF == 1)
|
||||||
return Scalar;
|
return Scalar;
|
||||||
return VectorType::get(Scalar, {VF, isScalable});
|
return VectorType::get(Scalar, ElementCount::get(VF, isScalable));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Identify if the intrinsic is trivially vectorizable.
|
/// Identify if the intrinsic is trivially vectorizable.
|
||||||
|
@ -446,7 +446,8 @@ public:
|
|||||||
|
|
||||||
static VectorType *get(Type *ElementType, unsigned NumElements,
|
static VectorType *get(Type *ElementType, unsigned NumElements,
|
||||||
bool Scalable) {
|
bool Scalable) {
|
||||||
return VectorType::get(ElementType, {NumElements, Scalable});
|
return VectorType::get(ElementType,
|
||||||
|
ElementCount::get(NumElements, Scalable));
|
||||||
}
|
}
|
||||||
|
|
||||||
static VectorType *get(Type *ElementType, const VectorType *Other) {
|
static VectorType *get(Type *ElementType, const VectorType *Other) {
|
||||||
@ -640,7 +641,7 @@ public:
|
|||||||
};
|
};
|
||||||
|
|
||||||
inline ElementCount VectorType::getElementCount() const {
|
inline ElementCount VectorType::getElementCount() const {
|
||||||
return ElementCount(ElementQuantity, isa<ScalableVectorType>(this));
|
return ElementCount::get(ElementQuantity, isa<ScalableVectorType>(this));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Class to represent pointers.
|
/// Class to represent pointers.
|
||||||
|
@ -134,7 +134,9 @@ namespace Intrinsic {
|
|||||||
unsigned Pointer_AddressSpace;
|
unsigned Pointer_AddressSpace;
|
||||||
unsigned Struct_NumElements;
|
unsigned Struct_NumElements;
|
||||||
unsigned Argument_Info;
|
unsigned Argument_Info;
|
||||||
ElementCount Vector_Width;
|
// There is no default constructor in `ElementCount`, so we need
|
||||||
|
// to explicitly initialize this field with a value.
|
||||||
|
ElementCount Vector_Width = ElementCount::getFixed(0);
|
||||||
};
|
};
|
||||||
|
|
||||||
enum ArgKind {
|
enum ArgKind {
|
||||||
@ -190,8 +192,7 @@ namespace Intrinsic {
|
|||||||
static IITDescriptor getVector(unsigned Width, bool IsScalable) {
|
static IITDescriptor getVector(unsigned Width, bool IsScalable) {
|
||||||
IITDescriptor Result;
|
IITDescriptor Result;
|
||||||
Result.Kind = Vector;
|
Result.Kind = Vector;
|
||||||
Result.Vector_Width.Min = Width;
|
Result.Vector_Width = ElementCount::get(Width, IsScalable);
|
||||||
Result.Vector_Width.Scalable = IsScalable;
|
|
||||||
return Result;
|
return Result;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -737,7 +737,7 @@ namespace llvm {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ElementCount getVectorElementCount() const {
|
ElementCount getVectorElementCount() const {
|
||||||
return { getVectorNumElements(), isScalableVector() };
|
return ElementCount::get(getVectorNumElements(), isScalableVector());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Given a vector type, return the minimum number of elements it contains.
|
/// Given a vector type, return the minimum number of elements it contains.
|
||||||
|
@ -26,16 +26,23 @@ namespace llvm {
|
|||||||
template <typename T> struct DenseMapInfo;
|
template <typename T> struct DenseMapInfo;
|
||||||
|
|
||||||
class ElementCount {
|
class ElementCount {
|
||||||
|
private:
|
||||||
|
/// Prevent code from using initializer-list contructors like
|
||||||
|
/// ElementCount EC = {<unsigned>, <bool>}. The static `get*`
|
||||||
|
/// methods below are preferred, as users should always make a
|
||||||
|
/// conscious choice on the type of `ElementCount` they are
|
||||||
|
/// requesting.
|
||||||
|
ElementCount(unsigned Min, bool Scalable) : Min(Min), Scalable(Scalable) {}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
|
/// No default constructor. Users should use one of the `get*`
|
||||||
|
/// static methods below, as they should always make a conscious
|
||||||
|
/// choice on the type of `ElementCount` they are requesting.
|
||||||
|
ElementCount() = delete;
|
||||||
unsigned Min; // Minimum number of vector elements.
|
unsigned Min; // Minimum number of vector elements.
|
||||||
bool Scalable; // If true, NumElements is a multiple of 'Min' determined
|
bool Scalable; // If true, NumElements is a multiple of 'Min' determined
|
||||||
// at runtime rather than compile time.
|
// at runtime rather than compile time.
|
||||||
|
|
||||||
ElementCount() = default;
|
|
||||||
|
|
||||||
ElementCount(unsigned Min, bool Scalable)
|
|
||||||
: Min(Min), Scalable(Scalable) {}
|
|
||||||
|
|
||||||
ElementCount operator*(unsigned RHS) {
|
ElementCount operator*(unsigned RHS) {
|
||||||
return { Min * RHS, Scalable };
|
return { Min * RHS, Scalable };
|
||||||
}
|
}
|
||||||
@ -54,7 +61,13 @@ public:
|
|||||||
bool operator!=(unsigned RHS) const { return !(*this == RHS); }
|
bool operator!=(unsigned RHS) const { return !(*this == RHS); }
|
||||||
|
|
||||||
ElementCount NextPowerOf2() const {
|
ElementCount NextPowerOf2() const {
|
||||||
return ElementCount(llvm::NextPowerOf2(Min), Scalable);
|
return {(unsigned)llvm::NextPowerOf2(Min), Scalable};
|
||||||
|
}
|
||||||
|
|
||||||
|
static ElementCount getFixed(unsigned Min) { return {Min, false}; }
|
||||||
|
static ElementCount getScalable(unsigned Min) { return {Min, true}; }
|
||||||
|
static ElementCount get(unsigned Min, bool Scalable) {
|
||||||
|
return {Min, Scalable};
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -279,8 +292,12 @@ inline TypeSize alignTo(TypeSize Size, uint64_t Align) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <> struct DenseMapInfo<ElementCount> {
|
template <> struct DenseMapInfo<ElementCount> {
|
||||||
static inline ElementCount getEmptyKey() { return {~0U, true}; }
|
static inline ElementCount getEmptyKey() {
|
||||||
static inline ElementCount getTombstoneKey() { return {~0U - 1, false}; }
|
return ElementCount::getScalable(~0U);
|
||||||
|
}
|
||||||
|
static inline ElementCount getTombstoneKey() {
|
||||||
|
return ElementCount::getFixed(~0U - 1);
|
||||||
|
}
|
||||||
static unsigned getHashValue(const ElementCount& EltCnt) {
|
static unsigned getHashValue(const ElementCount& EltCnt) {
|
||||||
if (EltCnt.Scalable)
|
if (EltCnt.Scalable)
|
||||||
return (EltCnt.Min * 37U) - 1U;
|
return (EltCnt.Min * 37U) - 1U;
|
||||||
|
@ -310,7 +310,7 @@ ElementCount getECFromSignature(FunctionType *Signature) {
|
|||||||
if (auto *VTy = dyn_cast<VectorType>(Ty))
|
if (auto *VTy = dyn_cast<VectorType>(Ty))
|
||||||
return VTy->getElementCount();
|
return VTy->getElementCount();
|
||||||
|
|
||||||
return ElementCount(/*Min=*/1, /*Scalable=*/false);
|
return ElementCount::getFixed(/*Min=*/1);
|
||||||
}
|
}
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
|
@ -7395,7 +7395,7 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
|
|||||||
// All vector parameters should have the same vector width.
|
// All vector parameters should have the same vector width.
|
||||||
ElementCount GEPWidth = BaseType->isVectorTy()
|
ElementCount GEPWidth = BaseType->isVectorTy()
|
||||||
? cast<VectorType>(BaseType)->getElementCount()
|
? cast<VectorType>(BaseType)->getElementCount()
|
||||||
: ElementCount(0, false);
|
: ElementCount::getFixed(0);
|
||||||
|
|
||||||
while (EatIfPresent(lltok::comma)) {
|
while (EatIfPresent(lltok::comma)) {
|
||||||
if (Lex.getKind() == lltok::MetadataVar) {
|
if (Lex.getKind() == lltok::MetadataVar) {
|
||||||
@ -7408,7 +7408,7 @@ int LLParser::ParseGetElementPtr(Instruction *&Inst, PerFunctionState &PFS) {
|
|||||||
|
|
||||||
if (auto *ValVTy = dyn_cast<VectorType>(Val->getType())) {
|
if (auto *ValVTy = dyn_cast<VectorType>(Val->getType())) {
|
||||||
ElementCount ValNumEl = ValVTy->getElementCount();
|
ElementCount ValNumEl = ValVTy->getElementCount();
|
||||||
if (GEPWidth != ElementCount(0, false) && GEPWidth != ValNumEl)
|
if (GEPWidth != ElementCount::getFixed(0) && GEPWidth != ValNumEl)
|
||||||
return Error(EltLoc,
|
return Error(EltLoc,
|
||||||
"getelementptr vector index has a wrong number of elements");
|
"getelementptr vector index has a wrong number of elements");
|
||||||
GEPWidth = ValNumEl;
|
GEPWidth = ValNumEl;
|
||||||
|
@ -729,15 +729,15 @@ static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
|
|||||||
assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
|
assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
|
||||||
"Mixing scalable and fixed vectors when copying in parts");
|
"Mixing scalable and fixed vectors when copying in parts");
|
||||||
|
|
||||||
ElementCount DestEltCnt;
|
Optional<ElementCount> DestEltCnt;
|
||||||
|
|
||||||
if (IntermediateVT.isVector())
|
if (IntermediateVT.isVector())
|
||||||
DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
|
DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
|
||||||
else
|
else
|
||||||
DestEltCnt = ElementCount(NumIntermediates, false);
|
DestEltCnt = ElementCount::getFixed(NumIntermediates);
|
||||||
|
|
||||||
EVT BuiltVectorTy = EVT::getVectorVT(
|
EVT BuiltVectorTy = EVT::getVectorVT(
|
||||||
*DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt);
|
*DAG.getContext(), IntermediateVT.getScalarType(), DestEltCnt.getValue());
|
||||||
if (ValueVT != BuiltVectorTy) {
|
if (ValueVT != BuiltVectorTy) {
|
||||||
if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
|
if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
|
||||||
Val = Widened;
|
Val = Widened;
|
||||||
@ -3746,7 +3746,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
|
|||||||
bool IsVectorGEP = I.getType()->isVectorTy();
|
bool IsVectorGEP = I.getType()->isVectorTy();
|
||||||
ElementCount VectorElementCount =
|
ElementCount VectorElementCount =
|
||||||
IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
|
IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
|
||||||
: ElementCount(0, false);
|
: ElementCount::getFixed(0);
|
||||||
|
|
||||||
if (IsVectorGEP && !N.getValueType().isVector()) {
|
if (IsVectorGEP && !N.getValueType().isVector()) {
|
||||||
LLVMContext &Context = *DAG.getContext();
|
LLVMContext &Context = *DAG.getContext();
|
||||||
|
@ -866,7 +866,7 @@ TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
|
|||||||
if (NumElts == 1)
|
if (NumElts == 1)
|
||||||
return LegalizeKind(TypeScalarizeVector, EltVT);
|
return LegalizeKind(TypeScalarizeVector, EltVT);
|
||||||
|
|
||||||
if (VT.getVectorElementCount() == ElementCount(1, true))
|
if (VT.getVectorElementCount() == ElementCount::getScalable(1))
|
||||||
report_fatal_error("Cannot legalize this vector");
|
report_fatal_error("Cannot legalize this vector");
|
||||||
|
|
||||||
// Try to widen vector elements until the element type is a power of two and
|
// Try to widen vector elements until the element type is a power of two and
|
||||||
|
@ -49,8 +49,7 @@ EVT EVT::getExtendedVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements,
|
|||||||
|
|
||||||
EVT EVT::getExtendedVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) {
|
EVT EVT::getExtendedVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) {
|
||||||
EVT ResultVT;
|
EVT ResultVT;
|
||||||
ResultVT.LLVMTy =
|
ResultVT.LLVMTy = VectorType::get(VT.getTypeForEVT(Context), EC);
|
||||||
VectorType::get(VT.getTypeForEVT(Context), {EC.Min, EC.Scalable});
|
|
||||||
assert(ResultVT.isExtended() && "Type is not extended!");
|
assert(ResultVT.isExtended() && "Type is not extended!");
|
||||||
return ResultVT;
|
return ResultVT;
|
||||||
}
|
}
|
||||||
|
@ -919,7 +919,8 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
|
|||||||
ArrayRef<int> Mask) {
|
ArrayRef<int> Mask) {
|
||||||
auto *V1VTy = cast<VectorType>(V1->getType());
|
auto *V1VTy = cast<VectorType>(V1->getType());
|
||||||
unsigned MaskNumElts = Mask.size();
|
unsigned MaskNumElts = Mask.size();
|
||||||
ElementCount MaskEltCount = {MaskNumElts, isa<ScalableVectorType>(V1VTy)};
|
auto MaskEltCount =
|
||||||
|
ElementCount::get(MaskNumElts, isa<ScalableVectorType>(V1VTy));
|
||||||
Type *EltTy = V1VTy->getElementType();
|
Type *EltTy = V1VTy->getElementType();
|
||||||
|
|
||||||
// Undefined shuffle mask -> undefined value.
|
// Undefined shuffle mask -> undefined value.
|
||||||
|
@ -2245,7 +2245,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
|
|||||||
unsigned AS = C->getType()->getPointerAddressSpace();
|
unsigned AS = C->getType()->getPointerAddressSpace();
|
||||||
Type *ReqTy = DestTy->getPointerTo(AS);
|
Type *ReqTy = DestTy->getPointerTo(AS);
|
||||||
|
|
||||||
ElementCount EltCount = {0, false};
|
auto EltCount = ElementCount::getFixed(0);
|
||||||
if (VectorType *VecTy = dyn_cast<VectorType>(C->getType()))
|
if (VectorType *VecTy = dyn_cast<VectorType>(C->getType()))
|
||||||
EltCount = VecTy->getElementCount();
|
EltCount = VecTy->getElementCount();
|
||||||
else
|
else
|
||||||
@ -2938,7 +2938,7 @@ Constant *ConstantDataVector::getSplat(unsigned NumElts, Constant *V) {
|
|||||||
return getFP(V->getType(), Elts);
|
return getFP(V->getType(), Elts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ConstantVector::getSplat({NumElts, false}, V);
|
return ConstantVector::getSplat(ElementCount::getFixed(NumElts), V);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -997,7 +997,7 @@ Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
|
|||||||
|
|
||||||
Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
|
Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
|
||||||
const Twine &Name) {
|
const Twine &Name) {
|
||||||
ElementCount EC(NumElts, false);
|
auto EC = ElementCount::getFixed(NumElts);
|
||||||
return CreateVectorSplat(EC, V, Name);
|
return CreateVectorSplat(EC, V, Name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3330,9 +3330,9 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
|
|||||||
// scalar types means that checking that vector lengths match also checks that
|
// scalar types means that checking that vector lengths match also checks that
|
||||||
// scalars are not being converted to vectors or vectors to scalars).
|
// scalars are not being converted to vectors or vectors to scalars).
|
||||||
ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount()
|
ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount()
|
||||||
: ElementCount(0, false);
|
: ElementCount::getFixed(0);
|
||||||
ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount()
|
ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount()
|
||||||
: ElementCount(0, false);
|
: ElementCount::getFixed(0);
|
||||||
|
|
||||||
// Switch on the opcode provided
|
// Switch on the opcode provided
|
||||||
switch (op) {
|
switch (op) {
|
||||||
@ -3390,9 +3390,9 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) {
|
|||||||
if (SrcIsVec && DstIsVec)
|
if (SrcIsVec && DstIsVec)
|
||||||
return SrcEC == DstEC;
|
return SrcEC == DstEC;
|
||||||
if (SrcIsVec)
|
if (SrcIsVec)
|
||||||
return SrcEC == ElementCount(1, false);
|
return SrcEC == ElementCount::getFixed(1);
|
||||||
if (DstIsVec)
|
if (DstIsVec)
|
||||||
return DstEC == ElementCount(1, false);
|
return DstEC == ElementCount::getFixed(1);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -619,7 +619,7 @@ FixedVectorType *FixedVectorType::get(Type *ElementType, unsigned NumElts) {
|
|||||||
"be an integer, floating point, or "
|
"be an integer, floating point, or "
|
||||||
"pointer type.");
|
"pointer type.");
|
||||||
|
|
||||||
ElementCount EC(NumElts, false);
|
auto EC = ElementCount::getFixed(NumElts);
|
||||||
|
|
||||||
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
|
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
|
||||||
VectorType *&Entry = ElementType->getContext()
|
VectorType *&Entry = ElementType->getContext()
|
||||||
@ -641,7 +641,7 @@ ScalableVectorType *ScalableVectorType::get(Type *ElementType,
|
|||||||
"be an integer, floating point, or "
|
"be an integer, floating point, or "
|
||||||
"pointer type.");
|
"pointer type.");
|
||||||
|
|
||||||
ElementCount EC(MinNumElts, true);
|
auto EC = ElementCount::getScalable(MinNumElts);
|
||||||
|
|
||||||
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
|
LLVMContextImpl *pImpl = ElementType->getContext().pImpl;
|
||||||
VectorType *&Entry = ElementType->getContext()
|
VectorType *&Entry = ElementType->getContext()
|
||||||
|
@ -11985,7 +11985,8 @@ static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
|
|||||||
|
|
||||||
unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8;
|
unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8;
|
||||||
unsigned ByteSize = VT.getSizeInBits().getKnownMinSize() / 8;
|
unsigned ByteSize = VT.getSizeInBits().getKnownMinSize() / 8;
|
||||||
EVT ByteVT = EVT::getVectorVT(Ctx, MVT::i8, { ByteSize, true });
|
EVT ByteVT =
|
||||||
|
EVT::getVectorVT(Ctx, MVT::i8, ElementCount::getScalable(ByteSize));
|
||||||
|
|
||||||
// Convert everything to the domain of EXT (i.e bytes).
|
// Convert everything to the domain of EXT (i.e bytes).
|
||||||
SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
|
SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
|
||||||
|
@ -1804,10 +1804,10 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
|
|||||||
// FIXME: If the step is non-constant, we create the vector splat with
|
// FIXME: If the step is non-constant, we create the vector splat with
|
||||||
// IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
|
// IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
|
||||||
// handle a constant vector splat.
|
// handle a constant vector splat.
|
||||||
Value *SplatVF =
|
Value *SplatVF = isa<Constant>(Mul)
|
||||||
isa<Constant>(Mul)
|
? ConstantVector::getSplat(ElementCount::getFixed(VF),
|
||||||
? ConstantVector::getSplat({VF, false}, cast<Constant>(Mul))
|
cast<Constant>(Mul))
|
||||||
: Builder.CreateVectorSplat(VF, Mul);
|
: Builder.CreateVectorSplat(VF, Mul);
|
||||||
Builder.restoreIP(CurrIP);
|
Builder.restoreIP(CurrIP);
|
||||||
|
|
||||||
// We may need to add the step a number of times, depending on the unroll
|
// We may need to add the step a number of times, depending on the unroll
|
||||||
@ -3399,7 +3399,8 @@ unsigned LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
|
|||||||
// If we can't emit a vector call for this function, then the currently found
|
// If we can't emit a vector call for this function, then the currently found
|
||||||
// cost is the cost we need to return.
|
// cost is the cost we need to return.
|
||||||
NeedToScalarize = true;
|
NeedToScalarize = true;
|
||||||
VFShape Shape = VFShape::get(*CI, {VF, false}, false /*HasGlobalPred*/);
|
VFShape Shape =
|
||||||
|
VFShape::get(*CI, ElementCount::getFixed(VF), false /*HasGlobalPred*/);
|
||||||
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
|
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
|
||||||
|
|
||||||
if (!TLI || CI->isNoBuiltin() || !VecFunc)
|
if (!TLI || CI->isNoBuiltin() || !VecFunc)
|
||||||
@ -3860,7 +3861,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
|
|||||||
// incoming scalar reduction.
|
// incoming scalar reduction.
|
||||||
VectorStart = ReductionStartValue;
|
VectorStart = ReductionStartValue;
|
||||||
} else {
|
} else {
|
||||||
Identity = ConstantVector::getSplat({VF, false}, Iden);
|
Identity = ConstantVector::getSplat(ElementCount::getFixed(VF), Iden);
|
||||||
|
|
||||||
// This vector is the Identity vector where the first element is the
|
// This vector is the Identity vector where the first element is the
|
||||||
// incoming scalar reduction.
|
// incoming scalar reduction.
|
||||||
@ -4541,8 +4542,8 @@ void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPUser &ArgOperands,
|
|||||||
assert(VectorF && "Can't retrieve vector intrinsic.");
|
assert(VectorF && "Can't retrieve vector intrinsic.");
|
||||||
} else {
|
} else {
|
||||||
// Use vector version of the function call.
|
// Use vector version of the function call.
|
||||||
const VFShape Shape =
|
const VFShape Shape = VFShape::get(*CI, ElementCount::getFixed(VF),
|
||||||
VFShape::get(*CI, {VF, false} /*EC*/, false /*HasGlobalPred*/);
|
false /*HasGlobalPred*/);
|
||||||
#ifndef NDEBUG
|
#ifndef NDEBUG
|
||||||
assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
|
assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
|
||||||
"Can't create vector function.");
|
"Can't create vector function.");
|
||||||
|
@ -3029,7 +3029,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
|
|||||||
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
|
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
|
||||||
|
|
||||||
VFShape Shape = VFShape::get(
|
VFShape Shape = VFShape::get(
|
||||||
*CI, {static_cast<unsigned int>(VL.size()), false /*Scalable*/},
|
*CI, ElementCount::getFixed(static_cast<unsigned int>(VL.size())),
|
||||||
false /*HasGlobalPred*/);
|
false /*HasGlobalPred*/);
|
||||||
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
|
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
|
||||||
|
|
||||||
@ -3264,9 +3264,9 @@ getVectorCallCosts(CallInst *CI, VectorType *VecTy, TargetTransformInfo *TTI,
|
|||||||
int IntrinsicCost =
|
int IntrinsicCost =
|
||||||
TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
|
TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
|
||||||
|
|
||||||
auto Shape =
|
auto Shape = VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
|
||||||
VFShape::get(*CI, {static_cast<unsigned>(VecTy->getNumElements()), false},
|
VecTy->getNumElements())),
|
||||||
false /*HasGlobalPred*/);
|
false /*HasGlobalPred*/);
|
||||||
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
|
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
|
||||||
int LibCost = IntrinsicCost;
|
int LibCost = IntrinsicCost;
|
||||||
if (!CI->isNoBuiltin() && VecFunc) {
|
if (!CI->isNoBuiltin() && VecFunc) {
|
||||||
@ -4553,9 +4553,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
|
|||||||
|
|
||||||
Function *CF;
|
Function *CF;
|
||||||
if (!UseIntrinsic) {
|
if (!UseIntrinsic) {
|
||||||
VFShape Shape = VFShape::get(
|
VFShape Shape =
|
||||||
*CI, {static_cast<unsigned>(VecTy->getNumElements()), false},
|
VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
|
||||||
false /*HasGlobalPred*/);
|
VecTy->getNumElements())),
|
||||||
|
false /*HasGlobalPred*/);
|
||||||
CF = VFDatabase(*CI).getVectorizedFunction(Shape);
|
CF = VFDatabase(*CI).getVectorizedFunction(Shape);
|
||||||
} else {
|
} else {
|
||||||
Type *Tys[] = {FixedVectorType::get(CI->getType(), E->Scalars.size())};
|
Type *Tys[] = {FixedVectorType::get(CI->getType(), E->Scalars.size())};
|
||||||
|
@ -93,7 +93,8 @@ TEST_F(BasicTest, isSplat) {
|
|||||||
Value *SplatC = IRB.CreateVectorSplat(5, ScalarC);
|
Value *SplatC = IRB.CreateVectorSplat(5, ScalarC);
|
||||||
EXPECT_TRUE(isSplatValue(SplatC));
|
EXPECT_TRUE(isSplatValue(SplatC));
|
||||||
|
|
||||||
Value *SplatC_SVE = IRB.CreateVectorSplat(ElementCount(5, true), ScalarC);
|
Value *SplatC_SVE =
|
||||||
|
IRB.CreateVectorSplat(ElementCount::getScalable(5), ScalarC);
|
||||||
EXPECT_TRUE(isSplatValue(SplatC_SVE));
|
EXPECT_TRUE(isSplatValue(SplatC_SVE));
|
||||||
|
|
||||||
// FIXME: Constant splat analysis does not allow undef elements.
|
// FIXME: Constant splat analysis does not allow undef elements.
|
||||||
@ -502,7 +503,7 @@ protected:
|
|||||||
SmallVector<VFParameter, 8> &ExpectedParams = Expected.Parameters;
|
SmallVector<VFParameter, 8> &ExpectedParams = Expected.Parameters;
|
||||||
|
|
||||||
void buildShape(unsigned VF, bool IsScalable, bool HasGlobalPred) {
|
void buildShape(unsigned VF, bool IsScalable, bool HasGlobalPred) {
|
||||||
Shape = VFShape::get(*CI, {VF, IsScalable}, HasGlobalPred);
|
Shape = VFShape::get(*CI, ElementCount::get(VF, IsScalable), HasGlobalPred);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool validParams(ArrayRef<VFParameter> Parameters) {
|
bool validParams(ArrayRef<VFParameter> Parameters) {
|
||||||
|
@ -49,12 +49,12 @@ TEST(ScalableVectorMVTsTest, HelperFuncs) {
|
|||||||
ASSERT_TRUE(Vnx4i32.isScalableVector());
|
ASSERT_TRUE(Vnx4i32.isScalableVector());
|
||||||
|
|
||||||
// Create with separate llvm::ElementCount
|
// Create with separate llvm::ElementCount
|
||||||
auto EltCnt = ElementCount(2, true);
|
auto EltCnt = ElementCount::getScalable(2);
|
||||||
EVT Vnx2i32 = EVT::getVectorVT(Ctx, MVT::i32, EltCnt);
|
EVT Vnx2i32 = EVT::getVectorVT(Ctx, MVT::i32, EltCnt);
|
||||||
ASSERT_TRUE(Vnx2i32.isScalableVector());
|
ASSERT_TRUE(Vnx2i32.isScalableVector());
|
||||||
|
|
||||||
// Create with inline llvm::ElementCount
|
// Create with inline llvm::ElementCount
|
||||||
EVT Vnx2i64 = EVT::getVectorVT(Ctx, MVT::i64, {2, true});
|
EVT Vnx2i64 = EVT::getVectorVT(Ctx, MVT::i64, ElementCount::getScalable(2));
|
||||||
ASSERT_TRUE(Vnx2i64.isScalableVector());
|
ASSERT_TRUE(Vnx2i64.isScalableVector());
|
||||||
|
|
||||||
// Check that changing scalar types/element count works
|
// Check that changing scalar types/element count works
|
||||||
@ -66,7 +66,7 @@ TEST(ScalableVectorMVTsTest, HelperFuncs) {
|
|||||||
EXPECT_EQ(EVT::getVectorVT(Ctx, MVT::i64, EltCnt / 2), MVT::nxv1i64);
|
EXPECT_EQ(EVT::getVectorVT(Ctx, MVT::i64, EltCnt / 2), MVT::nxv1i64);
|
||||||
|
|
||||||
// Check that float->int conversion works
|
// Check that float->int conversion works
|
||||||
EVT Vnx2f64 = EVT::getVectorVT(Ctx, MVT::f64, {2, true});
|
EVT Vnx2f64 = EVT::getVectorVT(Ctx, MVT::f64, ElementCount::getScalable(2));
|
||||||
EXPECT_EQ(Vnx2f64.changeTypeToInteger(), Vnx2i64);
|
EXPECT_EQ(Vnx2f64.changeTypeToInteger(), Vnx2i64);
|
||||||
|
|
||||||
// Check fields inside llvm::ElementCount
|
// Check fields inside llvm::ElementCount
|
||||||
@ -77,7 +77,7 @@ TEST(ScalableVectorMVTsTest, HelperFuncs) {
|
|||||||
// Check that fixed-length vector types aren't scalable.
|
// Check that fixed-length vector types aren't scalable.
|
||||||
EVT V8i32 = EVT::getVectorVT(Ctx, MVT::i32, 8);
|
EVT V8i32 = EVT::getVectorVT(Ctx, MVT::i32, 8);
|
||||||
ASSERT_FALSE(V8i32.isScalableVector());
|
ASSERT_FALSE(V8i32.isScalableVector());
|
||||||
EVT V4f64 = EVT::getVectorVT(Ctx, MVT::f64, {4, false});
|
EVT V4f64 = EVT::getVectorVT(Ctx, MVT::f64, ElementCount::getFixed(4));
|
||||||
ASSERT_FALSE(V4f64.isScalableVector());
|
ASSERT_FALSE(V4f64.isScalableVector());
|
||||||
|
|
||||||
// Check that llvm::ElementCount works for fixed-length types.
|
// Check that llvm::ElementCount works for fixed-length types.
|
||||||
@ -90,7 +90,8 @@ TEST(ScalableVectorMVTsTest, IRToVTTranslation) {
|
|||||||
LLVMContext Ctx;
|
LLVMContext Ctx;
|
||||||
|
|
||||||
Type *Int64Ty = Type::getInt64Ty(Ctx);
|
Type *Int64Ty = Type::getInt64Ty(Ctx);
|
||||||
VectorType *ScV8Int64Ty = VectorType::get(Int64Ty, {8, true});
|
VectorType *ScV8Int64Ty =
|
||||||
|
VectorType::get(Int64Ty, ElementCount::getScalable(8));
|
||||||
|
|
||||||
// Check that we can map a scalable IR type to an MVT
|
// Check that we can map a scalable IR type to an MVT
|
||||||
MVT Mnxv8i64 = MVT::getVT(ScV8Int64Ty);
|
MVT Mnxv8i64 = MVT::getVT(ScV8Int64Ty);
|
||||||
@ -110,7 +111,7 @@ TEST(ScalableVectorMVTsTest, IRToVTTranslation) {
|
|||||||
TEST(ScalableVectorMVTsTest, VTToIRTranslation) {
|
TEST(ScalableVectorMVTsTest, VTToIRTranslation) {
|
||||||
LLVMContext Ctx;
|
LLVMContext Ctx;
|
||||||
|
|
||||||
EVT Enxv4f64 = EVT::getVectorVT(Ctx, MVT::f64, {4, true});
|
EVT Enxv4f64 = EVT::getVectorVT(Ctx, MVT::f64, ElementCount::getScalable(4));
|
||||||
|
|
||||||
Type *Ty = Enxv4f64.getTypeForEVT(Ctx);
|
Type *Ty = Enxv4f64.getTypeForEVT(Ctx);
|
||||||
VectorType *ScV4Float64Ty = cast<VectorType>(Ty);
|
VectorType *ScV4Float64Ty = cast<VectorType>(Ty);
|
||||||
|
@ -92,8 +92,8 @@ TEST(OperationsTest, SourcePreds) {
|
|||||||
ConstantStruct::get(StructType::create(Ctx, "OpaqueStruct"));
|
ConstantStruct::get(StructType::create(Ctx, "OpaqueStruct"));
|
||||||
Constant *a =
|
Constant *a =
|
||||||
ConstantArray::get(ArrayType::get(i32->getType(), 2), {i32, i32});
|
ConstantArray::get(ArrayType::get(i32->getType(), 2), {i32, i32});
|
||||||
Constant *v8i8 = ConstantVector::getSplat({8, false}, i8);
|
Constant *v8i8 = ConstantVector::getSplat(ElementCount::getFixed(8), i8);
|
||||||
Constant *v4f16 = ConstantVector::getSplat({4, false}, f16);
|
Constant *v4f16 = ConstantVector::getSplat(ElementCount::getFixed(4), f16);
|
||||||
Constant *p0i32 =
|
Constant *p0i32 =
|
||||||
ConstantPointerNull::get(PointerType::get(i32->getType(), 0));
|
ConstantPointerNull::get(PointerType::get(i32->getType(), 0));
|
||||||
|
|
||||||
|
@ -646,8 +646,8 @@ TEST(ConstantsTest, GetSplatValueRoundTrip) {
|
|||||||
Type *Int8Ty = Type::getInt8Ty(Context);
|
Type *Int8Ty = Type::getInt8Ty(Context);
|
||||||
|
|
||||||
for (unsigned Min : {1, 2, 8}) {
|
for (unsigned Min : {1, 2, 8}) {
|
||||||
ElementCount ScalableEC = {Min, true};
|
auto ScalableEC = ElementCount::getScalable(Min);
|
||||||
ElementCount FixedEC = {Min, false};
|
auto FixedEC = ElementCount::getFixed(Min);
|
||||||
|
|
||||||
for (auto EC : {ScalableEC, FixedEC}) {
|
for (auto EC : {ScalableEC, FixedEC}) {
|
||||||
for (auto *Ty : {FloatTy, Int32Ty, Int8Ty}) {
|
for (auto *Ty : {FloatTy, Int32Ty, Int8Ty}) {
|
||||||
|
@ -1445,8 +1445,8 @@ TEST_F(PatternMatchTest, ConstantPredicateType) {
|
|||||||
EXPECT_TRUE(match(CF32Pi, cstfp_pred_ty<always_true_pred<APFloat>>()));
|
EXPECT_TRUE(match(CF32Pi, cstfp_pred_ty<always_true_pred<APFloat>>()));
|
||||||
EXPECT_FALSE(match(CF32Pi, cstfp_pred_ty<always_false_pred<APFloat>>()));
|
EXPECT_FALSE(match(CF32Pi, cstfp_pred_ty<always_false_pred<APFloat>>()));
|
||||||
|
|
||||||
ElementCount FixedEC(4, false);
|
auto FixedEC = ElementCount::getFixed(4);
|
||||||
ElementCount ScalableEC(4, true);
|
auto ScalableEC = ElementCount::getScalable(4);
|
||||||
|
|
||||||
// Vector splat
|
// Vector splat
|
||||||
|
|
||||||
|
@ -59,13 +59,13 @@ TEST(VectorTypesTest, FixedLength) {
|
|||||||
dyn_cast<FixedVectorType>(VectorType::get(Int32Ty, V8Int32Ty));
|
dyn_cast<FixedVectorType>(VectorType::get(Int32Ty, V8Int32Ty));
|
||||||
EXPECT_VTY_EQ(V8Int32Ty, V8Int32Ty2);
|
EXPECT_VTY_EQ(V8Int32Ty, V8Int32Ty2);
|
||||||
|
|
||||||
auto *V8Int16Ty =
|
auto *V8Int16Ty = dyn_cast<FixedVectorType>(
|
||||||
dyn_cast<FixedVectorType>(VectorType::get(Int16Ty, {8, false}));
|
VectorType::get(Int16Ty, ElementCount::getFixed(8)));
|
||||||
ASSERT_NE(nullptr, V8Int16Ty);
|
ASSERT_NE(nullptr, V8Int16Ty);
|
||||||
EXPECT_EQ(V8Int16Ty->getNumElements(), 8U);
|
EXPECT_EQ(V8Int16Ty->getNumElements(), 8U);
|
||||||
EXPECT_EQ(V8Int16Ty->getElementType()->getScalarSizeInBits(), 16U);
|
EXPECT_EQ(V8Int16Ty->getElementType()->getScalarSizeInBits(), 16U);
|
||||||
|
|
||||||
ElementCount EltCnt(4, false);
|
auto EltCnt = ElementCount::getFixed(4);
|
||||||
auto *V4Int64Ty = dyn_cast<FixedVectorType>(VectorType::get(Int64Ty, EltCnt));
|
auto *V4Int64Ty = dyn_cast<FixedVectorType>(VectorType::get(Int64Ty, EltCnt));
|
||||||
ASSERT_NE(nullptr, V4Int64Ty);
|
ASSERT_NE(nullptr, V4Int64Ty);
|
||||||
EXPECT_EQ(V4Int64Ty->getNumElements(), 4U);
|
EXPECT_EQ(V4Int64Ty->getNumElements(), 4U);
|
||||||
@ -153,13 +153,13 @@ TEST(VectorTypesTest, Scalable) {
|
|||||||
dyn_cast<ScalableVectorType>(VectorType::get(Int32Ty, ScV8Int32Ty));
|
dyn_cast<ScalableVectorType>(VectorType::get(Int32Ty, ScV8Int32Ty));
|
||||||
EXPECT_VTY_EQ(ScV8Int32Ty, ScV8Int32Ty2);
|
EXPECT_VTY_EQ(ScV8Int32Ty, ScV8Int32Ty2);
|
||||||
|
|
||||||
auto *ScV8Int16Ty =
|
auto *ScV8Int16Ty = dyn_cast<ScalableVectorType>(
|
||||||
dyn_cast<ScalableVectorType>(VectorType::get(Int16Ty, {8, true}));
|
VectorType::get(Int16Ty, ElementCount::getScalable(8)));
|
||||||
ASSERT_NE(nullptr, ScV8Int16Ty);
|
ASSERT_NE(nullptr, ScV8Int16Ty);
|
||||||
EXPECT_EQ(ScV8Int16Ty->getMinNumElements(), 8U);
|
EXPECT_EQ(ScV8Int16Ty->getMinNumElements(), 8U);
|
||||||
EXPECT_EQ(ScV8Int16Ty->getElementType()->getScalarSizeInBits(), 16U);
|
EXPECT_EQ(ScV8Int16Ty->getElementType()->getScalarSizeInBits(), 16U);
|
||||||
|
|
||||||
ElementCount EltCnt(4, true);
|
auto EltCnt = ElementCount::getScalable(4);
|
||||||
auto *ScV4Int64Ty =
|
auto *ScV4Int64Ty =
|
||||||
dyn_cast<ScalableVectorType>(VectorType::get(Int64Ty, EltCnt));
|
dyn_cast<ScalableVectorType>(VectorType::get(Int64Ty, EltCnt));
|
||||||
ASSERT_NE(nullptr, ScV4Int64Ty);
|
ASSERT_NE(nullptr, ScV4Int64Ty);
|
||||||
@ -225,14 +225,15 @@ TEST(VectorTypesTest, BaseVectorType) {
|
|||||||
Type *Int16Ty = Type::getInt16Ty(Ctx);
|
Type *Int16Ty = Type::getInt16Ty(Ctx);
|
||||||
Type *Int32Ty = Type::getInt32Ty(Ctx);
|
Type *Int32Ty = Type::getInt32Ty(Ctx);
|
||||||
|
|
||||||
std::array<VectorType *, 8> VTys = {VectorType::get(Int16Ty, {4, true}),
|
std::array<VectorType *, 8> VTys = {
|
||||||
VectorType::get(Int16Ty, {4, false}),
|
VectorType::get(Int16Ty, ElementCount::getScalable(4)),
|
||||||
VectorType::get(Int16Ty, {2, true}),
|
VectorType::get(Int16Ty, ElementCount::getFixed(4)),
|
||||||
VectorType::get(Int16Ty, {2, false}),
|
VectorType::get(Int16Ty, ElementCount::getScalable(2)),
|
||||||
VectorType::get(Int32Ty, {4, true}),
|
VectorType::get(Int16Ty, ElementCount::getFixed(2)),
|
||||||
VectorType::get(Int32Ty, {4, false}),
|
VectorType::get(Int32Ty, ElementCount::getScalable(4)),
|
||||||
VectorType::get(Int32Ty, {2, true}),
|
VectorType::get(Int32Ty, ElementCount::getFixed(4)),
|
||||||
VectorType::get(Int32Ty, {2, false})};
|
VectorType::get(Int32Ty, ElementCount::getScalable(2)),
|
||||||
|
VectorType::get(Int32Ty, ElementCount::getFixed(2))};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
The comparison matrix is symmetric, so we only check the upper triangle:
|
The comparison matrix is symmetric, so we only check the upper triangle:
|
||||||
|
@ -57,7 +57,7 @@ TEST(VerifierTest, Freeze) {
|
|||||||
ConstantInt *CI = ConstantInt::get(ITy, 0);
|
ConstantInt *CI = ConstantInt::get(ITy, 0);
|
||||||
|
|
||||||
// Valid type : freeze(<2 x i32>)
|
// Valid type : freeze(<2 x i32>)
|
||||||
Constant *CV = ConstantVector::getSplat({2, false}, CI);
|
Constant *CV = ConstantVector::getSplat(ElementCount::getFixed(2), CI);
|
||||||
FreezeInst *FI_vec = new FreezeInst(CV);
|
FreezeInst *FI_vec = new FreezeInst(CV);
|
||||||
FI_vec->insertBefore(RI);
|
FI_vec->insertBefore(RI);
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user