RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 18:54:02 +01:00
Code Issues Projects Releases Wiki Activity

[VPlan] VPlan version of InterleavedAccessInfo.

Browse Source 
This patch turns InterleaveGroup into a template with the instruction type
being a template parameter. It also adds a VPInterleavedAccessInfo class, which
only contains a mapping from VPInstructions to their respective InterleaveGroup.
As we do not have access to scalar evolution in VPlan, we can re-use
convert InterleavedAccessInfo to VPInterleavedAccess info.


Reviewers: Ayal, mssimpso, hfinkel, dcaballe, rengolin, mkuper, hsaito

Reviewed By: rengolin

Differential Revision: https://reviews.llvm.org/D49489

llvm-svn: 346758
This commit is contained in:
Florian Hahn 2018-11-13 15:58:18 +00:00
parent 995481e224
commit 8a6923f88c
6 changed files with 166 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
include/llvm/Analysis/VectorUtils.h
View File

@ -24,7 +24,7 @@ namespace llvm {
template <typename T> class ArrayRef;
class DemandedBits;
class GetElementPtrInst;
class InterleaveGroup;
template <typename InstTy> class InterleaveGroup;
class Loop;
class ScalarEvolution;
class TargetTransformInfo;
@ -138,7 +138,7 @@ Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
/// create[*]Mask() utilities which create a shuffle mask (mask that
/// consists of indices).
Constant *createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
const InterleaveGroup &Group);
const InterleaveGroup<Instruction> &Group);
/// Create a mask with replicated elements.
///
@ -233,9 +233,12 @@ Value *concatenateVectors(IRBuilder<> &Builder, ArrayRef<Value *> Vecs);
///
/// Note: the interleaved load group could have gaps (missing members), but
/// the interleaved store group doesn't allow gaps.
class InterleaveGroup {
template <typename InstTy> class InterleaveGroup {
public:
InterleaveGroup(Instruction *Instr, int Stride, unsigned Align)
InterleaveGroup(unsigned Factor, bool Reverse, unsigned Align)
: Factor(Factor), Reverse(Reverse), Align(Align), InsertPos(nullptr) {}
InterleaveGroup(InstTy *Instr, int Stride, unsigned Align)
: Align(Align), InsertPos(Instr) {
assert(Align && "The alignment should be non-zero");
@ -256,7 +259,7 @@ public:
/// negative if it is the new leader.
///
/// \returns false if the instruction doesn't belong to the group.
bool insertMember(Instruction *Instr, int Index, unsigned NewAlign) {
bool insertMember(InstTy *Instr, int Index, unsigned NewAlign) {
assert(NewAlign && "The new member's alignment should be non-zero");
int Key = Index + SmallestKey;
@ -288,7 +291,7 @@ public:
/// Get the member with the given index \p Index
///
/// \returns nullptr if contains no such member.
Instruction *getMember(unsigned Index) const {
InstTy *getMember(unsigned Index) const {
int Key = SmallestKey + Index;
auto Member = Members.find(Key);
if (Member == Members.end())
@ -299,16 +302,17 @@ public:
/// Get the index for the given member. Unlike the key in the member
/// map, the index starts from 0.
unsigned getIndex(Instruction *Instr) const {
for (auto I : Members)
unsigned getIndex(const InstTy *Instr) const {
for (auto I : Members) {
if (I.second == Instr)
return I.first - SmallestKey;
}
llvm_unreachable("InterleaveGroup contains no such member");
}
Instruction *getInsertPos() const { return InsertPos; }
void setInsertPos(Instruction *Inst) { InsertPos = Inst; }
InstTy *getInsertPos() const { return InsertPos; }
void setInsertPos(InstTy *Inst) { InsertPos = Inst; }
/// Add metadata (e.g. alias info) from the instructions in this group to \p
/// NewInst.
@ -316,12 +320,7 @@ public:
/// FIXME: this function currently does not add noalias metadata a'la
/// addNewMedata. To do that we need to compute the intersection of the
/// noalias info from all members.
void addMetadata(Instruction *NewInst) const {
SmallVector<Value *, 4> VL;
std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
[](std::pair<int, Instruction *> p) { return p.second; });
propagateMetadata(NewInst, VL);
}
void addMetadata(InstTy *NewInst) const;
/// Returns true if this Group requires a scalar iteration to handle gaps.
bool requiresScalarEpilogue() const {
@ -344,7 +343,7 @@ private:
unsigned Factor; // Interleave Factor.
bool Reverse;
unsigned Align;
DenseMap<int, Instruction *> Members;
DenseMap<int, InstTy *> Members;
int SmallestKey = 0;
int LargestKey = 0;
@ -359,7 +358,7 @@ private:
// store i32 %even
// %odd = add i32 // Def of %odd
// store i32 %odd // Insert Position
Instruction *InsertPos;
InstTy *InsertPos;
};
/// Drive the analysis of interleaved memory accesses in the loop.
@ -390,7 +389,7 @@ public:
/// formation for predicated accesses, we may be able to relax this limitation
/// in the future once we handle more complicated blocks.
void reset() {
SmallPtrSet<InterleaveGroup *, 4> DelSet;
SmallPtrSet<InterleaveGroup<Instruction> *, 4> DelSet;
// Avoid releasing a pointer twice.
for (auto &I : InterleaveGroupMap)
DelSet.insert(I.second);
@ -409,11 +408,16 @@ public:
/// Get the interleave group that \p Instr belongs to.
///
/// \returns nullptr if doesn't have such group.
InterleaveGroup *getInterleaveGroup(Instruction *Instr) const {
auto Group = InterleaveGroupMap.find(Instr);
if (Group == InterleaveGroupMap.end())
return nullptr;
return Group->second;
InterleaveGroup<Instruction> *
getInterleaveGroup(const Instruction *Instr) const {
if (InterleaveGroupMap.count(Instr))
return InterleaveGroupMap.find(Instr)->second;
return nullptr;
}
iterator_range<SmallPtrSetIterator<llvm::InterleaveGroup<Instruction> *>>
getInterleaveGroups() {
return make_range(InterleaveGroups.begin(), InterleaveGroups.end());
}
/// Returns true if an interleaved group that may access memory
@ -443,7 +447,9 @@ private:
bool RequiresScalarEpilogue = false;
/// Holds the relationships between the members and the interleave group.
DenseMap<Instruction *, InterleaveGroup *> InterleaveGroupMap;
DenseMap<Instruction *, InterleaveGroup<Instruction> *> InterleaveGroupMap;
SmallPtrSet<InterleaveGroup<Instruction> *, 4> InterleaveGroups;
/// Holds dependences among the memory accesses in the loop. It maps a source
/// access to a set of dependent sink accesses.
@ -476,19 +482,23 @@ private:
/// stride \p Stride and alignment \p Align.
///
/// \returns the newly created interleave group.
InterleaveGroup *createInterleaveGroup(Instruction *Instr, int Stride,
unsigned Align) {
assert(!isInterleaved(Instr) && "Already in an interleaved access group");
InterleaveGroupMap[Instr] = new InterleaveGroup(Instr, Stride, Align);
InterleaveGroup<Instruction> *
createInterleaveGroup(Instruction *Instr, int Stride, unsigned Align) {
assert(!InterleaveGroupMap.count(Instr) &&
"Already in an interleaved access group");
InterleaveGroupMap[Instr] =
new InterleaveGroup<Instruction>(Instr, Stride, Align);
InterleaveGroups.insert(InterleaveGroupMap[Instr]);
return InterleaveGroupMap[Instr];
}
/// Release the group and remove all the relationships.
void releaseGroup(InterleaveGroup *Group) {
void releaseGroup(InterleaveGroup<Instruction> *Group) {
for (unsigned i = 0; i < Group->getFactor(); i++)
if (Instruction *Member = Group->getMember(i))
InterleaveGroupMap.erase(Member);
InterleaveGroups.erase(Group);
delete Group;
}

34
lib/Analysis/VectorUtils.cpp
View File

@ -505,8 +505,9 @@ Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
return Inst;
}
Constant *llvm::createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
const InterleaveGroup &Group) {
Constant *
llvm::createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
const InterleaveGroup<Instruction> &Group) {
// All 1's means mask is not needed.
if (Group.getNumMembers() == Group.getFactor())
return nullptr;
@ -720,9 +721,9 @@ void InterleavedAccessInfo::analyzeInterleaving(
collectDependences();
// Holds all interleaved store groups temporarily.
SmallSetVector<InterleaveGroup *, 4> StoreGroups;
SmallSetVector<InterleaveGroup<Instruction> *, 4> StoreGroups;
// Holds all interleaved load groups temporarily.
SmallSetVector<InterleaveGroup *, 4> LoadGroups;
SmallSetVector<InterleaveGroup<Instruction> *, 4> LoadGroups;
// Search in bottom-up program order for pairs of accesses (A and B) that can
// form interleaved load or store groups. In the algorithm below, access A
@ -744,7 +745,7 @@ void InterleavedAccessInfo::analyzeInterleaving(
// Initialize a group for B if it has an allowable stride. Even if we don't
// create a group for B, we continue with the bottom-up algorithm to ensure
// we don't break any of B's dependences.
InterleaveGroup *Group = nullptr;
InterleaveGroup<Instruction> *Group = nullptr;
if (isStrided(DesB.Stride) &&
(!isPredicated(B->getParent()) || EnablePredicatedInterleavedMemAccesses)) {
Group = getInterleaveGroup(B);
@ -789,7 +790,7 @@ void InterleavedAccessInfo::analyzeInterleaving(
// illegal code motion. A will then be free to form another group with
// instructions that precede it.
if (isInterleaved(A)) {
InterleaveGroup *StoreGroup = getInterleaveGroup(A);
InterleaveGroup<Instruction> *StoreGroup = getInterleaveGroup(A);
StoreGroups.remove(StoreGroup);
releaseGroup(StoreGroup);
}
@ -868,7 +869,7 @@ void InterleavedAccessInfo::analyzeInterleaving(
} // Iteration over B accesses.
// Remove interleaved store groups with gaps.
for (InterleaveGroup *Group : StoreGroups)
for (auto *Group : StoreGroups)
if (Group->getNumMembers() != Group->getFactor()) {
LLVM_DEBUG(
dbgs() << "LV: Invalidate candidate interleaved store group due "
@ -889,7 +890,7 @@ void InterleavedAccessInfo::analyzeInterleaving(
// This means that we can forcefully peel the loop in order to only have to
// check the first pointer for no-wrap. When we'll change to use Assume=true
// we'll only need at most one runtime check per interleaved group.
for (InterleaveGroup *Group : LoadGroups) {
for (auto *Group : LoadGroups) {
// Case 1: A full group. Can Skip the checks; For full groups, if the wide
// load would wrap around the address space we would do a memory access at
// nullptr even without the transformation.
@ -947,9 +948,9 @@ void InterleavedAccessInfo::invalidateGroupsRequiringScalarEpilogue() {
return;
// Avoid releasing a Group twice.
SmallPtrSet<InterleaveGroup *, 4> DelSet;
SmallPtrSet<InterleaveGroup<Instruction> *, 4> DelSet;
for (auto &I : InterleaveGroupMap) {
InterleaveGroup *Group = I.second;
InterleaveGroup<Instruction> *Group = I.second;
if (Group->requiresScalarEpilogue())
DelSet.insert(Group);
}
@ -964,3 +965,16 @@ void InterleavedAccessInfo::invalidateGroupsRequiringScalarEpilogue() {
RequiresScalarEpilogue = false;
}
template <>
void InterleaveGroup<Instruction>::addMetadata(Instruction *NewInst) const {
SmallVector<Value *, 4> VL;
std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
[](std::pair<int, Instruction *> p) { return p.second; });
propagateMetadata(NewInst, VL);
}
template <typename InstT>
void InterleaveGroup<InstT>::addMetadata(InstT *NewInst) const {
llvm_unreachable("addMetadata can only be used for Instruction");
}

13
lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -976,7 +976,7 @@ public:
/// Save vectorization decision \p W and \p Cost taken by the cost model for
/// interleaving group \p Grp and vector width \p VF.
void setWideningDecision(const InterleaveGroup *Grp, unsigned VF,
void setWideningDecision(const InterleaveGroup<Instruction> *Grp, unsigned VF,
InstWidening W, unsigned Cost) {
assert(VF >= 2 && "Expected VF >=2");
/// Broadcast this decicion to all instructions inside the group.
@ -1131,7 +1131,8 @@ public:
}
/// Get the interleaved access group that \p Instr belongs to.
const InterleaveGroup *getInterleavedAccessGroup(Instruction *Instr) {
const InterleaveGroup<Instruction> *
getInterleavedAccessGroup(Instruction *Instr) {
return InterleaveInfo.getInterleaveGroup(Instr);
}
@ -1994,7 +1995,8 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,
VectorParts *BlockInMask) {
const InterleaveGroup *Group = Cost->getInterleavedAccessGroup(Instr);
const InterleaveGroup<Instruction> *Group =
Cost->getInterleavedAccessGroup(Instr);
assert(Group && "Fail to get an interleaved access group.");
// Skip if current instruction is not the insert position.
@ -6377,7 +6379,7 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
VPInterleaveRecipe *VPRecipeBuilder::tryToInterleaveMemory(Instruction *I,
VFRange &Range,
VPlanPtr &Plan) {
const InterleaveGroup *IG = CM.getInterleavedAccessGroup(I);
const InterleaveGroup<Instruction> *IG = CM.getInterleavedAccessGroup(I);
if (!IG)
return nullptr;
@ -6793,7 +6795,8 @@ LoopVectorizationPlanner::buildVPlanWithVPRecipes(
// I is a member of an InterleaveGroup for Range.Start. If it's an adjunct
// member of the IG, do not construct any Recipe for it.
const InterleaveGroup *IG = CM.getInterleavedAccessGroup(Instr);
const InterleaveGroup<Instruction> *IG =
CM.getInterleavedAccessGroup(Instr);
if (IG && Instr != IG->getInsertPos() &&
Range.Start >= 2 && // Query is illegal for VF == 1
CM.getWideningDecision(Instr, Range.Start) ==

45
lib/Transforms/Vectorize/VPlan.cpp
View File

@ -680,3 +680,48 @@ void VPWidenMemoryInstructionRecipe::print(raw_ostream &O,
}
template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
Old2NewTy &Old2New,
InterleavedAccessInfo &IAI) {
ReversePostOrderTraversal<VPBlockBase *> RPOT(Region->getEntry());
for (VPBlockBase *Base : RPOT) {
visitBlock(Base, Old2New, IAI);
}
}
void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
InterleavedAccessInfo &IAI) {
if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
for (VPRecipeBase &VPI : *VPBB) {
assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions");
auto *VPInst = cast<VPInstruction>(&VPI);
auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
auto *IG = IAI.getInterleaveGroup(Inst);
if (!IG)
continue;
auto NewIGIter = Old2New.find(IG);
if (NewIGIter == Old2New.end())
Old2New[IG] = new InterleaveGroup<VPInstruction>(
IG->getFactor(), IG->isReverse(), IG->getAlignment());
if (Inst == IG->getInsertPos())
Old2New[IG]->setInsertPos(VPInst);
InterleaveGroupMap[VPInst] = Old2New[IG];
InterleaveGroupMap[VPInst]->insertMember(
VPInst, IG->getIndex(Inst),
IG->isReverse() ? (-1) * int(IG->getFactor()) : IG->getFactor());
}
} else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
visitRegion(Region, Old2New, IAI);
else
llvm_unreachable("Unsupported kind of VPBlock.");
}
VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
InterleavedAccessInfo &IAI) {
Old2NewTy Old2New;
visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
}

52
lib/Transforms/Vectorize/VPlan.h
View File

@ -38,6 +38,7 @@
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/IRBuilder.h"
#include <algorithm>
#include <cassert>
@ -52,7 +53,8 @@ class LoopVectorizationCostModel;
class BasicBlock;
class DominatorTree;
class InnerLoopVectorizer;
class InterleaveGroup;
template <class T> class InterleaveGroup;
class LoopInfo;
class raw_ostream;
class Value;
class VPBasicBlock;
@ -771,11 +773,11 @@ public:
/// or stores into one wide load/store and shuffles.
class VPInterleaveRecipe : public VPRecipeBase {
private:
const InterleaveGroup *IG;
const InterleaveGroup<Instruction> *IG;
std::unique_ptr<VPUser> User;
public:
VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Mask)
VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Mask)
: VPRecipeBase(VPInterleaveSC), IG(IG) {
if (Mask) // Create a VPInstruction to register as a user of the mask.
User.reset(new VPUser({Mask}));
@ -793,7 +795,7 @@ public:
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent) const override;
const InterleaveGroup *getInterleaveGroup() { return IG; }
const InterleaveGroup<Instruction> *getInterleaveGroup() { return IG; }
};
/// VPReplicateRecipe replicates a given instruction producing multiple scalar
@ -1464,6 +1466,48 @@ public:
}
};
class VPInterleavedAccessInfo {
private:
DenseMap<VPInstruction *, InterleaveGroup<VPInstruction> *>
InterleaveGroupMap;
/// Type for mapping of instruction based interleave groups to VPInstruction
/// interleave groups
using Old2NewTy = DenseMap<InterleaveGroup<Instruction> *,
InterleaveGroup<VPInstruction> *>;
/// Recursively \p Region and populate VPlan based interleave groups based on
/// \p IAI.
void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
InterleavedAccessInfo &IAI);
/// Recursively traverse \p Block and populate VPlan based interleave groups
/// based on \p IAI.
void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
InterleavedAccessInfo &IAI);
public:
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI);
~VPInterleavedAccessInfo() {
SmallPtrSet<InterleaveGroup<VPInstruction> *, 4> DelSet;
// Avoid releasing a pointer twice.
for (auto &I : InterleaveGroupMap)
DelSet.insert(I.second);
for (auto *Ptr : DelSet)
delete Ptr;
}
/// Get the interleave group that \p Instr belongs to.
///
/// \returns nullptr if doesn't have such group.
InterleaveGroup<VPInstruction> *
getInterleaveGroup(VPInstruction *Instr) const {
if (InterleaveGroupMap.count(Instr))
return InterleaveGroupMap.find(Instr)->second;
return nullptr;
}
};
} // end namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H

1
lib/Transforms/Vectorize/VPlanValue.h
View File

@ -40,6 +40,7 @@ class VPValue {
friend class VPBuilder;
friend class VPlanHCFGTransforms;
friend class VPBasicBlock;
friend class VPInterleavedAccessInfo;
private:
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).