RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 10:42:39 +01:00
Code Issues Projects Releases Wiki Activity

BranchProb: modify the definition of an edge in BranchProbabilityInfo to handle

Browse Source 
the case of multiple edges from one block to another.

A simple example is a switch statement with multiple values to the same
destination. The definition of an edge is modified from a pair of blocks to
a pair of PredBlock and an index into the successors.

Also set the weight correctly when building SelectionDAG from LLVM IR,
especially when converting a Switch.
IntegersSubsetMapping is updated to calculate the weight for each cluster.

llvm-svn: 162572
This commit is contained in:
Manman Ren 2012-08-24 18:14:27 +00:00
parent 746c77c658
commit 6342812033
8 changed files with 239 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
include/llvm/Analysis/BranchProbabilityInfo.h
View File

@ -28,11 +28,14 @@ class raw_ostream;
///
/// This is a function analysis pass which provides information on the relative
/// probabilities of each "edge" in the function's CFG where such an edge is
/// defined by a pair of basic blocks. The probability for a given block and
/// a successor block are always relative to the probabilities of the other
/// successor blocks. Another way of looking at it is that the probabilities
/// for a given block B and each of its successors should sum to exactly
/// one (100%).
/// defined by a pair (PredBlock and an index in the successors). The
/// probability of an edge from one block is always relative to the
/// probabilities of other edges from the block. The probabilites of all edges
/// from a block sum to exactly one (100%).
/// We use a pair (PredBlock and an index in the successors) to uniquely
/// identify an edge, since we can have multiple edges from Src to Dst.
/// As an example, we can have a switch which jumps to Dst with value 0 and
/// value 10.
class BranchProbabilityInfo : public FunctionPass {
public:
static char ID;
@ -51,6 +54,12 @@ public:
/// (0%) and one (100%) of this edge executing, relative to other edges
/// leaving the 'Src' block. The returned probability is never zero, and can
/// only be one if the source block has only one successor.
BranchProbability getEdgeProbability(const BasicBlock *Src,
unsigned IndexInSuccessors) const;
/// \brief Get the probability of going from Src to Dst.
///
/// It returns the sum of all probabilities for edges from Src to Dst.
BranchProbability getEdgeProbability(const BasicBlock *Src,
const BasicBlock *Dst) const;
@ -74,25 +83,34 @@ public:
raw_ostream &printEdgeProbability(raw_ostream &OS, const BasicBlock *Src,
const BasicBlock *Dst) const;
/// \brief Get the raw edge weight calculated for the block pair.
/// \brief Get the raw edge weight calculated for the edge.
///
/// This returns the raw edge weight. It is guaranteed to fall between 1 and
/// UINT32_MAX. Note that the raw edge weight is not meaningful in isolation.
/// This interface should be very carefully, and primarily by routines that
/// are updating the analysis by later calling setEdgeWeight.
uint32_t getEdgeWeight(const BasicBlock *Src,
unsigned IndexInSuccessors) const;
/// \brief Get the raw edge weight calculated for the block pair.
///
/// This returns the sum of all raw edge weights from Src to Dst.
/// It is guaranteed to fall between 1 and UINT32_MAX.
uint32_t getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const;
/// \brief Set the raw edge weight for the block pair.
/// \brief Set the raw edge weight for a given edge.
///
/// This allows a pass to explicitly set the edge weight for a block. It can
/// This allows a pass to explicitly set the edge weight for an edge. It can
/// be used when updating the CFG to update and preserve the branch
/// probability information. Read the implementation of how these edge
/// weights are calculated carefully before using!
void setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst,
void setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors,
uint32_t Weight);
private:
typedef std::pair<const BasicBlock *, const BasicBlock *> Edge;
// Since we allow duplicate edges from one basic block to another, we use
// a pair (PredBlock and an index in the successors) to specify an edge.
typedef std::pair<const BasicBlock *, unsigned> Edge;
// Default weight value. Used when we don't have information about the edge.
// TODO: DEFAULT_WEIGHT makes sense during static predication, when none of

18
include/llvm/Support/IntegersSubsetMapping.h
View File

@ -42,6 +42,7 @@ public:
struct RangeEx : public RangeTy {
RangeEx() : Weight(1) {}
RangeEx(const RangeTy &R) : RangeTy(R), Weight(1) {}
RangeEx(const RangeTy &R, unsigned W) : RangeTy(R), Weight(W) {}
RangeEx(const IntTy &C) : RangeTy(C), Weight(1) {}
RangeEx(const IntTy &L, const IntTy &H) : RangeTy(L, H), Weight(1) {}
RangeEx(const IntTy &L, const IntTy &H, unsigned W) :
@ -316,13 +317,13 @@ public:
Items.clear();
const IntTy *Low = &OldItems.begin()->first.getLow();
const IntTy *High = &OldItems.begin()->first.getHigh();
unsigned Weight = 1;
unsigned Weight = OldItems.begin()->first.Weight;
SuccessorClass *Successor = OldItems.begin()->second;
for (CaseItemIt j = OldItems.begin(), i = j++, e = OldItems.end();
j != e; i = j++) {
if (isJoinable(i, j)) {
const IntTy *CurHigh = &j->first.getHigh();
++Weight;
Weight += j->first.Weight;
if (*CurHigh > *High)
High = CurHigh;
} else {
@ -330,7 +331,7 @@ public:
add(R, Successor);
Low = &j->first.getLow();
High = &j->first.getHigh();
Weight = 1;
Weight = j->first.Weight;
Successor = j->second;
}
}
@ -362,10 +363,17 @@ public:
/// Adds all ranges and values from given ranges set to the current
/// mapping.
void add(const IntegersSubsetTy &CRS, SuccessorClass *S = 0) {
void add(const IntegersSubsetTy &CRS, SuccessorClass *S = 0,
unsigned Weight = 0) {
unsigned ItemWeight = 1;
if (Weight)
// Weight is associated with CRS, for now we perform a division to
// get the weight for each item.
ItemWeight = Weight / CRS.getNumItems();
for (unsigned i = 0, e = CRS.getNumItems(); i < e; ++i) {
RangeTy R = CRS.getItem(i);
add(R, S);
RangeEx REx(R, ItemWeight);
add(REx, S);
}
}

138
lib/Analysis/BranchProbabilityInfo.cpp
View File

@ -115,14 +115,14 @@ bool BranchProbabilityInfo::calcUnreachableHeuristics(BasicBlock *BB) {
return false;
}
SmallPtrSet<BasicBlock *, 4> UnreachableEdges;
SmallPtrSet<BasicBlock *, 4> ReachableEdges;
SmallVector<unsigned, 4> UnreachableEdges;
SmallVector<unsigned, 4> ReachableEdges;
for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
if (PostDominatedByUnreachable.count(*I))
UnreachableEdges.insert(*I);
UnreachableEdges.push_back(I.getSuccessorIndex());
else
ReachableEdges.insert(*I);
ReachableEdges.push_back(I.getSuccessorIndex());
}
// If all successors are in the set of blocks post-dominated by unreachable,
@ -136,18 +136,19 @@ bool BranchProbabilityInfo::calcUnreachableHeuristics(BasicBlock *BB) {
return false;
uint32_t UnreachableWeight =
std::max(UR_TAKEN_WEIGHT / UnreachableEdges.size(), MIN_WEIGHT);
for (SmallPtrSet<BasicBlock *, 4>::iterator I = UnreachableEdges.begin(),
E = UnreachableEdges.end();
std::max(UR_TAKEN_WEIGHT / (unsigned)UnreachableEdges.size(), MIN_WEIGHT);
for (SmallVector<unsigned, 4>::iterator I = UnreachableEdges.begin(),
E = UnreachableEdges.end();
I != E; ++I)
setEdgeWeight(BB, *I, UnreachableWeight);
if (ReachableEdges.empty())
return true;
uint32_t ReachableWeight =
std::max(UR_NONTAKEN_WEIGHT / ReachableEdges.size(), NORMAL_WEIGHT);
for (SmallPtrSet<BasicBlock *, 4>::iterator I = ReachableEdges.begin(),
E = ReachableEdges.end();
std::max(UR_NONTAKEN_WEIGHT / (unsigned)ReachableEdges.size(),
NORMAL_WEIGHT);
for (SmallVector<unsigned, 4>::iterator I = ReachableEdges.begin(),
E = ReachableEdges.end();
I != E; ++I)
setEdgeWeight(BB, *I, ReachableWeight);
@ -187,7 +188,7 @@ bool BranchProbabilityInfo::calcMetadataWeights(BasicBlock *BB) {
}
assert(Weights.size() == TI->getNumSuccessors() && "Checked above");
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
setEdgeWeight(BB, TI->getSuccessor(i), Weights[i]);
setEdgeWeight(BB, i, Weights[i]);
return true;
}
@ -211,19 +212,17 @@ bool BranchProbabilityInfo::calcPointerHeuristics(BasicBlock *BB) {
assert(CI->getOperand(1)->getType()->isPointerTy());
BasicBlock *Taken = BI->getSuccessor(0);
BasicBlock *NonTaken = BI->getSuccessor(1);
// p != 0 -> isProb = true
// p == 0 -> isProb = false
// p != q -> isProb = true
// p == q -> isProb = false;
unsigned TakenIdx = 0, NonTakenIdx = 1;
bool isProb = CI->getPredicate() == ICmpInst::ICMP_NE;
if (!isProb)
std::swap(Taken, NonTaken);
std::swap(TakenIdx, NonTakenIdx);
setEdgeWeight(BB, Taken, PH_TAKEN_WEIGHT);
setEdgeWeight(BB, NonTaken, PH_NONTAKEN_WEIGHT);
setEdgeWeight(BB, TakenIdx, PH_TAKEN_WEIGHT);
setEdgeWeight(BB, NonTakenIdx, PH_NONTAKEN_WEIGHT);
return true;
}
@ -234,17 +233,17 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
if (!L)
return false;
SmallPtrSet<BasicBlock *, 8> BackEdges;
SmallPtrSet<BasicBlock *, 8> ExitingEdges;
SmallPtrSet<BasicBlock *, 8> InEdges; // Edges from header to the loop.
SmallVector<unsigned, 8> BackEdges;
SmallVector<unsigned, 8> ExitingEdges;
SmallVector<unsigned, 8> InEdges; // Edges from header to the loop.
for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
if (!L->contains(*I))
ExitingEdges.insert(*I);
ExitingEdges.push_back(I.getSuccessorIndex());
else if (L->getHeader() == *I)
BackEdges.insert(*I);
BackEdges.push_back(I.getSuccessorIndex());
else
InEdges.insert(*I);
InEdges.push_back(I.getSuccessorIndex());
}
if (uint32_t numBackEdges = BackEdges.size()) {
@ -252,10 +251,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
if (backWeight < NORMAL_WEIGHT)
backWeight = NORMAL_WEIGHT;
for (SmallPtrSet<BasicBlock *, 8>::iterator EI = BackEdges.begin(),
for (SmallVector<unsigned, 8>::iterator EI = BackEdges.begin(),
EE = BackEdges.end(); EI != EE; ++EI) {
BasicBlock *Back = *EI;
setEdgeWeight(BB, Back, backWeight);
setEdgeWeight(BB, *EI, backWeight);
}
}
@ -264,10 +262,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
if (inWeight < NORMAL_WEIGHT)
inWeight = NORMAL_WEIGHT;
for (SmallPtrSet<BasicBlock *, 8>::iterator EI = InEdges.begin(),
for (SmallVector<unsigned, 8>::iterator EI = InEdges.begin(),
EE = InEdges.end(); EI != EE; ++EI) {
BasicBlock *Back = *EI;
setEdgeWeight(BB, Back, inWeight);
setEdgeWeight(BB, *EI, inWeight);
}
}
@ -276,10 +273,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
if (exitWeight < MIN_WEIGHT)
exitWeight = MIN_WEIGHT;
for (SmallPtrSet<BasicBlock *, 8>::iterator EI = ExitingEdges.begin(),
for (SmallVector<unsigned, 8>::iterator EI = ExitingEdges.begin(),
EE = ExitingEdges.end(); EI != EE; ++EI) {
BasicBlock *Exiting = *EI;
setEdgeWeight(BB, Exiting, exitWeight);
setEdgeWeight(BB, *EI, exitWeight);
}
}
@ -335,14 +331,13 @@ bool BranchProbabilityInfo::calcZeroHeuristics(BasicBlock *BB) {
return false;
}
BasicBlock *Taken = BI->getSuccessor(0);
BasicBlock *NonTaken = BI->getSuccessor(1);
unsigned TakenIdx = 0, NonTakenIdx = 1;
if (!isProb)
std::swap(Taken, NonTaken);
std::swap(TakenIdx, NonTakenIdx);
setEdgeWeight(BB, Taken, ZH_TAKEN_WEIGHT);
setEdgeWeight(BB, NonTaken, ZH_NONTAKEN_WEIGHT);
setEdgeWeight(BB, TakenIdx, ZH_TAKEN_WEIGHT);
setEdgeWeight(BB, NonTakenIdx, ZH_NONTAKEN_WEIGHT);
return true;
}
@ -372,14 +367,13 @@ bool BranchProbabilityInfo::calcFloatingPointHeuristics(BasicBlock *BB) {
return false;
}
BasicBlock *Taken = BI->getSuccessor(0);
BasicBlock *NonTaken = BI->getSuccessor(1);
unsigned TakenIdx = 0, NonTakenIdx = 1;
if (!isProb)
std::swap(Taken, NonTaken);
std::swap(TakenIdx, NonTakenIdx);
setEdgeWeight(BB, Taken, FPH_TAKEN_WEIGHT);
setEdgeWeight(BB, NonTaken, FPH_NONTAKEN_WEIGHT);
setEdgeWeight(BB, TakenIdx, FPH_TAKEN_WEIGHT);
setEdgeWeight(BB, NonTakenIdx, FPH_NONTAKEN_WEIGHT);
return true;
}
@ -389,11 +383,8 @@ bool BranchProbabilityInfo::calcInvokeHeuristics(BasicBlock *BB) {
if (!II)
return false;
BasicBlock *Normal = II->getNormalDest();
BasicBlock *Unwind = II->getUnwindDest();
setEdgeWeight(BB, Normal, IH_TAKEN_WEIGHT);
setEdgeWeight(BB, Unwind, IH_NONTAKEN_WEIGHT);
setEdgeWeight(BB, 0/*Index for Normal*/, IH_TAKEN_WEIGHT);
setEdgeWeight(BB, 1/*Index for Unwind*/, IH_NONTAKEN_WEIGHT);
return true;
}
@ -450,8 +441,7 @@ uint32_t BranchProbabilityInfo::getSumForBlock(const BasicBlock *BB) const {
uint32_t Sum = 0;
for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
const BasicBlock *Succ = *I;
uint32_t Weight = getEdgeWeight(BB, Succ);
uint32_t Weight = getEdgeWeight(BB, I.getSuccessorIndex());
uint32_t PrevSum = Sum;
Sum += Weight;
@ -494,11 +484,13 @@ BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const {
return 0;
}
// Return edge's weight. If can't find it, return DEFAULT_WEIGHT value.
/// Get the raw edge weight for the edge. If can't find it, return
/// DEFAULT_WEIGHT value. Here an edge is specified using PredBlock and an index
/// to the successors.
uint32_t BranchProbabilityInfo::
getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
Edge E(Src, Dst);
DenseMap<Edge, uint32_t>::const_iterator I = Weights.find(E);
getEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors) const {
DenseMap<Edge, uint32_t>::const_iterator I =
Weights.find(std::make_pair(Src, IndexInSuccessors));
if (I != Weights.end())
return I->second;
@ -506,15 +498,43 @@ getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
return DEFAULT_WEIGHT;
}
void BranchProbabilityInfo::
setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst, uint32_t Weight) {
Weights[std::make_pair(Src, Dst)] = Weight;
DEBUG(dbgs() << "set edge " << Src->getName() << " -> "
<< Dst->getName() << " weight to " << Weight
<< (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n"));
/// Get the raw edge weight calculated for the block pair. This returns the sum
/// of all raw edge weights from Src to Dst.
uint32_t BranchProbabilityInfo::
getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
uint32_t Weight = 0;
DenseMap<Edge, uint32_t>::const_iterator MapI;
for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I)
if (*I == Dst) {
MapI = Weights.find(std::make_pair(Src, I.getSuccessorIndex()));
if (MapI != Weights.end())
Weight += MapI->second;
}
return (Weight == 0) ? DEFAULT_WEIGHT : Weight;
}
/// Set the edge weight for a given edge specified by PredBlock and an index
/// to the successors.
void BranchProbabilityInfo::
setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors,
uint32_t Weight) {
Weights[std::make_pair(Src, IndexInSuccessors)] = Weight;
DEBUG(dbgs() << "set edge " << Src->getName() << " -> "
<< IndexInSuccessors << " successor weight to "
<< Weight << "\n");
}
/// Get an edge's probability, relative to other out-edges from Src.
BranchProbability BranchProbabilityInfo::
getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const {
uint32_t N = getEdgeWeight(Src, IndexInSuccessors);
uint32_t D = getSumForBlock(Src);
return BranchProbability(N, D);
}
/// Get the probability of going from Src to Dst. It returns the sum of all
/// probabilities for edges from Src to Dst.
BranchProbability BranchProbabilityInfo::
getEdgeProbability(const BasicBlock *Src, const BasicBlock *Dst) const {

68
lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
View File

@ -1766,6 +1766,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
/// visitBitTestCase - this function produces one "bit test"
void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
MachineBasicBlock* NextMBB,
uint32_t BranchWeightToNext,
unsigned Reg,
BitTestCase &B,
MachineBasicBlock *SwitchBB) {
@ -1803,8 +1804,10 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
ISD::SETNE);
}
addSuccessorWithWeight(SwitchBB, B.TargetBB);
addSuccessorWithWeight(SwitchBB, NextMBB);
// The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
// The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurDebugLoc(),
MVT::Other, getControlRoot(),
@ -1927,6 +1930,7 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
if (++BBI != FuncInfo.MF->end())
NextBlock = BBI;
BranchProbabilityInfo *BPI = FuncInfo.BPI;
// If any two of the cases has the same destination, and if one value
// is the same as the other, but has one bit unset that the other has set,
// use bit manipulation to do two compares at once. For example:
@ -1960,8 +1964,12 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
ISD::SETEQ);
// Update successor info.
addSuccessorWithWeight(SwitchBB, Small.BB);
addSuccessorWithWeight(SwitchBB, Default);
// Both Small and Big will jump to Small.BB, so we sum up the weights.
addSuccessorWithWeight(SwitchBB, Small.BB,
Small.ExtraWeight + Big.ExtraWeight);
addSuccessorWithWeight(SwitchBB, Default,
// The default destination is the first successor in IR.
BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
// Insert the true branch.
SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
@ -1979,14 +1987,13 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
}
// Order cases by weight so the most likely case will be checked first.
BranchProbabilityInfo *BPI = FuncInfo.BPI;
uint32_t UnhandledWeights = 0;
if (BPI) {
for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
uint32_t IWeight = BPI->getEdgeWeight(SwitchBB->getBasicBlock(),
I->BB->getBasicBlock());
uint32_t IWeight = I->ExtraWeight;
UnhandledWeights += IWeight;
for (CaseItr J = CR.Range.first; J < I; ++J) {
uint32_t JWeight = BPI->getEdgeWeight(SwitchBB->getBasicBlock(),
J->BB->getBasicBlock());
uint32_t JWeight = J->ExtraWeight;
if (IWeight > JWeight)
std::swap(*I, *J);
}
@ -2035,10 +2042,12 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
LHS = I->Low; MHS = SV; RHS = I->High;
}
uint32_t ExtraWeight = I->ExtraWeight;
// The false weight should be sum of all un-handled cases.
UnhandledWeights -= I->ExtraWeight;
CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
/* me */ CurBlock,
/* trueweight */ ExtraWeight / 2, /* falseweight */ ExtraWeight / 2);
/* trueweight */ I->ExtraWeight,
/* falseweight */ UnhandledWeights);
// If emitting the first comparison, just call visitSwitchCase to emit the
// code into the current block. Otherwise, push the CaseBlock onto the
@ -2138,13 +2147,28 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
}
}
// Calculate weight for each unique destination in CR.
DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
if (FuncInfo.BPI)
for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
DestWeights.find(I->BB);
if (Itr != DestWeights.end())
Itr->second += I->ExtraWeight;
else
DestWeights[I->BB] = I->ExtraWeight;
}
// Update successor info. Add one edge to each unique successor.
BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
E = DestBBs.end(); I != E; ++I) {
if (!SuccsHandled[(*I)->getNumber()]) {
SuccsHandled[(*I)->getNumber()] = true;
addSuccessorWithWeight(JumpTableBB, *I);
DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
DestWeights.find(*I);
addSuccessorWithWeight(JumpTableBB, *I,
Itr != DestWeights.end() ? Itr->second : 0);
}
}
@ -2375,7 +2399,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
if (i == count) {
assert((count < 3) && "Too much destinations to test!");
CasesBits.push_back(CaseBits(0, Dest, 0));
CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
count++;
}
@ -2384,6 +2408,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
uint64_t lo = (lowValue - lowBound).getZExtValue();
uint64_t hi = (highValue - lowBound).getZExtValue();
CasesBits[i].ExtraWeight += I->ExtraWeight;
for (uint64_t j = lo; j <= hi; j++) {
CasesBits[i].Mask |= 1ULL << j;
@ -2411,7 +2436,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
CurMF->insert(BBI, CaseBB);
BTC.push_back(BitTestCase(CasesBits[i].Mask,
CaseBB,
CasesBits[i].BB));
CasesBits[i].BB, CasesBits[i].ExtraWeight));
// Put SV in a virtual register to make it available from the new blocks.
ExportFromCurrentBlock(SV);
@ -2439,30 +2464,25 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
Clusterifier TheClusterifier;
BranchProbabilityInfo *BPI = FuncInfo.BPI;
// Start with "simple" cases
for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
i != e; ++i) {
const BasicBlock *SuccBB = i.getCaseSuccessor();
MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
TheClusterifier.add(i.getCaseValueEx(), SMBB);
TheClusterifier.add(i.getCaseValueEx(), SMBB,
BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0);
}
TheClusterifier.optimize();
BranchProbabilityInfo *BPI = FuncInfo.BPI;
size_t numCmps = 0;
for (Clusterifier::RangeIterator i = TheClusterifier.begin(),
e = TheClusterifier.end(); i != e; ++i, ++numCmps) {
Clusterifier::Cluster &C = *i;
unsigned W = 0;
if (BPI) {
W = BPI->getEdgeWeight(SI.getParent(), C.second->getBasicBlock());
if (!W)
W = 16;
W *= C.first.Weight;
BPI->setEdgeWeight(SI.getParent(), C.second->getBasicBlock(), W);
}
// Update edge weight for the cluster.
unsigned W = C.first.Weight;
// FIXME: Currently work with ConstantInt based numbers.
// Changing it to APInt based is a pretty heavy for this commit.

13
lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
View File

@ -150,9 +150,11 @@ private:
uint64_t Mask;
MachineBasicBlock* BB;
unsigned Bits;
uint32_t ExtraWeight;
CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits):
Mask(mask), BB(bb), Bits(bits) { }
CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits,
uint32_t Weight):
Mask(mask), BB(bb), Bits(bits), ExtraWeight(Weight) { }
};
typedef std::vector<Case> CaseVector;
@ -247,11 +249,13 @@ private:
typedef std::pair<JumpTableHeader, JumpTable> JumpTableBlock;
struct BitTestCase {
BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr):
Mask(M), ThisBB(T), TargetBB(Tr) { }
BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr,
uint32_t Weight):
Mask(M), ThisBB(T), TargetBB(Tr), ExtraWeight(Weight) { }
uint64_t Mask;
MachineBasicBlock *ThisBB;
MachineBasicBlock *TargetBB;
uint32_t ExtraWeight;
};
typedef SmallVector<BitTestCase, 3> BitTestInfo;
@ -452,6 +456,7 @@ public:
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB);
void visitBitTestCase(BitTestBlock &BB,
MachineBasicBlock* NextMBB,
uint32_t BranchWeightToNext,
unsigned Reg,
BitTestCase &B,
MachineBasicBlock *SwitchBB);

7
lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
View File

@ -1209,7 +1209,12 @@ SelectionDAGISel::FinishBasicBlock() {
CodeGenAndEmitDAG();
}
uint32_t UnhandledWeight = 0;
for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j)
UnhandledWeight += SDB->BitTestCases[i].Cases[j].ExtraWeight;
for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) {
UnhandledWeight -= SDB->BitTestCases[i].Cases[j].ExtraWeight;
// Set the current basic block to the mbb we wish to insert the code into
FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB;
FuncInfo->InsertPt = FuncInfo->MBB->end();
@ -1217,12 +1222,14 @@ SelectionDAGISel::FinishBasicBlock() {
if (j+1 != ej)
SDB->visitBitTestCase(SDB->BitTestCases[i],
SDB->BitTestCases[i].Cases[j+1].ThisBB,
UnhandledWeight,
SDB->BitTestCases[i].Reg,
SDB->BitTestCases[i].Cases[j],
FuncInfo->MBB);
else
SDB->visitBitTestCase(SDB->BitTestCases[i],
SDB->BitTestCases[i].Default,
UnhandledWeight,
SDB->BitTestCases[i].Reg,
SDB->BitTestCases[i].Cases[j],
FuncInfo->MBB);

27
test/Analysis/BranchProbabilityInfo/basic.ll
View File

@ -88,3 +88,30 @@ exit:
}
!1 = metadata !{metadata !"branch_weights", i32 4, i32 4, i32 64, i32 4, i32 4}
define i32 @test4(i32 %x) nounwind uwtable readnone ssp {
; CHECK: Printing analysis {{.*}} for function 'test4'
entry:
%conv = sext i32 %x to i64
switch i64 %conv, label %return [
i64 0, label %sw.bb
i64 1, label %sw.bb
i64 2, label %sw.bb
i64 5, label %sw.bb1
], !prof !2
; CHECK: edge entry -> return probability is 7 / 85
; CHECK: edge entry -> sw.bb probability is 14 / 85
; CHECK: edge entry -> sw.bb1 probability is 64 / 85
sw.bb:
br label %return
sw.bb1:
br label %return
return:
%retval.0 = phi i32 [ 5, %sw.bb1 ], [ 1, %sw.bb ], [ 0, %entry ]
ret i32 %retval.0
}
!2 = metadata !{metadata !"branch_weights", i32 7, i32 6, i32 4, i32 4, i32 64}

32
test/CodeGen/Generic/MachineBranchProb.ll Normal file
View File

@ -0,0 +1,32 @@
; RUN: llc < %s -print-machineinstrs=expand-isel-pseudos -o /dev/null 2>&1 | FileCheck %s
; Make sure we have the correct weight attached to each successor.
define i32 @test2(i32 %x) nounwind uwtable readnone ssp {
; CHECK: Machine code for function test2:
entry:
%conv = sext i32 %x to i64
switch i64 %conv, label %return [
i64 0, label %sw.bb
i64 1, label %sw.bb
i64 4, label %sw.bb
i64 5, label %sw.bb1
], !prof !0
; CHECK: BB#0: derived from LLVM BB %entry
; CHECK: Successors according to CFG: BB#2(64) BB#4(14)
; CHECK: BB#4: derived from LLVM BB %entry
; CHECK: Successors according to CFG: BB#1(10) BB#5(4)
; CHECK: BB#5: derived from LLVM BB %entry
; CHECK: Successors according to CFG: BB#1(4) BB#3(7)
sw.bb:
br label %return
sw.bb1:
br label %return
return:
%retval.0 = phi i32 [ 5, %sw.bb1 ], [ 1, %sw.bb ], [ 0, %entry ]
ret i32 %retval.0
}
!0 = metadata !{metadata !"branch_weights", i32 7, i32 6, i32 4, i32 4, i32 64}