Introduce memoization for ScalarEvolution dominates and properlyDominates

queries, and SCEVExpander getRelevantLoop queries.

llvm-svn: 119595

include/llvm/Analysis/ScalarEvolution.h

@@ -151,6 +151,14 @@ namespace llvm {
       LoopComputable  ///< The SCEV varies predictably with the loop.
     };
 
+    /// BlockDisposition - An enum describing the relationship between a
+    /// SCEV and a basic block.
+    enum BlockDisposition {
+      DoesNotDominateBlock,  ///< The SCEV does not dominate the block.
+      DominatesBlock,        ///< The SCEV dominates the block.
+      ProperlyDominatesBlock ///< The SCEV properly dominates the block.
+    };
+
   private:
     /// SCEVCallbackVH - A CallbackVH to arrange for ScalarEvolution to be
     /// notified whenever a Value is deleted.
@@ -246,6 +254,13 @@ namespace llvm {
     /// computeLoopDisposition - Compute a LoopDisposition value.
     LoopDisposition computeLoopDisposition(const SCEV *S, const Loop *L);
 
+    /// BlockDispositions - Memoized computeBlockDisposition results.
+    std::map<const SCEV *,
+             std::map<const BasicBlock *, BlockDisposition> > BlockDispositions;
+
+    /// computeBlockDisposition - Compute a BlockDisposition value.
+    BlockDisposition computeBlockDisposition(const SCEV *S, const BasicBlock *BB);
+
     /// UnsignedRanges - Memoized results from getUnsignedRange
     DenseMap<const SCEV *, ConstantRange> UnsignedRanges;
 
@@ -697,13 +712,17 @@ namespace llvm {
     /// to compute the value of the expression at any particular loop iteration.
     bool hasComputableLoopEvolution(const SCEV *S, const Loop *L);
 
+    /// getLoopDisposition - Return the "disposition" of the given SCEV with
+    /// respect to the given block.
+    BlockDisposition getBlockDisposition(const SCEV *S, const BasicBlock *BB);
+
     /// dominates - Return true if elements that makes up the given SCEV
     /// dominate the specified basic block.
-    bool dominates(const SCEV *S, BasicBlock *BB) const;
+    bool dominates(const SCEV *S, const BasicBlock *BB);
 
     /// properlyDominates - Return true if elements that makes up the given SCEV
     /// properly dominate the specified basic block.
-    bool properlyDominates(const SCEV *S, BasicBlock *BB) const;
+    bool properlyDominates(const SCEV *S, const BasicBlock *BB);
 
     /// hasOperand - Test whether the given SCEV has Op as a direct or
     /// indirect operand.

include/llvm/Analysis/ScalarEvolutionExpander.h

@@ -35,6 +35,9 @@ namespace llvm {
     std::set<AssertingVH<Value> > InsertedValues;
     std::set<AssertingVH<Value> > InsertedPostIncValues;
 
+    /// RelevantLoops - A memoization of the "relevant" loop for a given SCEV.
+    DenseMap<const SCEV *, const Loop *> RelevantLoops;
+
     /// PostIncLoops - Addrecs referring to any of the given loops are expanded
     /// in post-inc mode. For example, expanding {1,+,1}<L> in post-inc mode
     /// returns the add instruction that adds one to the phi for {0,+,1}<L>,
@@ -168,6 +171,9 @@ namespace llvm {
       return InsertedValues.count(I) || InsertedPostIncValues.count(I);
     }
 
+    /// getRelevantLoop - Determine the most "relevant" loop for the given SCEV.
+    const Loop *getRelevantLoop(const SCEV *);
+
     Value *visitConstant(const SCEVConstant *S) {
       return S->getValue();
     }

lib/Analysis/ScalarEvolution.cpp

@@ -5790,6 +5790,7 @@ void ScalarEvolution::releaseMemory() {
   ConstantEvolutionLoopExitValue.clear();
   ValuesAtScopes.clear();
   LoopDispositions.clear();
+  BlockDispositions.clear();
   UnsignedRanges.clear();
   SignedRanges.clear();
   UniqueSCEVs.clear();
@@ -5990,64 +5991,35 @@ bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
   return getLoopDisposition(S, L) == LoopComputable;
 }
 
-bool ScalarEvolution::dominates(const SCEV *S, BasicBlock *BB) const {
+ScalarEvolution::BlockDisposition
-  switch (S->getSCEVType()) {
+ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
-  case scConstant:
+  std::map<const BasicBlock *, BlockDisposition> &Values = BlockDispositions[S];
-    return true;
+  std::pair<std::map<const BasicBlock *, BlockDisposition>::iterator, bool>
-  case scTruncate:
+    Pair = Values.insert(std::make_pair(BB, DoesNotDominateBlock));
-  case scZeroExtend:
+  if (!Pair.second)
-  case scSignExtend:
+    return Pair.first->second;
-    return dominates(cast<SCEVCastExpr>(S)->getOperand(), BB);
+
-  case scAddRecExpr: {
+  BlockDisposition D = computeBlockDisposition(S, BB);
-    const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
+  return BlockDispositions[S][BB] = D;
-    if (!DT->dominates(AR->getLoop()->getHeader(), BB))
-      return false;
-  }
-  // FALL THROUGH into SCEVNAryExpr handling.
-  case scAddExpr:
-  case scMulExpr:
-  case scUMaxExpr:
-  case scSMaxExpr: {
-    const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S);
-    for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
-         I != E; ++I)
-      if (!dominates(*I, BB))
-        return false;
-    return true;
-  }
-  case scUDivExpr: {
-    const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
-    return dominates(UDiv->getLHS(), BB) && dominates(UDiv->getRHS(), BB);
-  }
-  case scUnknown:
-    if (Instruction *I =
-          dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue()))
-      return DT->dominates(I->getParent(), BB);
-    return true;
-  case scCouldNotCompute:
-    llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
-    return false;
-  default: break;
-  }
-  llvm_unreachable("Unknown SCEV kind!");
-  return false;
 }
 
-bool ScalarEvolution::properlyDominates(const SCEV *S, BasicBlock *BB) const {
+ScalarEvolution::BlockDisposition
+ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
   switch (S->getSCEVType()) {
   case scConstant:
-    return true;
+    return ProperlyDominatesBlock;
   case scTruncate:
   case scZeroExtend:
   case scSignExtend:
-    return properlyDominates(cast<SCEVCastExpr>(S)->getOperand(), BB);
+    return getBlockDisposition(cast<SCEVCastExpr>(S)->getOperand(), BB);
   case scAddRecExpr: {
     // This uses a "dominates" query instead of "properly dominates" query
-    // because the instruction which produces the addrec's value is a PHI, and
+    // to test for proper dominance too, because the instruction which
-    // a PHI effectively properly dominates its entire containing block.
+    // produces the addrec's value is a PHI, and a PHI effectively properly
+    // dominates its entire containing block.
     const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(S);
     if (!DT->dominates(AR->getLoop()->getHeader(), BB))
-      return false;
+      return DoesNotDominateBlock;
   }
   // FALL THROUGH into SCEVNAryExpr handling.
   case scAddExpr:
@@ -6055,29 +6027,54 @@ bool ScalarEvolution::properlyDominates(const SCEV *S, BasicBlock *BB) const {
   case scUMaxExpr:
   case scSMaxExpr: {
     const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(S);
+    bool Proper = true;
     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
-         I != E; ++I)
+         I != E; ++I) {
-      if (!properlyDominates(*I, BB))
+      BlockDisposition D = getBlockDisposition(*I, BB);
-        return false;
+      if (D == DoesNotDominateBlock)
-    return true;
+        return DoesNotDominateBlock;
+      if (D == DominatesBlock)
+        Proper = false;
+    }
+    return Proper ? ProperlyDominatesBlock : DominatesBlock;
   }
   case scUDivExpr: {
     const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
-    return properlyDominates(UDiv->getLHS(), BB) &&
+    const SCEV *LHS = UDiv->getLHS(), *RHS = UDiv->getRHS();
-           properlyDominates(UDiv->getRHS(), BB);
+    BlockDisposition LD = getBlockDisposition(LHS, BB);
+    if (LD == DoesNotDominateBlock)
+      return DoesNotDominateBlock;
+    BlockDisposition RD = getBlockDisposition(RHS, BB);
+    if (RD == DoesNotDominateBlock)
+      return DoesNotDominateBlock;
+    return (LD == ProperlyDominatesBlock && RD == ProperlyDominatesBlock) ?
+      ProperlyDominatesBlock : DominatesBlock;
   }
   case scUnknown:
     if (Instruction *I =
-          dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue()))
+          dyn_cast<Instruction>(cast<SCEVUnknown>(S)->getValue())) {
-      return DT->properlyDominates(I->getParent(), BB);
+      if (I->getParent() == BB)
-    return true;
+        return DominatesBlock;
+      if (DT->properlyDominates(I->getParent(), BB))
+        return ProperlyDominatesBlock;
+      return DoesNotDominateBlock;
+    }
+    return ProperlyDominatesBlock;
   case scCouldNotCompute:
     llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
-    return false;
+    return DoesNotDominateBlock;
   default: break;
   }
   llvm_unreachable("Unknown SCEV kind!");
-  return false;
+  return DoesNotDominateBlock;
+}
+
+bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
+  return getBlockDisposition(S, BB) >= DominatesBlock;
+}
+
+bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
+  return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
 }
 
 bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
@@ -6125,6 +6122,7 @@ bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
 void ScalarEvolution::forgetMemoizedResults(const SCEV *S) {
   ValuesAtScopes.erase(S);
   LoopDispositions.erase(S);
+  BlockDispositions.erase(S);
   UnsignedRanges.erase(S);
   SignedRanges.erase(S);
 }

lib/Analysis/ScalarEvolutionExpander.cpp

@@ -608,15 +608,22 @@ static const Loop *PickMostRelevantLoop(const Loop *A, const Loop *B,
   return A; // Arbitrarily break the tie.
 }
 
-/// GetRelevantLoop - Get the most relevant loop associated with the given
+/// getRelevantLoop - Get the most relevant loop associated with the given
 /// expression, according to PickMostRelevantLoop.
-static const Loop *GetRelevantLoop(const SCEV *S, LoopInfo &LI,
+const Loop *SCEVExpander::getRelevantLoop(const SCEV *S) {
-                                   DominatorTree &DT) {
+  // Test whether we've already computed the most relevant loop for this SCEV.
+  std::pair<DenseMap<const SCEV *, const Loop *>::iterator, bool> Pair =
+    RelevantLoops.insert(std::make_pair(S, static_cast<const Loop *>(0)));
+  if (!Pair.second)
+    return Pair.first->second;
+
   if (isa<SCEVConstant>(S))
+    // A constant has no relevant loops.
     return 0;
   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
     if (const Instruction *I = dyn_cast<Instruction>(U->getValue()))
-      return LI.getLoopFor(I->getParent());
+      return Pair.first->second = SE.LI->getLoopFor(I->getParent());
+    // A non-instruction has no relevant loops.
     return 0;
   }
   if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) {
@@ -625,16 +632,22 @@ static const Loop *GetRelevantLoop(const SCEV *S, LoopInfo &LI,
       L = AR->getLoop();
     for (SCEVNAryExpr::op_iterator I = N->op_begin(), E = N->op_end();
          I != E; ++I)
-      L = PickMostRelevantLoop(L, GetRelevantLoop(*I, LI, DT), DT);
+      L = PickMostRelevantLoop(L, getRelevantLoop(*I), *SE.DT);
-    return L;
+    return RelevantLoops[N] = L;
+  }
+  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) {
+    const Loop *Result = getRelevantLoop(C->getOperand());
+    return RelevantLoops[C] = Result;
+  }
+  if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
+    const Loop *Result =
+      PickMostRelevantLoop(getRelevantLoop(D->getLHS()),
+                           getRelevantLoop(D->getRHS()),
+                           *SE.DT);
+    return RelevantLoops[D] = Result;
   }
-  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S))
-    return GetRelevantLoop(C->getOperand(), LI, DT);
-  if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S))
-    return PickMostRelevantLoop(GetRelevantLoop(D->getLHS(), LI, DT),
-                                GetRelevantLoop(D->getRHS(), LI, DT),
-                                DT);
   llvm_unreachable("Unexpected SCEV type!");
+  return 0;
 }
 
 namespace {
@@ -682,8 +695,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
   SmallVector<std::pair<const Loop *, const SCEV *>, 8> OpsAndLoops;
   for (std::reverse_iterator<SCEVAddExpr::op_iterator> I(S->op_end()),
        E(S->op_begin()); I != E; ++I)
-    OpsAndLoops.push_back(std::make_pair(GetRelevantLoop(*I, *SE.LI, *SE.DT),
+    OpsAndLoops.push_back(std::make_pair(getRelevantLoop(*I), *I));
-                                         *I));
 
   // Sort by loop. Use a stable sort so that constants follow non-constants and
   // pointer operands precede non-pointer operands.
@@ -752,8 +764,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
   SmallVector<std::pair<const Loop *, const SCEV *>, 8> OpsAndLoops;
   for (std::reverse_iterator<SCEVMulExpr::op_iterator> I(S->op_end()),
        E(S->op_begin()); I != E; ++I)
-    OpsAndLoops.push_back(std::make_pair(GetRelevantLoop(*I, *SE.LI, *SE.DT),
+    OpsAndLoops.push_back(std::make_pair(getRelevantLoop(*I), *I));
-                                         *I));
 
   // Sort by loop. Use a stable sort so that constants follow non-constants.
   std::stable_sort(OpsAndLoops.begin(), OpsAndLoops.end(), LoopCompare(*SE.DT));