From aab80ed89c78d5f66f542b8a9595ee0c5e86b392 Mon Sep 17 00:00:00 2001 From: Silviu Baranga Date: Wed, 6 Apr 2016 14:06:32 +0000 Subject: [PATCH] Revert r265535 until we know how we can fix the bots llvm-svn: 265541 --- include/llvm/Analysis/ScalarEvolution.h | 249 ++------------ lib/Analysis/LoopAccessAnalysis.cpp | 4 +- lib/Analysis/ScalarEvolution.cpp | 315 +++++------------- lib/Analysis/ScalarEvolutionExpander.cpp | 4 +- lib/Transforms/Vectorize/LoopVectorize.cpp | 4 +- .../ScalarEvolution/predicated-trip-count.ll | 109 ------ .../AArch64/backedge-overflow.ll | 166 --------- .../X86/vectorization-remarks-missed.ll | 3 +- 8 files changed, 125 insertions(+), 729 deletions(-) delete mode 100644 test/Analysis/ScalarEvolution/predicated-trip-count.ll delete mode 100644 test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index f102f2eccdf..3a84e77b53a 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -270,17 +270,10 @@ namespace llvm { } }; - /// SCEVWrapPredicate - This class represents an assumption made on an AddRec - /// expression. Given an affine AddRec expression {a,+,b}, we assume that it - /// has the nssw or nusw flags (defined below) in the first X iterations of - /// the loop, where X is a SCEV expression returned by - /// getPredicatedBackedgeTakenCount). - /// - /// Note that this does not imply that X is equal to the backedge taken - /// count. This means that if we have a nusw predicate for i32 {0,+,1} with a - /// predicated backedge taken count of X, we only guarantee that {0,+,1} has - /// nusw in the first X iterations. {0,+,1} may still wrap in the loop if we - /// have more than X iterations. + /// SCEVWrapPredicate - This class represents an assumption + /// made on an AddRec expression. Given an affine AddRec expression + /// {a,+,b}, we assume that it has the nssw or nusw flags (defined + /// below). class SCEVWrapPredicate final : public SCEVPredicate { public: /// Similar to SCEV::NoWrapFlags, but with slightly different semantics @@ -527,14 +520,9 @@ namespace llvm { const SCEV *Exact; const SCEV *Max; - /// A predicate union guard for this ExitLimit. The result is only - /// valid if this predicate evaluates to 'true' at run-time. - SCEVUnionPredicate Pred; - /*implicit*/ ExitLimit(const SCEV *E) : Exact(E), Max(E) {} - ExitLimit(const SCEV *E, const SCEV *M, SCEVUnionPredicate &P) - : Exact(E), Max(M), Pred(P) { + ExitLimit(const SCEV *E, const SCEV *M) : Exact(E), Max(M) { assert((isa(Exact) || !isa(Max)) && "Exact is not allowed to be less precise than Max"); @@ -546,146 +534,30 @@ namespace llvm { return !isa(Exact) || !isa(Max); } - - /// Test whether this ExitLimit contains all information. - bool hasFullInfo() const { return !isa(Exact); } }; - /// Forward declaration of ExitNotTakenExtras - struct ExitNotTakenExtras; - /// Information about the number of times a particular loop exit may be /// reached before exiting the loop. struct ExitNotTakenInfo { AssertingVH ExitingBlock; const SCEV *ExactNotTaken; - - PointerIntPair ExtraInfo; + PointerIntPair NextExit; ExitNotTakenInfo() : ExitingBlock(nullptr), ExactNotTaken(nullptr) {} - ExitNotTakenInfo(BasicBlock *ExitBlock, const SCEV *Expr, - ExitNotTakenExtras *Ptr) - : ExitingBlock(ExitBlock), ExactNotTaken(Expr) { - ExtraInfo.setPointer(Ptr); - } /// Return true if all loop exits are computable. - bool isCompleteList() const { return ExtraInfo.getInt() == 0; } - - /// Sets the incomplete property, indicating that one of the loop exits - /// doesn't have a corresponding ExitNotTakenInfo entry. - void setIncomplete() { ExtraInfo.setInt(1); } - - /// Returns a pointer to the predicate associated with this information, - /// or nullptr if this doesn't exist (meaning always true). - SCEVUnionPredicate *getPred() const { - if (auto *Info = ExtraInfo.getPointer()) - return &Info->Pred; - - return nullptr; + bool isCompleteList() const { + return NextExit.getInt() == 0; } - /// Return true if the SCEV predicate associated with this information - /// is always true. - bool hasAlwaysTruePred() const { - return !getPred() || getPred()->isAlwaysTrue(); + void setIncomplete() { NextExit.setInt(1); } + + /// Return a pointer to the next exit's not-taken info. + ExitNotTakenInfo *getNextExit() const { + return NextExit.getPointer(); } - /// Defines a simple forward iterator for ExitNotTakenInfo. - class ExitNotTakenInfoIterator - : public std::iterator { - const ExitNotTakenInfo *Start; - unsigned Position; - - public: - ExitNotTakenInfoIterator(const ExitNotTakenInfo *Start, - unsigned Position) - : Start(Start), Position(Position) {} - - const ExitNotTakenInfo &operator*() const { - if (Position == 0) - return *Start; - - return Start->ExtraInfo.getPointer()->Exits[Position - 1]; - } - - const ExitNotTakenInfo *operator->() const { - if (Position == 0) - return Start; - - return &Start->ExtraInfo.getPointer()->Exits[Position - 1]; - } - - bool operator==(const ExitNotTakenInfoIterator &RHS) const { - return Start == RHS.Start && Position == RHS.Position; - } - - bool operator!=(const ExitNotTakenInfoIterator &RHS) const { - return Start != RHS.Start || Position != RHS.Position; - } - - ExitNotTakenInfoIterator &operator++() { // Preincrement - if (!Start) - return *this; - - unsigned Elements = - Start->ExtraInfo.getPointer() - ? Start->ExtraInfo.getPointer()->Exits.size() + 1 - : 1; - - ++Position; - - // We've run out of elements. - if (Position == Elements) { - Start = nullptr; - Position = 0; - } - - return *this; - } - ExitNotTakenInfoIterator operator++(int) { // Postincrement - ExitNotTakenInfoIterator Tmp = *this; - ++*this; - return Tmp; - } - }; - - /// Iterators - ExitNotTakenInfoIterator begin() const { - return ExitNotTakenInfoIterator(this, 0); - } - ExitNotTakenInfoIterator end() const { - return ExitNotTakenInfoIterator(nullptr, 0); - } - }; - - /// Describes the extra information that a ExitNotTakenInfo can have. - struct ExitNotTakenExtras { - /// The predicate associated with the ExitNotTakenInfo struct. - SCEVUnionPredicate Pred; - - /// The extra exits in the loop. Only the ExitNotTakenExtras structure - /// pointed to by the first ExitNotTakenInfo struct (associated with the - /// first loop exit) will populate this vector to prevent having - /// redundant information. - SmallVector Exits; - }; - - /// A struct containing the information attached to a backedge. - struct EdgeInfo { - EdgeInfo(BasicBlock *Block, const SCEV *Taken, SCEVUnionPredicate &P) : - ExitBlock(Block), Taken(Taken), Pred(std::move(P)) {} - - /// The exit basic block. - BasicBlock *ExitBlock; - - /// The (exact) number of time we take the edge back. - const SCEV *Taken; - - /// The SCEV predicated associated with Taken. If Pred doesn't evaluate - /// to true, the information in Taken is not valid (or equivalent with - /// a CouldNotCompute. - SCEVUnionPredicate Pred; + void setNextExit(ExitNotTakenInfo *ENT) { NextExit.setPointer(ENT); } }; /// Information about the backedge-taken count of a loop. This currently @@ -697,16 +569,16 @@ namespace llvm { ExitNotTakenInfo ExitNotTaken; /// An expression indicating the least maximum backedge-taken count of the - /// loop that is known, or a SCEVCouldNotCompute. This expression is only - /// valid if the predicates associated with all loop exits are true. + /// loop that is known, or a SCEVCouldNotCompute. const SCEV *Max; public: BackedgeTakenInfo() : Max(nullptr) {} /// Initialize BackedgeTakenInfo from a list of exact exit counts. - BackedgeTakenInfo(SmallVectorImpl &ExitCounts, bool Complete, - const SCEV *MaxCount); + BackedgeTakenInfo( + SmallVectorImpl< std::pair > &ExitCounts, + bool Complete, const SCEV *MaxCount); /// Test whether this BackedgeTakenInfo contains any computed information, /// or whether it's all SCEVCouldNotCompute values. @@ -714,27 +586,11 @@ namespace llvm { return ExitNotTaken.ExitingBlock || !isa(Max); } - /// Test whether this BackedgeTakenInfo contains complete information. - bool hasFullInfo() const { return ExitNotTaken.isCompleteList(); } - /// Return an expression indicating the exact backedge-taken count of the - /// loop if it is known or SCEVCouldNotCompute otherwise. This is the + /// loop if it is known, or SCEVCouldNotCompute otherwise. This is the /// number of times the loop header can be guaranteed to execute, minus /// one. - /// - /// If the SCEV predicate associated with the answer can be different - /// from AlwaysTrue, we must add a (non null) Predicates argument. - /// The SCEV predicate associated with the answer will be added to - /// Predicates. A run-time check needs to be emitted for the SCEV - /// predicate in order for the answer to be valid. - /// - /// Note that we should always know if we need to pass a predicate - /// argument or not from the way the ExitCounts vector was computed. - /// If we allowed SCEV predicates to be generated when populating this - /// vector, this information can contain them and therefore a - /// SCEVPredicate argument should be added to getExact. - const SCEV *getExact(ScalarEvolution *SE, - SCEVUnionPredicate *Predicates = nullptr) const; + const SCEV *getExact(ScalarEvolution *SE) const; /// Return the number of times this loop exit may fall through to the back /// edge, or SCEVCouldNotCompute. The loop is guaranteed not to exit via @@ -755,11 +611,7 @@ namespace llvm { /// Cache the backedge-taken count of the loops for this function as they /// are computed. - DenseMap BackedgeTakenCounts; - - /// Cache the predicated backedge-taken count of the loops for this - /// function as they are computed. - DenseMap PredicatedBackedgeTakenCounts; + DenseMap BackedgeTakenCounts; /// This map contains entries for all of the PHI instructions that we /// attempt to compute constant evolutions for. This allows us to avoid @@ -861,49 +713,33 @@ namespace llvm { void forgetSymbolicName(Instruction *I, const SCEV *SymName); /// Return the BackedgeTakenInfo for the given loop, lazily computing new - /// values if the loop hasn't been analyzed yet. The returned result is - /// guaranteed not to be predicated. + /// values if the loop hasn't been analyzed yet. const BackedgeTakenInfo &getBackedgeTakenInfo(const Loop *L); - /// Similar to getBackedgeTakenInfo, but will add predicates as required - /// with the purpose of returning complete information. - const BackedgeTakenInfo &getPredicatedBackedgeTakenInfo(const Loop *L); - /// Compute the number of times the specified loop will iterate. - /// If AllowPredicates is set, we will create new SCEV predicates as - /// necessary in order to return an exact answer. - BackedgeTakenInfo computeBackedgeTakenCount(const Loop *L, - bool AllowPredicates = false); + BackedgeTakenInfo computeBackedgeTakenCount(const Loop *L); /// Compute the number of times the backedge of the specified loop will - /// execute if it exits via the specified block. If AllowPredicates is set, - /// this call will try to use a minimal set of SCEV predicates in order to - /// return an exact answer. - ExitLimit computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, - bool AllowPredicates = false); + /// execute if it exits via the specified block. + ExitLimit computeExitLimit(const Loop *L, BasicBlock *ExitingBlock); /// Compute the number of times the backedge of the specified loop will /// execute if its exit condition were a conditional branch of ExitCond, - /// TBB, and FBB. If AllowPredicates is set, this call will try to use a - /// minimal set of SCEV predicates in order to return an exact answer. + /// TBB, and FBB. ExitLimit computeExitLimitFromCond(const Loop *L, Value *ExitCond, BasicBlock *TBB, BasicBlock *FBB, - bool IsSubExpr, - bool AllowPredicates = false); + bool IsSubExpr); /// Compute the number of times the backedge of the specified loop will /// execute if its exit condition were a conditional branch of the ICmpInst - /// ExitCond, TBB, and FBB. If AllowPredicates is set, this call will try - /// to use a minimal set of SCEV predicates in order to return an exact - /// answer. + /// ExitCond, TBB, and FBB. ExitLimit computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, BasicBlock *TBB, BasicBlock *FBB, - bool IsSubExpr, - bool AllowPredicates = false); + bool IsSubExpr); /// Compute the number of times the backedge of the specified loop will /// execute if its exit condition were a switch with a single exiting case @@ -941,10 +777,7 @@ namespace llvm { /// Return the number of times an exit condition comparing the specified /// value to zero will execute. If not computable, return CouldNotCompute. - /// If AllowPredicates is set, this call will try to use a minimal set of - /// SCEV predicates in order to return an exact answer. - ExitLimit HowFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr, - bool AllowPredicates = false); + ExitLimit HowFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr); /// Return the number of times an exit condition checking the specified /// value for nonzero will execute. If not computable, return @@ -954,15 +787,10 @@ namespace llvm { /// Return the number of times an exit condition containing the specified /// less-than comparison will execute. If not computable, return /// CouldNotCompute. isSigned specifies whether the less-than is signed. - /// If AllowPredicates is set, this call will try to use a minimal set of - /// SCEV predicates in order to return an exact answer. - ExitLimit HowManyLessThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, - bool isSigned, bool IsSubExpr, - bool AllowPredicates = false); - + ExitLimit HowManyLessThans(const SCEV *LHS, const SCEV *RHS, + const Loop *L, bool isSigned, bool IsSubExpr); ExitLimit HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS, - const Loop *L, bool isSigned, bool IsSubExpr, - bool AllowPredicates = false); + const Loop *L, bool isSigned, bool IsSubExpr); /// Return a predecessor of BB (which may not be an immediate predecessor) /// which has exactly one successor from which BB is reachable, or null if @@ -1340,13 +1168,6 @@ namespace llvm { /// const SCEV *getBackedgeTakenCount(const Loop *L); - /// Similar to getBackedgeTakenCount, except it will add a set of - /// SCEV predicates to Predicates that are required to be true in order for - /// the answer to be correct. Predicates can be checked with run-time - /// checks and can be used to perform loop versioning. - const SCEV *getPredicatedBackedgeTakenCount(const Loop *L, - SCEVUnionPredicate &Predicates); - /// Similar to getBackedgeTakenCount, except return the least SCEV value /// that is known never to be less than the actual backedge taken count. const SCEV *getMaxBackedgeTakenCount(const Loop *L); @@ -1672,8 +1493,6 @@ namespace llvm { /// by ScalarEvolution is guaranteed to be preserved, even when adding new /// predicates. const SCEV *getSCEV(Value *V); - /// Get the (predicated) backedge count for the analyzed loop. - const SCEV *getBackedgeTakenCount(); /// \brief Adds a new predicate. void addPredicate(const SCEVPredicate &Pred); /// \brief Attempts to produce an AddRecExpr for V by adding additional @@ -1717,8 +1536,6 @@ namespace llvm { /// figure out if the predicate has changed from the last rewrite of the /// SCEV. If so, we need to perform a new rewrite. unsigned Generation; - /// The backedge taken count. - const SCEV *BackedgeCount; }; } diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp index d1eac460ca3..c67c581a018 100644 --- a/lib/Analysis/LoopAccessAnalysis.cpp +++ b/lib/Analysis/LoopAccessAnalysis.cpp @@ -140,7 +140,7 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, bool WritePtr, else { const SCEVAddRecExpr *AR = dyn_cast(Sc); assert(AR && "Invalid addrec expression"); - const SCEV *Ex = PSE.getBackedgeTakenCount(); + const SCEV *Ex = SE->getBackedgeTakenCount(Lp); ScStart = AR->getStart(); ScEnd = AR->evaluateAtIteration(Ex, *SE); @@ -1460,7 +1460,7 @@ bool LoopAccessInfo::canAnalyzeLoop() { } // ScalarEvolution needs to be able to find the exit count. - const SCEV *ExitCount = PSE.getBackedgeTakenCount(); + const SCEV *ExitCount = PSE.getSE()->getBackedgeTakenCount(TheLoop); if (ExitCount == PSE.getSE()->getCouldNotCompute()) { emitAnalysis(LoopAccessReport() << "could not determine number of loop iterations"); diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index 36b434899f5..402a1b7c4ea 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -5223,12 +5223,6 @@ const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) { return getBackedgeTakenInfo(L).getExact(ExitingBlock, this); } -const SCEV * -ScalarEvolution::getPredicatedBackedgeTakenCount(const Loop *L, - SCEVUnionPredicate &Preds) { - return getPredicatedBackedgeTakenInfo(L).getExact(this, &Preds); -} - /// getBackedgeTakenCount - If the specified loop has a predictable /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute /// object. The backedge-taken count is the number of times the loop header @@ -5263,23 +5257,6 @@ PushLoopPHIs(const Loop *L, SmallVectorImpl &Worklist) { Worklist.push_back(PN); } -const ScalarEvolution::BackedgeTakenInfo & -ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) { - auto &BTI = getBackedgeTakenInfo(L); - if (BTI.hasFullInfo()) - return BTI; - - auto Pair = PredicatedBackedgeTakenCounts.insert({L, BackedgeTakenInfo()}); - - if (!Pair.second) - return Pair.first->second; - - BackedgeTakenInfo Result = - computeBackedgeTakenCount(L, /*AllowPredicates=*/true); - - return PredicatedBackedgeTakenCounts.find(L)->second = Result; -} - const ScalarEvolution::BackedgeTakenInfo & ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { // Initially insert an invalid entry for this loop. If the insertion @@ -5360,17 +5337,12 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { /// compute a trip count, or if the loop is deleted. void ScalarEvolution::forgetLoop(const Loop *L) { // Drop any stored trip count value. - auto RemoveLoopFromBackedgeMap = - [L](DenseMap &Map) { - auto BTCPos = Map.find(L); - if (BTCPos != Map.end()) { - BTCPos->second.clear(); - Map.erase(BTCPos); - } - }; - - RemoveLoopFromBackedgeMap(BackedgeTakenCounts); - RemoveLoopFromBackedgeMap(PredicatedBackedgeTakenCounts); + DenseMap::iterator BTCPos = + BackedgeTakenCounts.find(L); + if (BTCPos != BackedgeTakenCounts.end()) { + BTCPos->second.clear(); + BackedgeTakenCounts.erase(BTCPos); + } // Drop information about expressions based on loop-header PHIs. SmallVector Worklist; @@ -5439,8 +5411,7 @@ void ScalarEvolution::forgetValue(Value *V) { /// is the caller's responsibility to specify the relevant loop exit using /// getExact(ExitingBlock, SE). const SCEV * -ScalarEvolution::BackedgeTakenInfo::getExact( - ScalarEvolution *SE, SCEVUnionPredicate *Preds) const { +ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const { // If any exits were not computable, the loop is not computable. if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute(); @@ -5449,20 +5420,16 @@ ScalarEvolution::BackedgeTakenInfo::getExact( assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info"); const SCEV *BECount = nullptr; - for (auto &ENT : ExitNotTaken) { - assert(ENT.ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); + for (const ExitNotTakenInfo *ENT = &ExitNotTaken; + ENT != nullptr; ENT = ENT->getNextExit()) { + + assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV"); if (!BECount) - BECount = ENT.ExactNotTaken; - else if (BECount != ENT.ExactNotTaken) + BECount = ENT->ExactNotTaken; + else if (BECount != ENT->ExactNotTaken) return SE->getCouldNotCompute(); - if (Preds && ENT.getPred()) - Preds->add(ENT.getPred()); - - assert((Preds || ENT.hasAlwaysTruePred()) && - "Predicate should be always true!"); } - assert(BECount && "Invalid not taken count for loop exit"); return BECount; } @@ -5471,20 +5438,18 @@ ScalarEvolution::BackedgeTakenInfo::getExact( const SCEV * ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock, ScalarEvolution *SE) const { - for (auto &ENT : ExitNotTaken) - if (ENT.ExitingBlock == ExitingBlock && ENT.hasAlwaysTruePred()) - return ENT.ExactNotTaken; + for (const ExitNotTakenInfo *ENT = &ExitNotTaken; + ENT != nullptr; ENT = ENT->getNextExit()) { + if (ENT->ExitingBlock == ExitingBlock) + return ENT->ExactNotTaken; + } return SE->getCouldNotCompute(); } /// getMax - Get the max backedge taken count for the loop. const SCEV * ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const { - for (auto &ENT : ExitNotTaken) - if (!ENT.hasAlwaysTruePred()) - return SE->getCouldNotCompute(); - return Max ? Max : SE->getCouldNotCompute(); } @@ -5496,19 +5461,22 @@ bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S, if (!ExitNotTaken.ExitingBlock) return false; - for (auto &ENT : ExitNotTaken) - if (ENT.ExactNotTaken != SE->getCouldNotCompute() && - SE->hasOperand(ENT.ExactNotTaken, S)) - return true; + for (const ExitNotTakenInfo *ENT = &ExitNotTaken; + ENT != nullptr; ENT = ENT->getNextExit()) { + if (ENT->ExactNotTaken != SE->getCouldNotCompute() + && SE->hasOperand(ENT->ExactNotTaken, S)) { + return true; + } + } return false; } /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each /// computable exit into a persistent ExitNotTakenInfo array. ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( - SmallVectorImpl &ExitCounts, bool Complete, const SCEV *MaxCount) - : Max(MaxCount) { + SmallVectorImpl< std::pair > &ExitCounts, + bool Complete, const SCEV *MaxCount) : Max(MaxCount) { if (!Complete) ExitNotTaken.setIncomplete(); @@ -5516,43 +5484,18 @@ ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( unsigned NumExits = ExitCounts.size(); if (NumExits == 0) return; - ExitNotTaken.ExitingBlock = ExitCounts[0].ExitBlock; - ExitNotTaken.ExactNotTaken = ExitCounts[0].Taken; - - // Determine the number of ExitNotTakenExtras structures that we need. - unsigned ExtraInfoSize = 0; - if (NumExits > 1) - ExtraInfoSize = 1 + std::count_if(std::next(ExitCounts.begin()), - ExitCounts.end(), [](EdgeInfo &Entry) { - return !Entry.Pred.isAlwaysTrue(); - }); - else if (!ExitCounts[0].Pred.isAlwaysTrue()) - ExtraInfoSize = 1; - - ExitNotTakenExtras *ENT = nullptr; - - // Allocate the ExitNotTakenExtras structures and initialize the first - // element (ExitNotTaken). - if (ExtraInfoSize > 0) { - ENT = new ExitNotTakenExtras[ExtraInfoSize]; - ExitNotTaken.ExtraInfo.setPointer(&ENT[0]); - *ExitNotTaken.getPred() = std::move(ExitCounts[0].Pred); - } - - if (NumExits == 1) - return; - - auto &Exits = ExitNotTaken.ExtraInfo.getPointer()->Exits; + ExitNotTaken.ExitingBlock = ExitCounts[0].first; + ExitNotTaken.ExactNotTaken = ExitCounts[0].second; + if (NumExits == 1) return; // Handle the rare case of multiple computable exits. - for (unsigned i = 1, PredPos = 1; i < NumExits; ++i) { - ExitNotTakenExtras *Ptr = nullptr; - if (!ExitCounts[i].Pred.isAlwaysTrue()) { - Ptr = &ENT[PredPos++]; - Ptr->Pred = std::move(ExitCounts[i].Pred); - } + ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1]; - Exits.emplace_back(ExitCounts[i].ExitBlock, ExitCounts[i].Taken, Ptr); + ExitNotTakenInfo *PrevENT = &ExitNotTaken; + for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) { + PrevENT->setNextExit(ENT); + ENT->ExitingBlock = ExitCounts[i].first; + ENT->ExactNotTaken = ExitCounts[i].second; } } @@ -5560,18 +5503,17 @@ ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( void ScalarEvolution::BackedgeTakenInfo::clear() { ExitNotTaken.ExitingBlock = nullptr; ExitNotTaken.ExactNotTaken = nullptr; - delete[] ExitNotTaken.ExtraInfo.getPointer(); + delete[] ExitNotTaken.getNextExit(); } /// computeBackedgeTakenCount - Compute the number of times the backedge /// of the specified loop will execute. ScalarEvolution::BackedgeTakenInfo -ScalarEvolution::computeBackedgeTakenCount(const Loop *L, - bool AllowPredicates) { +ScalarEvolution::computeBackedgeTakenCount(const Loop *L) { SmallVector ExitingBlocks; L->getExitingBlocks(ExitingBlocks); - SmallVector ExitCounts; + SmallVector, 4> ExitCounts; bool CouldComputeBECount = true; BasicBlock *Latch = L->getLoopLatch(); // may be NULL. const SCEV *MustExitMaxBECount = nullptr; @@ -5579,13 +5521,9 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts // and compute maxBECount. - // Do a union of all the predicates here. for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { BasicBlock *ExitBB = ExitingBlocks[i]; - ExitLimit EL = computeExitLimit(L, ExitBB, AllowPredicates); - - assert((AllowPredicates || EL.Pred.isAlwaysTrue()) && - "Predicated exit limit when predicates are not allowed!"); + ExitLimit EL = computeExitLimit(L, ExitBB); // 1. For each exit that can be computed, add an entry to ExitCounts. // CouldComputeBECount is true only if all exits can be computed. @@ -5594,7 +5532,7 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, // we won't be able to compute an exact value for the loop. CouldComputeBECount = false; else - ExitCounts.emplace_back(EdgeInfo(ExitBB, EL.Exact, EL.Pred)); + ExitCounts.push_back({ExitBB, EL.Exact}); // 2. Derive the loop's MaxBECount from each exit's max number of // non-exiting iterations. Partition the loop exits into two kinds: @@ -5628,8 +5566,7 @@ ScalarEvolution::computeBackedgeTakenCount(const Loop *L, } ScalarEvolution::ExitLimit -ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, - bool AllowPredicates) { +ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock) { // Okay, we've chosen an exiting block. See what condition causes us to exit // at this block and remember the exit block and whether all other targets @@ -5694,9 +5631,9 @@ ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock, if (BranchInst *BI = dyn_cast(Term)) { assert(BI->isConditional() && "If unconditional, it can't be in loop!"); // Proceed to the next level to examine the exit condition expression. - return computeExitLimitFromCond( - L, BI->getCondition(), BI->getSuccessor(0), BI->getSuccessor(1), - /*ControlsExit=*/IsOnlyExit, AllowPredicates); + return computeExitLimitFromCond(L, BI->getCondition(), BI->getSuccessor(0), + BI->getSuccessor(1), + /*ControlsExit=*/IsOnlyExit); } if (SwitchInst *SI = dyn_cast(Term)) @@ -5719,19 +5656,16 @@ ScalarEvolution::computeExitLimitFromCond(const Loop *L, Value *ExitCond, BasicBlock *TBB, BasicBlock *FBB, - bool ControlsExit, - bool AllowPredicates) { + bool ControlsExit) { // Check if the controlling expression for this loop is an And or Or. if (BinaryOperator *BO = dyn_cast(ExitCond)) { if (BO->getOpcode() == Instruction::And) { // Recurse on the operands of the and. bool EitherMayExit = L->contains(TBB); ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, - ControlsExit && !EitherMayExit, - AllowPredicates); + ControlsExit && !EitherMayExit); ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, - ControlsExit && !EitherMayExit, - AllowPredicates); + ControlsExit && !EitherMayExit); const SCEV *BECount = getCouldNotCompute(); const SCEV *MaxBECount = getCouldNotCompute(); if (EitherMayExit) { @@ -5758,9 +5692,6 @@ ScalarEvolution::computeExitLimitFromCond(const Loop *L, BECount = EL0.Exact; } - SCEVUnionPredicate NP; - NP.add(&EL0.Pred); - NP.add(&EL1.Pred); // There are cases (e.g. PR26207) where computeExitLimitFromCond is able // to be more aggressive when computing BECount than when computing // MaxBECount. In these cases it is possible for EL0.Exact and EL1.Exact @@ -5769,17 +5700,15 @@ ScalarEvolution::computeExitLimitFromCond(const Loop *L, !isa(BECount)) MaxBECount = BECount; - return ExitLimit(BECount, MaxBECount, NP); + return ExitLimit(BECount, MaxBECount); } if (BO->getOpcode() == Instruction::Or) { // Recurse on the operands of the or. bool EitherMayExit = L->contains(FBB); ExitLimit EL0 = computeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB, - ControlsExit && !EitherMayExit, - AllowPredicates); + ControlsExit && !EitherMayExit); ExitLimit EL1 = computeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB, - ControlsExit && !EitherMayExit, - AllowPredicates); + ControlsExit && !EitherMayExit); const SCEV *BECount = getCouldNotCompute(); const SCEV *MaxBECount = getCouldNotCompute(); if (EitherMayExit) { @@ -5806,25 +5735,14 @@ ScalarEvolution::computeExitLimitFromCond(const Loop *L, BECount = EL0.Exact; } - SCEVUnionPredicate NP; - NP.add(&EL0.Pred); - NP.add(&EL1.Pred); - return ExitLimit(BECount, MaxBECount, NP); + return ExitLimit(BECount, MaxBECount); } } // With an icmp, it may be feasible to compute an exact backedge-taken count. // Proceed to the next level to examine the icmp. - if (ICmpInst *ExitCondICmp = dyn_cast(ExitCond)) { - ExitLimit EL = - computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit); - if (EL.hasFullInfo() || !AllowPredicates) - return EL; - - // Try again, but use SCEV predicates this time. - return computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit, - /*AllowPredicates=*/true); - } + if (ICmpInst *ExitCondICmp = dyn_cast(ExitCond)) + return computeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit); // Check for a constant condition. These are normally stripped out by // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to @@ -5848,8 +5766,7 @@ ScalarEvolution::computeExitLimitFromICmp(const Loop *L, ICmpInst *ExitCond, BasicBlock *TBB, BasicBlock *FBB, - bool ControlsExit, - bool AllowPredicates) { + bool ControlsExit) { // If the condition was exit on true, convert the condition to exit on false ICmpInst::Predicate Cond; @@ -5906,8 +5823,7 @@ ScalarEvolution::computeExitLimitFromICmp(const Loop *L, switch (Cond) { case ICmpInst::ICMP_NE: { // while (X != Y) // Convert to: while (X-Y != 0) - ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit, - AllowPredicates); + ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit); if (EL.hasAnyInfo()) return EL; break; } @@ -5920,17 +5836,14 @@ ScalarEvolution::computeExitLimitFromICmp(const Loop *L, case ICmpInst::ICMP_SLT: case ICmpInst::ICMP_ULT: { // while (X < Y) bool IsSigned = Cond == ICmpInst::ICMP_SLT; - ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, ControlsExit, - AllowPredicates); + ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, ControlsExit); if (EL.hasAnyInfo()) return EL; break; } case ICmpInst::ICMP_SGT: case ICmpInst::ICMP_UGT: { // while (X > Y) bool IsSigned = Cond == ICmpInst::ICMP_SGT; - ExitLimit EL = - HowManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit, - AllowPredicates); + ExitLimit EL = HowManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit); if (EL.hasAnyInfo()) return EL; break; } @@ -6192,8 +6105,7 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit( unsigned BitWidth = getTypeSizeInBits(RHS->getType()); const SCEV *UpperBound = getConstant(getEffectiveSCEVType(RHS->getType()), BitWidth); - SCEVUnionPredicate P; - return ExitLimit(getCouldNotCompute(), UpperBound, P); + return ExitLimit(getCouldNotCompute(), UpperBound); } return getCouldNotCompute(); @@ -6970,9 +6882,7 @@ SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) { /// effectively V != 0. We know and take advantage of the fact that this /// expression only being used in a comparison by zero context. ScalarEvolution::ExitLimit -ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, - bool AllowPredicates) { - SCEVUnionPredicate P; +ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit) { // If the value is a constant if (const SCEVConstant *C = dyn_cast(V)) { // If the value is already zero, the branch will execute zero times. @@ -6981,12 +6891,6 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, } const SCEVAddRecExpr *AddRec = dyn_cast(V); - if (!AddRec && AllowPredicates) - // Try to make this an AddRec using runtime tests, in the first X - // iterations of this loop, where X is the SCEV expression found by the - // algorithm below. - AddRec = convertSCEVToAddRecWithPredicates(V, L, P); - if (!AddRec || AddRec->getLoop() != L) return getCouldNotCompute(); @@ -7011,7 +6915,7 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, // should not accept a root of 2. const SCEV *Val = AddRec->evaluateAtIteration(R1, *this); if (Val->isZero()) - return ExitLimit(R1, R1, P); // We found a quadratic root! + return R1; // We found a quadratic root! } } return getCouldNotCompute(); @@ -7068,7 +6972,7 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, else MaxBECount = getConstant(CountDown ? CR.getUnsignedMax() : -CR.getUnsignedMin()); - return ExitLimit(Distance, MaxBECount, P); + return ExitLimit(Distance, MaxBECount); } // As a special case, handle the instance where Step is a positive power of @@ -7121,9 +7025,7 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, auto *NarrowTy = IntegerType::get(getContext(), NarrowWidth); auto *WideTy = Distance->getType(); - const SCEV *Limit = - getZeroExtendExpr(getTruncateExpr(ModuloResult, NarrowTy), WideTy); - return ExitLimit(Limit, Limit, P); + return getZeroExtendExpr(getTruncateExpr(ModuloResult, NarrowTy), WideTy); } } @@ -7135,15 +7037,13 @@ ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit, if (ControlsExit && AddRec->hasNoSelfWrap()) { const SCEV *Exact = getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step); - return ExitLimit(Exact, Exact, P); + return ExitLimit(Exact, Exact); } // Then, try to solve the above equation provided that Start is constant. - if (const SCEVConstant *StartC = dyn_cast(Start)) { - const SCEV *E = SolveLinEquationWithOverflow( - StepC->getValue()->getValue(), -StartC->getValue()->getValue(), *this); - return ExitLimit(E, E, P); - } + if (const SCEVConstant *StartC = dyn_cast(Start)) + return SolveLinEquationWithOverflow(StepC->getAPInt(), -StartC->getAPInt(), + *this); return getCouldNotCompute(); } @@ -8586,18 +8486,12 @@ const SCEV *ScalarEvolution::computeBECount(const SCEV *Delta, const SCEV *Step, ScalarEvolution::ExitLimit ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned, - bool ControlsExit, bool AllowPredicates) { - SCEVUnionPredicate P; + bool ControlsExit) { // We handle only IV < Invariant if (!isLoopInvariant(RHS, L)) return getCouldNotCompute(); const SCEVAddRecExpr *IV = dyn_cast(LHS); - if (!IV && AllowPredicates) - // Try to make this an AddRec using runtime tests, in the first X - // iterations of this loop, where X is the SCEV expression found by the - // algorithm below. - IV = convertSCEVToAddRecWithPredicates(LHS, L, P); // Avoid weird loops if (!IV || IV->getLoop() != L || !IV->isAffine()) @@ -8666,24 +8560,18 @@ ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS, if (isa(MaxBECount)) MaxBECount = BECount; - return ExitLimit(BECount, MaxBECount, P); + return ExitLimit(BECount, MaxBECount); } ScalarEvolution::ExitLimit ScalarEvolution::HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned, - bool ControlsExit, bool AllowPredicates) { - SCEVUnionPredicate P; + bool ControlsExit) { // We handle only IV > Invariant if (!isLoopInvariant(RHS, L)) return getCouldNotCompute(); const SCEVAddRecExpr *IV = dyn_cast(LHS); - if (!IV && AllowPredicates) - // Try to make this an AddRec using runtime tests, in the first X - // iterations of this loop, where X is the SCEV expression found by the - // algorithm below. - IV = convertSCEVToAddRecWithPredicates(LHS, L, P); // Avoid weird loops if (!IV || IV->getLoop() != L || !IV->isAffine()) @@ -8754,7 +8642,7 @@ ScalarEvolution::HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS, if (isa(MaxBECount)) MaxBECount = BECount; - return ExitLimit(BECount, MaxBECount, P); + return ExitLimit(BECount, MaxBECount); } /// getNumIterationsInRange - Return the number of iterations of this loop that @@ -9458,8 +9346,6 @@ ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg) ValueExprMap(std::move(Arg.ValueExprMap)), WalkingBEDominatingConds(false), ProvingSplitPredicate(false), BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)), - PredicatedBackedgeTakenCounts( - std::move(Arg.PredicatedBackedgeTakenCounts)), ConstantEvolutionLoopExitValue( std::move(Arg.ConstantEvolutionLoopExitValue)), ValuesAtScopes(std::move(Arg.ValuesAtScopes)), @@ -9492,8 +9378,6 @@ ScalarEvolution::~ScalarEvolution() { // that a loop had multiple computable exits. for (auto &BTCI : BackedgeTakenCounts) BTCI.second.clear(); - for (auto &BTCI : PredicatedBackedgeTakenCounts) - BTCI.second.clear(); assert(PendingLoopPredicates.empty() && "isImpliedCond garbage"); assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!"); @@ -9536,20 +9420,6 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, OS << "Unpredictable max backedge-taken count. "; } - OS << "\n" - "Loop "; - L->getHeader()->printAsOperand(OS, /*PrintType=*/false); - OS << ": "; - - SCEVUnionPredicate Pred; - auto PBT = SE->getPredicatedBackedgeTakenCount(L, Pred); - if (!isa(PBT)) { - OS << "Predicated backedge-taken count is " << *PBT << "\n"; - OS << " Predicates:\n"; - Pred.print(OS, 4); - } else { - OS << "Unpredictable predicated backedge-taken count. "; - } OS << "\n"; } @@ -9834,20 +9704,16 @@ void ScalarEvolution::forgetMemoizedResults(const SCEV *S) { ExprValueMap.erase(S); HasRecMap.erase(S); - auto RemoveSCEVFromBackedgeMap = - [S, this](DenseMap &Map) { - for (auto I = Map.begin(), E = Map.end(); I != E;) { - BackedgeTakenInfo &BEInfo = I->second; - if (BEInfo.hasOperand(S, this)) { - BEInfo.clear(); - Map.erase(I++); - } else - ++I; - } - }; - - RemoveSCEVFromBackedgeMap(BackedgeTakenCounts); - RemoveSCEVFromBackedgeMap(PredicatedBackedgeTakenCounts); + for (DenseMap::iterator I = + BackedgeTakenCounts.begin(), E = BackedgeTakenCounts.end(); I != E; ) { + BackedgeTakenInfo &BEInfo = I->second; + if (BEInfo.hasOperand(S, this)) { + BEInfo.clear(); + BackedgeTakenCounts.erase(I++); + } + else + ++I; + } } typedef DenseMap VerifyMap; @@ -10262,7 +10128,7 @@ void SCEVUnionPredicate::add(const SCEVPredicate *N) { PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE, Loop &L) - : SE(SE), L(L), Generation(0), BackedgeCount(nullptr) {} + : SE(SE), L(L), Generation(0) {} const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) { const SCEV *Expr = SE.getSCEV(V); @@ -10283,15 +10149,6 @@ const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) { return NewSCEV; } -const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() { - if (!BackedgeCount) { - SCEVUnionPredicate BackedgePred; - BackedgeCount = SE.getPredicatedBackedgeTakenCount(&L, BackedgePred); - addPredicate(BackedgePred); - } - return BackedgeCount; -} - void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) { if (Preds.implies(&Pred)) return; @@ -10357,10 +10214,10 @@ const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(Value *V) { return New; } -PredicatedScalarEvolution::PredicatedScalarEvolution( - const PredicatedScalarEvolution &Init) - : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds), - Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) { +PredicatedScalarEvolution:: +PredicatedScalarEvolution(const PredicatedScalarEvolution &Init) : + RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L), Preds(Init.Preds), + Generation(Init.Generation) { for (auto I = Init.FlagsMap.begin(), E = Init.FlagsMap.end(); I != E; ++I) FlagsMap.insert(*I); } diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp index d9d2a8a8b81..4db3c7fb7f9 100644 --- a/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/lib/Analysis/ScalarEvolutionExpander.cpp @@ -2004,9 +2004,7 @@ Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR, assert(AR->isAffine() && "Cannot generate RT check for " "non-affine expression"); - SCEVUnionPredicate Pred; - const SCEV *ExitCount = - SE.getPredicatedBackedgeTakenCount(AR->getLoop(), Pred); + const SCEV *ExitCount = SE.getBackedgeTakenCount(AR->getLoop()); const SCEV *Step = AR->getStepRecurrence(SE); const SCEV *Start = AR->getStart(); diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index 61d9aced7bb..201e9e939b3 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2778,7 +2778,7 @@ Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) { IRBuilder<> Builder(L->getLoopPreheader()->getTerminator()); // Find the loop boundaries. ScalarEvolution *SE = PSE.getSE(); - const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount(); + const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(OrigLoop); assert(BackedgeTakenCount != SE->getCouldNotCompute() && "Invalid loop count"); @@ -4425,7 +4425,7 @@ bool LoopVectorizationLegality::canVectorize() { } // ScalarEvolution needs to be able to find the exit count. - const SCEV *ExitCount = PSE.getBackedgeTakenCount(); + const SCEV *ExitCount = PSE.getSE()->getBackedgeTakenCount(TheLoop); if (ExitCount == PSE.getSE()->getCouldNotCompute()) { emitAnalysis(VectorizationReport() << "could not determine number of loop iterations"); diff --git a/test/Analysis/ScalarEvolution/predicated-trip-count.ll b/test/Analysis/ScalarEvolution/predicated-trip-count.ll deleted file mode 100644 index 2db0a8b5777..00000000000 --- a/test/Analysis/ScalarEvolution/predicated-trip-count.ll +++ /dev/null @@ -1,109 +0,0 @@ -; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s - -target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" - -@A = weak global [1000 x i32] zeroinitializer, align 32 - -; The resulting predicate is i16 {0,+,1} , meanining -; that the resulting backedge expression will be valid for: -; (1 + (-1 smax %M)) <= MAX_INT16 -; -; At the limit condition for M (MAX_INT16 - 1) we have in the -; last iteration: -; i0 <- MAX_INT16 -; i0.ext <- MAX_INT16 -; -; and therefore no wrapping happend for i0 or i0.ext -; throughout the execution of the loop. The resulting predicated -; backedge taken count is correct. - -; CHECK: Classifying expressions for: @test1 -; CHECK: %i.0.ext = sext i16 %i.0 to i32 -; CHECK-NEXT: --> (sext i16 {0,+,1}<%bb3> to i32) -; CHECK: Loop %bb3: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %bb3: Unpredictable max backedge-taken count. -; CHECK-NEXT: Loop %bb3: Predicated backedge-taken count is (1 + (-1 smax %M)) -; CHECK-NEXT: Predicates: -; CHECK-NEXT: {0,+,1}<%bb3> Added Flags: -define void @test1(i32 %N, i32 %M) { -entry: - br label %bb3 - -bb: ; preds = %bb3 - %tmp = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i16 %i.0 ; [#uses=1] - store i32 123, i32* %tmp - %tmp2 = add i16 %i.0, 1 ; [#uses=1] - br label %bb3 - -bb3: ; preds = %bb, %entry - %i.0 = phi i16 [ 0, %entry ], [ %tmp2, %bb ] ; [#uses=3] - %i.0.ext = sext i16 %i.0 to i32 - %tmp3 = icmp sle i32 %i.0.ext, %M ; [#uses=1] - br i1 %tmp3, label %bb, label %bb5 - -bb5: ; preds = %bb3 - br label %return - -return: ; preds = %bb5 - ret void -} - -; The predicated backedge taken count is: -; (2 + (zext i16 %Start to i32) + ((-2 + (-1 * (sext i16 %Start to i32))) -; smax (-1 + (-1 * %M))) -; ) - -; -1 + (-1 * %M) <= (-2 + (-1 * (sext i16 %Start to i32)) -; The predicated backedge taken count is 0. -; From the IR, this is correct since we will bail out at the -; first iteration. - - -; * -1 + (-1 * %M) > (-2 + (-1 * (sext i16 %Start to i32)) -; or: %M < 1 + (sext i16 %Start to i32) -; -; The predicated backedge taken count is 1 + (zext i16 %Start to i32) - %M -; -; If %M >= MIN_INT + 1, this predicated backedge taken count would be correct (even -; without predicates). However, for %M < MIN_INT this would be an infinite loop. -; In these cases, the {%Start,+,-1} predicate would be false, as the -; final value of the expression {%Start,+,-1} expression (%M - 1) would not be -; representable as an i16. - -; There is also a limit case here where the value of %M is MIN_INT. In this case -; we still have an infinite loop, since icmp sge %x, MIN_INT will always return -; true. - -; CHECK: Classifying expressions for: @test2 - -; CHECK: %i.0.ext = sext i16 %i.0 to i32 -; CHECK-NEXT: --> (sext i16 {%Start,+,-1}<%bb3> to i32) -; CHECK: Loop %bb3: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %bb3: Unpredictable max backedge-taken count. -; CHECK-NEXT: Loop %bb3: Predicated backedge-taken count is (2 + (sext i16 %Start to i32) + ((-2 + (-1 * (sext i16 %Start to i32))) smax (-1 + (-1 * %M)))) -; CHECK-NEXT: Predicates: -; CHECK-NEXT: {%Start,+,-1}<%bb3> Added Flags: - -define void @test2(i32 %N, i32 %M, i16 %Start) { -entry: - br label %bb3 - -bb: ; preds = %bb3 - %tmp = getelementptr [1000 x i32], [1000 x i32]* @A, i32 0, i16 %i.0 ; [#uses=1] - store i32 123, i32* %tmp - %tmp2 = sub i16 %i.0, 1 ; [#uses=1] - br label %bb3 - -bb3: ; preds = %bb, %entry - %i.0 = phi i16 [ %Start, %entry ], [ %tmp2, %bb ] ; [#uses=3] - %i.0.ext = sext i16 %i.0 to i32 - %tmp3 = icmp sge i32 %i.0.ext, %M ; [#uses=1] - br i1 %tmp3, label %bb, label %bb5 - -bb5: ; preds = %bb3 - br label %return - -return: ; preds = %bb5 - ret void -} - diff --git a/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll b/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll deleted file mode 100644 index aba47f6c628..00000000000 --- a/test/Transforms/LoopVectorize/AArch64/backedge-overflow.ll +++ /dev/null @@ -1,166 +0,0 @@ -; RUN: opt -mtriple=aarch64--linux-gnueabi -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s -S | FileCheck %s - -; The following tests contain loops for which SCEV cannot determine the backedge -; taken count. This is because the backedge taken condition is produced by an -; icmp with one of the sides being a loop varying non-AddRec expression. -; However, there is a possibility to normalize this to an AddRec expression -; using SCEV predicates. This allows us to compute a 'guarded' backedge count. -; The Loop Vectorizer is able to version to loop in order to use this guarded -; backedge count and vectorize more loops. - - -; CHECK-LABEL: test_sge -; CHECK-LABEL: vector.scevcheck -; CHECK-LABEL: vector.body -define void @test_sge(i32* noalias %A, - i32* noalias %B, - i32* noalias %C, i32 %N) { -entry: - %cmp13 = icmp eq i32 %N, 0 - br i1 %cmp13, label %for.end, label %for.body.preheader - -for.body.preheader: - br label %for.body - -for.body: - %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ 0, %for.body.preheader ] - %indvars.next = add i16 %indvars.iv, 1 - %indvars.ext = zext i16 %indvars.iv to i32 - - %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.ext - %0 = load i32, i32* %arrayidx, align 4 - %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.ext - %1 = load i32, i32* %arrayidx3, align 4 - - %mul4 = mul i32 %1, %0 - - %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.ext - store i32 %mul4, i32* %arrayidx7, align 4 - - %exitcond = icmp sge i32 %indvars.ext, %N - br i1 %exitcond, label %for.end.loopexit, label %for.body - -for.end.loopexit: - br label %for.end - -for.end: - ret void -} - -; CHECK-LABEL: test_uge -; CHECK-LABEL: vector.scevcheck -; CHECK-LABEL: vector.body -define void @test_uge(i32* noalias %A, - i32* noalias %B, - i32* noalias %C, i32 %N, i32 %Offset) { -entry: - %cmp13 = icmp eq i32 %N, 0 - br i1 %cmp13, label %for.end, label %for.body.preheader - -for.body.preheader: - br label %for.body - -for.body: - %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ 0, %for.body.preheader ] - %indvars.next = add i16 %indvars.iv, 1 - - %indvars.ext = sext i16 %indvars.iv to i32 - %indvars.access = add i32 %Offset, %indvars.ext - - %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.access - %0 = load i32, i32* %arrayidx, align 4 - %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.access - %1 = load i32, i32* %arrayidx3, align 4 - - %mul4 = add i32 %1, %0 - - %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.access - store i32 %mul4, i32* %arrayidx7, align 4 - - %exitcond = icmp uge i32 %indvars.ext, %N - br i1 %exitcond, label %for.end.loopexit, label %for.body - -for.end.loopexit: - br label %for.end - -for.end: - ret void -} - -; CHECK-LABEL: test_ule -; CHECK-LABEL: vector.scevcheck -; CHECK-LABEL: vector.body -define void @test_ule(i32* noalias %A, - i32* noalias %B, - i32* noalias %C, i32 %N, - i16 %M) { -entry: - %cmp13 = icmp eq i32 %N, 0 - br i1 %cmp13, label %for.end, label %for.body.preheader - -for.body.preheader: - br label %for.body - -for.body: - %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ %M, %for.body.preheader ] - %indvars.next = sub i16 %indvars.iv, 1 - %indvars.ext = zext i16 %indvars.iv to i32 - - %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.ext - %0 = load i32, i32* %arrayidx, align 4 - %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.ext - %1 = load i32, i32* %arrayidx3, align 4 - - %mul4 = mul i32 %1, %0 - - %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.ext - store i32 %mul4, i32* %arrayidx7, align 4 - - %exitcond = icmp ule i32 %indvars.ext, %N - br i1 %exitcond, label %for.end.loopexit, label %for.body - -for.end.loopexit: - br label %for.end - -for.end: - ret void -} - -; CHECK-LABEL: test_sle -; CHECK-LABEL: vector.scevcheck -; CHECK-LABEL: vector.body -define void @test_sle(i32* noalias %A, - i32* noalias %B, - i32* noalias %C, i32 %N, - i16 %M) { -entry: - %cmp13 = icmp eq i32 %N, 0 - br i1 %cmp13, label %for.end, label %for.body.preheader - -for.body.preheader: - br label %for.body - -for.body: - %indvars.iv = phi i16 [ %indvars.next, %for.body ], [ %M, %for.body.preheader ] - %indvars.next = sub i16 %indvars.iv, 1 - %indvars.ext = sext i16 %indvars.iv to i32 - - %arrayidx = getelementptr inbounds i32, i32* %B, i32 %indvars.ext - %0 = load i32, i32* %arrayidx, align 4 - %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %indvars.ext - %1 = load i32, i32* %arrayidx3, align 4 - - %mul4 = mul i32 %1, %0 - - %arrayidx7 = getelementptr inbounds i32, i32* %A, i32 %indvars.ext - store i32 %mul4, i32* %arrayidx7, align 4 - - %exitcond = icmp sle i32 %indvars.ext, %N - br i1 %exitcond, label %for.end.loopexit, label %for.body - -for.end.loopexit: - br label %for.end - -for.end: - ret void -} diff --git a/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll b/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll index 740ff3682be..42ec3b3ffda 100644 --- a/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll +++ b/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll @@ -54,9 +54,8 @@ for.body: ; preds = %entry, %for.body %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !16 %0 = trunc i64 %indvars.iv to i32, !dbg !16 - %ld = load i32, i32* %arrayidx, align 4 store i32 %0, i32* %arrayidx, align 4, !dbg !16, !tbaa !18 - %cmp3 = icmp sle i32 %ld, %Length, !dbg !22 + %cmp3 = icmp sle i32 %0, %Length, !dbg !22 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !12 %1 = trunc i64 %indvars.iv.next to i32 %cmp = icmp slt i32 %1, %Length, !dbg !12