diff --git a/include/llvm/Transforms/Utils/LoopUtils.h b/include/llvm/Transforms/Utils/LoopUtils.h index 561f9488062..0397eb95e76 100644 --- a/include/llvm/Transforms/Utils/LoopUtils.h +++ b/include/llvm/Transforms/Utils/LoopUtils.h @@ -362,6 +362,14 @@ private: BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA); +/// Ensure that all exit blocks of the loop are dedicated exits. +/// +/// For any loop exit block with non-loop predecessors, we split the loop +/// predecessors to use a dedicated loop exit block. We update the dominator +/// tree and loop info if provided, and will preserve LCSSA if requested. +bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, + bool PreserveLCSSA); + /// Ensures LCSSA form for every instruction from the Worklist in the scope of /// innermost containing loop. /// diff --git a/lib/Transforms/Utils/LoopSimplify.cpp b/lib/Transforms/Utils/LoopSimplify.cpp index f3db278ef1e..e21e34df8de 100644 --- a/lib/Transforms/Utils/LoopSimplify.cpp +++ b/lib/Transforms/Utils/LoopSimplify.cpp @@ -72,7 +72,6 @@ using namespace llvm; #define DEBUG_TYPE "loop-simplify" -STATISTIC(NumInserted, "Number of pre-header or exit blocks inserted"); STATISTIC(NumNested , "Number of nested loops split out"); // If the block isn't already, move the new block to right after some 'outside @@ -152,37 +151,6 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, DominatorTree *DT, return PreheaderBB; } -/// \brief Ensure that the loop preheader dominates all exit blocks. -/// -/// This method is used to split exit blocks that have predecessors outside of -/// the loop. -static BasicBlock *rewriteLoopExitBlock(Loop *L, BasicBlock *Exit, - DominatorTree *DT, LoopInfo *LI, - bool PreserveLCSSA) { - SmallVector LoopBlocks; - for (pred_iterator I = pred_begin(Exit), E = pred_end(Exit); I != E; ++I) { - BasicBlock *P = *I; - if (L->contains(P)) { - // Don't do this if the loop is exited via an indirect branch. - if (isa(P->getTerminator())) return nullptr; - - LoopBlocks.push_back(P); - } - } - - assert(!LoopBlocks.empty() && "No edges coming in from outside the loop?"); - BasicBlock *NewExitBB = nullptr; - - NewExitBB = SplitBlockPredecessors(Exit, LoopBlocks, ".loopexit", DT, LI, - PreserveLCSSA); - if (!NewExitBB) - return nullptr; - - DEBUG(dbgs() << "LoopSimplify: Creating dedicated exit block " - << NewExitBB->getName() << "\n"); - return NewExitBB; -} - /// Add the specified block, and all of its predecessors, to the specified set, /// if it's not already in there. Stop predecessor traversal when we reach /// StopBlock. @@ -346,16 +314,7 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader, // Split edges to exit blocks from the inner loop, if they emerged in the // process of separating the outer one. - SmallVector ExitBlocks; - L->getExitBlocks(ExitBlocks); - SmallSetVector ExitBlockSet(ExitBlocks.begin(), - ExitBlocks.end()); - for (BasicBlock *ExitBlock : ExitBlockSet) { - if (any_of(predecessors(ExitBlock), - [L](BasicBlock *BB) { return !L->contains(BB); })) { - rewriteLoopExitBlock(L, ExitBlock, DT, LI, PreserveLCSSA); - } - } + formDedicatedExitBlocks(L, DT, LI, PreserveLCSSA); if (PreserveLCSSA) { // Fix LCSSA form for L. Some values, which previously were only used inside @@ -563,29 +522,16 @@ ReprocessLoop: BasicBlock *Preheader = L->getLoopPreheader(); if (!Preheader) { Preheader = InsertPreheaderForLoop(L, DT, LI, PreserveLCSSA); - if (Preheader) { - ++NumInserted; + if (Preheader) Changed = true; - } } // Next, check to make sure that all exit nodes of the loop only have // predecessors that are inside of the loop. This check guarantees that the // loop preheader/header will dominate the exit blocks. If the exit block has // predecessors from outside of the loop, split the edge now. - SmallVector ExitBlocks; - L->getExitBlocks(ExitBlocks); - - SmallSetVector ExitBlockSet(ExitBlocks.begin(), - ExitBlocks.end()); - for (BasicBlock *ExitBlock : ExitBlockSet) { - if (any_of(predecessors(ExitBlock), - [L](BasicBlock *BB) { return !L->contains(BB); })) { - rewriteLoopExitBlock(L, ExitBlock, DT, LI, PreserveLCSSA); - ++NumInserted; - Changed = true; - } - } + if (formDedicatedExitBlocks(L, DT, LI, PreserveLCSSA)) + Changed = true; // If the header has more than two predecessors at this point (from the // preheader and from multiple backedges), we must adjust the loop. @@ -614,10 +560,8 @@ ReprocessLoop: // insert a new block that all backedges target, then make it jump to the // loop header. LoopLatch = insertUniqueBackedgeBlock(L, Preheader, DT, LI); - if (LoopLatch) { - ++NumInserted; + if (LoopLatch) Changed = true; - } } const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); @@ -645,7 +589,22 @@ ReprocessLoop: // loop-invariant instructions out of the way to open up more // opportunities, and the disadvantage of having the responsibility // to preserve dominator information. - if (ExitBlockSet.size() == 1) { + auto HasUniqueExitBlock = [&]() { + BasicBlock *UniqueExit = nullptr; + for (auto *ExitingBB : ExitingBlocks) + for (auto *SuccBB : successors(ExitingBB)) { + if (L->contains(SuccBB)) + continue; + + if (!UniqueExit) + UniqueExit = SuccBB; + else if (UniqueExit != SuccBB) + return false; + } + + return true; + }; + if (HasUniqueExitBlock()) { for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) { BasicBlock *ExitingBlock = ExitingBlocks[i]; if (!ExitingBlock->getSinglePredecessor()) continue; diff --git a/lib/Transforms/Utils/LoopUtils.cpp b/lib/Transforms/Utils/LoopUtils.cpp index e545bc0eb36..0ed33945ef4 100644 --- a/lib/Transforms/Utils/LoopUtils.cpp +++ b/lib/Transforms/Utils/LoopUtils.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/LoopUtils.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/GlobalsModRef.h" @@ -29,6 +30,7 @@ #include "llvm/IR/ValueHandle.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" using namespace llvm; using namespace llvm::PatternMatch; @@ -922,6 +924,69 @@ bool InductionDescriptor::isInductionPHI(PHINode *Phi, const Loop *TheLoop, return true; } +bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, + bool PreserveLCSSA) { + bool Changed = false; + + // We re-use a vector for the in-loop predecesosrs. + SmallVector InLoopPredecessors; + + auto RewriteExit = [&](BasicBlock *BB) { + assert(InLoopPredecessors.empty() && + "Must start with an empty predecessors list!"); + auto Cleanup = make_scope_exit([&] { InLoopPredecessors.clear(); }); + + // See if there are any non-loop predecessors of this exit block and + // keep track of the in-loop predecessors. + bool IsDedicatedExit = true; + for (auto *PredBB : predecessors(BB)) + if (L->contains(PredBB)) { + if (isa(PredBB->getTerminator())) + // We cannot rewrite exiting edges from an indirectbr. + return false; + + InLoopPredecessors.push_back(PredBB); + } else { + IsDedicatedExit = false; + } + + assert(!InLoopPredecessors.empty() && "Must have *some* loop predecessor!"); + + // Nothing to do if this is already a dedicated exit. + if (IsDedicatedExit) + return false; + + auto *NewExitBB = SplitBlockPredecessors( + BB, InLoopPredecessors, ".loopexit", DT, LI, PreserveLCSSA); + + if (!NewExitBB) + DEBUG(dbgs() << "WARNING: Can't create a dedicated exit block for loop: " + << *L << "\n"); + else + DEBUG(dbgs() << "LoopSimplify: Creating dedicated exit block " + << NewExitBB->getName() << "\n"); + return true; + }; + + // Walk the exit blocks directly rather than building up a data structure for + // them, but only visit each one once. + SmallPtrSet Visited; + for (auto *BB : L->blocks()) + for (auto *SuccBB : successors(BB)) { + // We're looking for exit blocks so skip in-loop successors. + if (L->contains(SuccBB)) + continue; + + // Visit each exit block exactly once. + if (!Visited.insert(SuccBB).second) + continue; + + Changed |= RewriteExit(SuccBB); + } + + return Changed; +} + /// \brief Returns the instructions that use values defined in the loop. SmallVector llvm::findDefsUsedOutsideOfLoop(Loop *L) { SmallVector UsedOutside; diff --git a/test/Transforms/LoopSimplify/basictest.ll b/test/Transforms/LoopSimplify/basictest.ll index 191e2b2797c..e5fb7b9907b 100644 --- a/test/Transforms/LoopSimplify/basictest.ll +++ b/test/Transforms/LoopSimplify/basictest.ll @@ -153,3 +153,84 @@ non_dedicated_exit: ; CHECK: non_dedicated_exit: ; CHECK-NEXT: ret void } + +; Check that we form what dedicated exits we can even when some exits are +; reached via indirectbr which precludes forming dedicated exits. +define void @test_form_some_dedicated_exits_despite_indirectbr(i8 %a, i8* %ptr, i8** %addr.ptr) { +; CHECK-LABEL: define void @test_form_some_dedicated_exits_despite_indirectbr( +entry: + switch i8 %a, label %loop.ph [ + i8 0, label %exit.a + i8 1, label %exit.b + i8 2, label %exit.c + ] +; CHECK: entry: +; CHECK-NEXT: switch i8 %a, label %loop.ph [ +; CHECK-NEXT: i8 0, label %exit.a +; CHECK-NEXT: i8 1, label %exit.b +; CHECK-NEXT: i8 2, label %exit.c +; CHECK-NEXT: ] + +loop.ph: + br label %loop.header +; CHECK: loop.ph: +; CHECK-NEXT: br label %loop.header + +loop.header: + %addr1 = load volatile i8*, i8** %addr.ptr + indirectbr i8* %addr1, [label %loop.body1, label %exit.a] +; CHECK: loop.header: +; CHECK-NEXT: %[[ADDR1:.*]] = load volatile i8*, i8** %addr.ptr +; CHECK-NEXT: indirectbr i8* %[[ADDR1]], [label %loop.body1, label %exit.a] + +loop.body1: + %b = load volatile i8, i8* %ptr + switch i8 %b, label %loop.body2 [ + i8 0, label %exit.a + i8 1, label %exit.b + i8 2, label %exit.c + ] +; CHECK: loop.body1: +; CHECK-NEXT: %[[B:.*]] = load volatile i8, i8* %ptr +; CHECK-NEXT: switch i8 %[[B]], label %loop.body2 [ +; CHECK-NEXT: i8 0, label %exit.a +; CHECK-NEXT: i8 1, label %[[LOOPEXIT:.*]] +; CHECK-NEXT: i8 2, label %exit.c +; CHECK-NEXT: ] + +loop.body2: + %addr2 = load volatile i8*, i8** %addr.ptr + indirectbr i8* %addr2, [label %loop.backedge, label %exit.c] +; CHECK: loop.body2: +; CHECK-NEXT: %[[ADDR2:.*]] = load volatile i8*, i8** %addr.ptr +; CHECK-NEXT: indirectbr i8* %[[ADDR2]], [label %loop.backedge, label %exit.c] + +loop.backedge: + br label %loop.header +; CHECK: loop.backedge: +; CHECK-NEXT: br label %loop.header + +exit.a: + ret void +; Check that there isn't a split loop exit. +; CHECK-NOT: br label %exit.a +; +; CHECK: exit.a: +; CHECK-NEXT: ret void + +exit.b: + ret void +; CHECK: [[LOOPEXIT]]: +; CHECK-NEXT: br label %exit.b +; +; CHECK: exit.b: +; CHECK-NEXT: ret void + +exit.c: + ret void +; Check that there isn't a split loop exit. +; CHECK-NOT: br label %exit.c +; +; CHECK: exit.c: +; CHECK-NEXT: ret void +} diff --git a/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll b/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll index a35596aff11..d115787b6ea 100644 --- a/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll +++ b/test/Transforms/LoopUnswitch/2011-11-18-SimpleSwitch.ll @@ -2,7 +2,6 @@ ; RUN: opt -loop-unswitch -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s ; RUN: opt -S -loop-unswitch -verify-loop-info -verify-dom-info < %s | FileCheck %s -; STATS: 1 loop-simplify - Number of pre-header or exit blocks inserted ; STATS: 2 loop-unswitch - Number of switches unswitched ; CHECK: %1 = icmp eq i32 %c, 1 diff --git a/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll b/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll index 393dd5c313a..c4e8d6f8898 100644 --- a/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll +++ b/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches-Threshold.ll @@ -2,7 +2,6 @@ ; RUN: opt -loop-unswitch -loop-unswitch-threshold 13 -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s ; RUN: opt -S -loop-unswitch -loop-unswitch-threshold 13 -verify-loop-info -verify-dom-info < %s | FileCheck %s -; STATS: 1 loop-simplify - Number of pre-header or exit blocks inserted ; STATS: 1 loop-unswitch - Number of switches unswitched ; ModuleID = '../llvm/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll' diff --git a/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll b/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll index 20f03c987eb..18e544d86ca 100644 --- a/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll +++ b/test/Transforms/LoopUnswitch/2011-11-18-TwoSwitches.ll @@ -2,7 +2,6 @@ ; RUN: opt -loop-unswitch -loop-unswitch-threshold 1000 -disable-output -stats -info-output-file - < %s | FileCheck --check-prefix=STATS %s ; RUN: opt -S -loop-unswitch -loop-unswitch-threshold 1000 -verify-loop-info -verify-dom-info < %s | FileCheck %s -; STATS: 1 loop-simplify - Number of pre-header or exit blocks inserted ; STATS: 3 loop-unswitch - Number of switches unswitched ; CHECK: %1 = icmp eq i32 %c, 1