[Coroutines][5/6] Add coroutine passes to pipeline

Summary:
Depends on https://reviews.llvm.org/D71901.

The fifth in a series of patches that ports the LLVM coroutines passes
to the new pass manager infrastructure.

The first 4 patches allow users to run coroutine passes by invoking, for
example `opt -passes=coro-early`. However, most of LLVM's tests for
coroutines use an option, `opt -enable-coroutines`, which adds all 4
coroutine passes to the appropriate legacy pass manager extension points.
This patch does the same, but using the new pass manager: when
coroutine features are enabled and the new pass manager is being used,
this adds the new-pass-manager-compliant coroutine passes to the pass
builder's pipeline.

This allows us to run all coroutine tests using the new pass manager
(besides those that use the coroutine retcon ABI used by the Swift
compiler, which is not yet supported in the new pass manager).

Reviewers: GorNishanov, lewissbaker, chandlerc, junparser, wenlei

Subscribers: wenlei, EricWF, Prazek, hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D71902

include/llvm/Passes/PassBuilder.h

@@ -92,6 +92,12 @@ public:
   /// is that of the flag: `-forget-scev-loop-unroll`.
   bool ForgetAllSCEVInLoopUnroll;
 
+  /// Tuning option to enable/disable coroutine intrinsic lowering. Its default
+  /// value is false. Frontends such as Clang may enable this conditionally. For
+  /// example, Clang enables this option if the flags `-std=c++2a` or above, or
+  /// `-fcoroutines-ts`, have been specified.
+  bool Coroutines;
+
   /// Tuning option to cap the number of calls to retrive clobbering accesses in
   /// MemorySSA, in LICM.
   unsigned LicmMssaOptCap;

lib/Passes/PassBuilder.cpp

@@ -242,6 +242,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
   SLPVectorization = RunSLPVectorization;
   LoopUnrolling = true;
   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
+  Coroutines = false;
   LicmMssaOptCap = SetLicmMssaOptCap;
   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
 }
@@ -721,6 +722,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   EarlyFPM.addPass(SROA());
   EarlyFPM.addPass(EarlyCSEPass());
   EarlyFPM.addPass(LowerExpectIntrinsicPass());
+  if (PTO.Coroutines)
+    EarlyFPM.addPass(CoroEarlyPass());
   if (Level == OptimizationLevel::O3)
     EarlyFPM.addPass(CallSiteSplittingPass());
 
@@ -844,6 +847,11 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
 
   MainCGPipeline.addPass(AttributorCGSCCPass());
 
+  if (PTO.Coroutines) {
+    MainCGPipeline.addPass(CoroSplitPass());
+    MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
+  }
+
   // Now deduce any function attributes based in the current code.
   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
 
@@ -1046,6 +1054,9 @@ ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
   // inserting redundancies into the program. This even includes SimplifyCFG.
   OptimizePM.addPass(SpeculateAroundPHIsPass());
 
+  if (PTO.Coroutines)
+    OptimizePM.addPass(CoroCleanupPass());
+
   for (auto &C : OptimizerLastEPCallbacks)
     C(OptimizePM, Level);
 
@@ -1129,6 +1140,12 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
   // Reduce the size of the IR as much as possible.
   MPM.addPass(GlobalOptPass());
 
+  // Module simplification splits coroutines, but does not fully clean up
+  // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
+  // on these, we schedule the cleanup here.
+  if (PTO.Coroutines)
+    MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
+
   return MPM;
 }
 
@@ -1965,6 +1982,20 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM,
             /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
             /* IsCS */ false, PGOOpt->ProfileFile,
             PGOOpt->ProfileRemappingFile);
+
+      // For IR that makes use of coroutines intrinsics, coroutine passes must
+      // be run, even at -O0.
+      if (PTO.Coroutines) {
+        MPM.addPass(createModuleToFunctionPassAdaptor(CoroEarlyPass()));
+
+        CGSCCPassManager CGPM(DebugLogging);
+        CGPM.addPass(CoroSplitPass());
+        CGPM.addPass(createCGSCCToFunctionPassAdaptor(CoroElidePass()));
+        MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
+
+        MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
+      }
+
       // Do nothing else at all!
       return Error::success();
     }

test/Transforms/Coroutines/ArgAddr.ll

@@ -1,6 +1,7 @@
 ; Need to move users of allocas that were moved into the coroutine frame after
 ; coro.begin.
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define nonnull i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/coro-cleanup.ll

@@ -1,5 +1,6 @@
 ; Make sure that all library helper coro intrinsics are lowered.
 ; RUN: opt < %s -O0 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -passes='default<O0>' -enable-coroutines -S | FileCheck %s
 
 ; CHECK-LABEL: @uses_library_support_coro_intrinsics(
 ; CHECK-NOT:     @llvm.coro

test/Transforms/Coroutines/coro-split-01.ll

@@ -1,5 +1,6 @@
 ; Tests that a coroutine is split, inlined into the caller and devirtualized.
 ; RUN: opt < %s -S -enable-coroutines -O2 | FileCheck %s
+; RUN: opt < %s -S -enable-coroutines -passes='default<O2>' | FileCheck %s
 
 define i8* @f() {
 entry:

test/Transforms/Coroutines/ex0.ll

@@ -1,5 +1,6 @@
 ; First example from Doc/Coroutines.rst (two block loop)
 ; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck %s
+; RUN: opt < %s -enable-coroutines -aa-pipeline=basic-aa -passes='default<O2>' -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/ex1.ll

@@ -1,5 +1,6 @@
 ; First example from Doc/Coroutines.rst (one block loop)
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/ex2.ll

@@ -1,5 +1,6 @@
 ; Second example from Doc/Coroutines.rst (custom alloc and free functions)
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/ex3.ll

@@ -1,5 +1,6 @@
 ; Third example from Doc/Coroutines.rst (two suspend points)
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/ex4.ll

@@ -1,5 +1,6 @@
 ; Fourth example from Doc/Coroutines.rst (coroutine promise)
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/ex5.ll

@@ -1,5 +1,6 @@
 ; Fifth example from Doc/Coroutines.rst (final suspend)
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/phi-coro-end.ll

@@ -1,5 +1,6 @@
 ; Verify that we correctly handle suspend when the coro.end block contains phi
 ; RUN: opt < %s -O2 -enable-coroutines -S | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='default<O2>' -enable-coroutines -S | FileCheck %s
 
 define i8* @f(i32 %n) {
 entry:

test/Transforms/Coroutines/restart-trigger.ll

@@ -4,6 +4,10 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -S -O0 -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s
 ; RUN: opt < %s -S -O1 -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s
+; The following tests use the new pass manager, and verify that the coroutine
+; passes re-run the CGSCC pipeline.
+; RUN: opt < %s -S -passes='default<O0>' -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s
+; RUN: opt < %s -S -passes='default<O1>' -enable-coroutines -debug-only=coro-split 2>&1 | FileCheck %s
 
 ; CHECK:      CoroSplit: Processing coroutine 'f' state: 0
 ; CHECK-NEXT: CoroSplit: Processing coroutine 'f' state: 1

test/Transforms/Coroutines/smoketest.ll

@@ -1,6 +1,7 @@
 ; Test that all coroutine passes run in the correct order at all optimization
 ; levels and -enable-coroutines adds coroutine passes to the pipeline.
 ;
+; Legacy pass manager:
 ; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O0 2>&1 | FileCheck %s
 ; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O1 2>&1 | FileCheck %s
 ; RUN: opt < %s -disable-output -enable-coroutines -debug-pass=Arguments -O2 2>&1 | FileCheck %s
@@ -9,6 +10,18 @@
 ; RUN:     -coro-early -coro-split -coro-elide -coro-cleanup 2>&1 | FileCheck %s
 ; RUN: opt < %s -disable-output -debug-pass=Arguments 2>&1 \
 ; RUN:     | FileCheck %s -check-prefix=NOCORO
+; New pass manager:
+; RUN: opt < %s -disable-output -passes='default<O0>' -enable-coroutines \
+; RUN:     -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM
+; RUN: opt < %s -disable-output -passes='default<O1>' -enable-coroutines \
+; RUN:     -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM
+; RUN: opt < %s -disable-output -passes='default<O2>' -enable-coroutines \
+; RUN:     -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM
+; RUN: opt < %s -disable-output -passes='default<O3>' -enable-coroutines \
+; RUN:     -debug-pass-manager 2>&1 | FileCheck %s -check-prefix=NEWPM
+; RUN: opt < %s -disable-output -debug-pass-manager \
+; RUN:     -passes='function(coro-early),cgscc(coro-split),function(coro-elide,coro-cleanup)' 2>&1 \
+; RUN:     | FileCheck %s -check-prefix=NEWPM
 
 ; CHECK: coro-early
 ; CHECK: coro-split
@@ -20,6 +33,11 @@
 ; NOCORO-NOT: coro-elide
 ; NOCORO-NOT: coro-cleanup
 
+; NEWPM: CoroEarlyPass
+; NEWPM: CoroSplitPass
+; NEWPM: CoroElidePass
+; NEWPM: CoroCleanupPass
+
 define void @foo() {
   ret void
 }

tools/opt/NewPMDriver.cpp

@@ -214,7 +214,7 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
                            bool ShouldPreserveAssemblyUseListOrder,
                            bool ShouldPreserveBitcodeUseListOrder,
                            bool EmitSummaryIndex, bool EmitModuleHash,
-                           bool EnableDebugify) {
+                           bool EnableDebugify, bool Coroutines) {
   bool VerifyEachPass = VK == VK_VerifyEachPass;
 
   Optional<PGOOptions> P;
@@ -259,7 +259,9 @@ bool llvm::runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
   StandardInstrumentations SI;
   SI.registerCallbacks(PIC);
 
-  PassBuilder PB(TM, PipelineTuningOptions(), P, &PIC);
+  PipelineTuningOptions PTO;
+  PTO.Coroutines = Coroutines;
+  PassBuilder PB(TM, PTO, P, &PIC);
   registerEPCallbacks(PB, VerifyEachPass, DebugPM);
 
   // Load requested pass plugins and let them register pass builder callbacks

tools/opt/NewPMDriver.h

@@ -64,7 +64,7 @@ bool runPassPipeline(StringRef Arg0, Module &M, TargetMachine *TM,
                      bool ShouldPreserveAssemblyUseListOrder,
                      bool ShouldPreserveBitcodeUseListOrder,
                      bool EmitSummaryIndex, bool EmitModuleHash,
-                     bool EnableDebugify);
+                     bool EnableDebugify, bool Coroutines);
 } // namespace llvm
 
 #endif

tools/opt/opt.cpp

@@ -716,7 +716,7 @@ int main(int argc, char **argv) {
                            RemarksFile.get(), PassPipeline, OK, VK,
                            PreserveAssemblyUseListOrder,
                            PreserveBitcodeUseListOrder, EmitSummaryIndex,
-                           EmitModuleHash, EnableDebugify)
+                           EmitModuleHash, EnableDebugify, Coroutines)
                ? 0
                : 1;
   }