[HotColdSplit] Mark inherently cold functions as such

If an inherently cold function is found, mark it as cold. For now this
means applying the `cold` and `minsize` attributes.

As a drive-by, revisit and clean up the criteria for considering a
function for splitting. Add tests.

llvm-svn: 351623

lib/Transforms/IPO/HotColdSplitting.cpp

@@ -153,15 +153,19 @@ static bool isProfitableToOutline(const BlockSequence &Region,
   return false;
 }
 
-/// Mark \p F cold. Return true if it's changed.
-static bool markEntireFunctionCold(Function &F) {
+/// Mark \p F cold. Based on this assumption, also optimize it for minimum size.
+/// Return true if the function is changed.
+static bool markFunctionCold(Function &F) {
   assert(!F.hasFnAttribute(Attribute::OptimizeNone) && "Can't mark this cold");
   bool Changed = false;
+  if (!F.hasFnAttribute(Attribute::Cold)) {
+    F.addFnAttr(Attribute::Cold);
+    Changed = true;
+  }
   if (!F.hasFnAttribute(Attribute::MinSize)) {
     F.addFnAttr(Attribute::MinSize);
     Changed = true;
   }
-  // TODO: Move this function into a cold section.
   return Changed;
 }
 
@@ -175,6 +179,7 @@ public:
   bool run(Module &M);
 
 private:
+  bool isFunctionCold(const Function &F) const;
   bool shouldOutlineFrom(const Function &F) const;
   bool outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
                           BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
@@ -208,28 +213,29 @@ public:
 
 } // end anonymous namespace
 
+/// Check whether \p F is inherently cold.
+bool HotColdSplitting::isFunctionCold(const Function &F) const {
+  if (F.hasFnAttribute(Attribute::Cold))
+    return true;
+
+  if (F.getCallingConv() == CallingConv::Cold)
+    return true;
+
+  if (PSI->isFunctionEntryCold(&F))
+    return true;
+
+  return false;
+}
+
 // Returns false if the function should not be considered for hot-cold split
 // optimization.
 bool HotColdSplitting::shouldOutlineFrom(const Function &F) const {
-  if (F.size() <= 2)
-    return false;
-
-  // TODO: Consider only skipping functions marked `optnone` or `cold`.
-
-  if (F.hasAddressTaken())
-    return false;
-
   if (F.hasFnAttribute(Attribute::AlwaysInline))
     return false;
 
   if (F.hasFnAttribute(Attribute::NoInline))
     return false;
 
-  if (F.getCallingConv() == CallingConv::Cold)
-    return false;
-
-  if (PSI->isFunctionEntryCold(&F))
-    return false;
   return true;
 }
 
@@ -259,9 +265,7 @@ Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region,
     }
     CI->setIsNoInline();
 
-    // Try to make the outlined code as small as possible on the assumption
-    // that it's cold.
-    markEntireFunctionCold(*OutF);
+    markFunctionCold(*OutF);
 
     LLVM_DEBUG(llvm::dbgs() << "Outlined Region: " << *OutF);
     ORE.emit([&]() {
@@ -492,7 +496,7 @@ bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
 
     if (Region.isEntireFunctionCold()) {
       LLVM_DEBUG(dbgs() << "Entire function is cold\n");
-      return markEntireFunctionCold(F);
+      return markFunctionCold(F);
     }
 
     // If this outlining region intersects with another, drop the new region.
@@ -548,10 +552,25 @@ bool HotColdSplitting::run(Module &M) {
   for (auto It = M.begin(), End = M.end(); It != End; ++It) {
     Function &F = *It;
 
+    // Do not touch declarations.
+    if (F.isDeclaration())
+      continue;
+
+    // Do not modify `optnone` functions.
+    if (F.hasFnAttribute(Attribute::OptimizeNone))
+      continue;
+
+    // Detect inherently cold functions and mark them as such.
+    if (isFunctionCold(F)) {
+      Changed |= markFunctionCold(F);
+      continue;
+    }
+
     if (!shouldOutlineFrom(F)) {
       LLVM_DEBUG(llvm::dbgs() << "Skipping " << F.getName() << "\n");
       continue;
     }
+
     LLVM_DEBUG(llvm::dbgs() << "Outlining in " << F.getName() << "\n");
     DominatorTree DT(F);
     PostDomTree PDT(F);

test/Transforms/HotColdSplit/X86/do-not-split.ll

@@ -43,6 +43,37 @@ entry:
   br i1 undef, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Do not split `noinline` functions.
+; CHECK-LABEL: @noinline_func
+; CHECK-NOT: noinline_func.cold.1
+define void @noinline_func() noinline {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Do not split `alwaysinline` functions.
+; CHECK-LABEL: @alwaysinline_func
+; CHECK-NOT: alwaysinline_func.cold.1
+define void @alwaysinline_func() alwaysinline {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
   br label %if.end
 
 if.end:                                           ; preds = %entry

test/Transforms/HotColdSplit/addr-taken.ll

@@ -0,0 +1,27 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]]
+define void @foo() noreturn cold {
+  unreachable
+}
+
+; CHECK: define {{.*}} @bar.cold.1{{.*}}#[[outlined_func_attr]]
+define void @bar() {
+  br i1 undef, label %normal, label %exit
+
+normal:
+  unreachable
+
+exit:
+  ret void
+}
+
+@take_addr_of_foo = global void ()* @foo
+@take_addr_of_bar = global void ()* @bar
+
+; CHECK: attributes #[[outlined_func_attr]] = {
+; CHECK-SAME: cold
+; CHECK-SAME: minsize

test/Transforms/HotColdSplit/minsize.ll

@@ -17,7 +17,15 @@ if.else:
   ret void
 }
 
+; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]]
+define void @foo() cold {
+  ret void
+}
+
 declare void @sink() cold
 
-; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr:[0-9]+]]
-; CHECK: attributes #[[outlined_func_attr]] = { {{.*}}minsize
+; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr]]
+
+; CHECK: attributes #[[outlined_func_attr]] = {
+; CHECK-SAME: cold
+; CHECK-SAME: minsize