From 98b18d2ada84f15819b38979a863d7b62b235dc4 Mon Sep 17 00:00:00 2001 From: Vedant Kumar Date: Sat, 19 Jan 2019 02:38:47 +0000 Subject: [PATCH] [HotColdSplit] Mark inherently cold functions as such If an inherently cold function is found, mark it as cold. For now this means applying the `cold` and `minsize` attributes. As a drive-by, revisit and clean up the criteria for considering a function for splitting. Add tests. llvm-svn: 351623 --- lib/Transforms/IPO/HotColdSplitting.cpp | 59 ++++++++++++------- .../HotColdSplit/X86/do-not-split.ll | 31 ++++++++++ test/Transforms/HotColdSplit/addr-taken.ll | 27 +++++++++ test/Transforms/HotColdSplit/minsize.ll | 12 +++- 4 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 test/Transforms/HotColdSplit/addr-taken.ll diff --git a/lib/Transforms/IPO/HotColdSplitting.cpp b/lib/Transforms/IPO/HotColdSplitting.cpp index d6ac7183f77..49b59504738 100644 --- a/lib/Transforms/IPO/HotColdSplitting.cpp +++ b/lib/Transforms/IPO/HotColdSplitting.cpp @@ -153,15 +153,19 @@ static bool isProfitableToOutline(const BlockSequence &Region, return false; } -/// Mark \p F cold. Return true if it's changed. -static bool markEntireFunctionCold(Function &F) { +/// Mark \p F cold. Based on this assumption, also optimize it for minimum size. +/// Return true if the function is changed. +static bool markFunctionCold(Function &F) { assert(!F.hasFnAttribute(Attribute::OptimizeNone) && "Can't mark this cold"); bool Changed = false; + if (!F.hasFnAttribute(Attribute::Cold)) { + F.addFnAttr(Attribute::Cold); + Changed = true; + } if (!F.hasFnAttribute(Attribute::MinSize)) { F.addFnAttr(Attribute::MinSize); Changed = true; } - // TODO: Move this function into a cold section. return Changed; } @@ -175,6 +179,7 @@ public: bool run(Module &M); private: + bool isFunctionCold(const Function &F) const; bool shouldOutlineFrom(const Function &F) const; bool outlineColdRegions(Function &F, ProfileSummaryInfo &PSI, BlockFrequencyInfo *BFI, TargetTransformInfo &TTI, @@ -208,28 +213,29 @@ public: } // end anonymous namespace +/// Check whether \p F is inherently cold. +bool HotColdSplitting::isFunctionCold(const Function &F) const { + if (F.hasFnAttribute(Attribute::Cold)) + return true; + + if (F.getCallingConv() == CallingConv::Cold) + return true; + + if (PSI->isFunctionEntryCold(&F)) + return true; + + return false; +} + // Returns false if the function should not be considered for hot-cold split // optimization. bool HotColdSplitting::shouldOutlineFrom(const Function &F) const { - if (F.size() <= 2) - return false; - - // TODO: Consider only skipping functions marked `optnone` or `cold`. - - if (F.hasAddressTaken()) - return false; - if (F.hasFnAttribute(Attribute::AlwaysInline)) return false; if (F.hasFnAttribute(Attribute::NoInline)) return false; - if (F.getCallingConv() == CallingConv::Cold) - return false; - - if (PSI->isFunctionEntryCold(&F)) - return false; return true; } @@ -259,9 +265,7 @@ Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region, } CI->setIsNoInline(); - // Try to make the outlined code as small as possible on the assumption - // that it's cold. - markEntireFunctionCold(*OutF); + markFunctionCold(*OutF); LLVM_DEBUG(llvm::dbgs() << "Outlined Region: " << *OutF); ORE.emit([&]() { @@ -492,7 +496,7 @@ bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI, if (Region.isEntireFunctionCold()) { LLVM_DEBUG(dbgs() << "Entire function is cold\n"); - return markEntireFunctionCold(F); + return markFunctionCold(F); } // If this outlining region intersects with another, drop the new region. @@ -548,10 +552,25 @@ bool HotColdSplitting::run(Module &M) { for (auto It = M.begin(), End = M.end(); It != End; ++It) { Function &F = *It; + // Do not touch declarations. + if (F.isDeclaration()) + continue; + + // Do not modify `optnone` functions. + if (F.hasFnAttribute(Attribute::OptimizeNone)) + continue; + + // Detect inherently cold functions and mark them as such. + if (isFunctionCold(F)) { + Changed |= markFunctionCold(F); + continue; + } + if (!shouldOutlineFrom(F)) { LLVM_DEBUG(llvm::dbgs() << "Skipping " << F.getName() << "\n"); continue; } + LLVM_DEBUG(llvm::dbgs() << "Outlining in " << F.getName() << "\n"); DominatorTree DT(F); PostDomTree PDT(F); diff --git a/test/Transforms/HotColdSplit/X86/do-not-split.ll b/test/Transforms/HotColdSplit/X86/do-not-split.ll index 8622f03b2ed..448e63ab1de 100644 --- a/test/Transforms/HotColdSplit/X86/do-not-split.ll +++ b/test/Transforms/HotColdSplit/X86/do-not-split.ll @@ -43,6 +43,37 @@ entry: br i1 undef, label %if.then, label %if.end if.then: ; preds = %entry + call void @sink() + br label %if.end + +if.end: ; preds = %entry + ret void +} + +; Do not split `noinline` functions. +; CHECK-LABEL: @noinline_func +; CHECK-NOT: noinline_func.cold.1 +define void @noinline_func() noinline { +entry: + br i1 undef, label %if.then, label %if.end + +if.then: ; preds = %entry + call void @sink() + br label %if.end + +if.end: ; preds = %entry + ret void +} + +; Do not split `alwaysinline` functions. +; CHECK-LABEL: @alwaysinline_func +; CHECK-NOT: alwaysinline_func.cold.1 +define void @alwaysinline_func() alwaysinline { +entry: + br i1 undef, label %if.then, label %if.end + +if.then: ; preds = %entry + call void @sink() br label %if.end if.end: ; preds = %entry diff --git a/test/Transforms/HotColdSplit/addr-taken.ll b/test/Transforms/HotColdSplit/addr-taken.ll new file mode 100644 index 00000000000..f2f448c8a46 --- /dev/null +++ b/test/Transforms/HotColdSplit/addr-taken.ll @@ -0,0 +1,27 @@ +; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s + +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.14.0" + +; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]] +define void @foo() noreturn cold { + unreachable +} + +; CHECK: define {{.*}} @bar.cold.1{{.*}}#[[outlined_func_attr]] +define void @bar() { + br i1 undef, label %normal, label %exit + +normal: + unreachable + +exit: + ret void +} + +@take_addr_of_foo = global void ()* @foo +@take_addr_of_bar = global void ()* @bar + +; CHECK: attributes #[[outlined_func_attr]] = { +; CHECK-SAME: cold +; CHECK-SAME: minsize diff --git a/test/Transforms/HotColdSplit/minsize.ll b/test/Transforms/HotColdSplit/minsize.ll index 6955c5c79eb..36bd0340718 100644 --- a/test/Transforms/HotColdSplit/minsize.ll +++ b/test/Transforms/HotColdSplit/minsize.ll @@ -17,7 +17,15 @@ if.else: ret void } +; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]] +define void @foo() cold { + ret void +} + declare void @sink() cold -; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr:[0-9]+]] -; CHECK: attributes #[[outlined_func_attr]] = { {{.*}}minsize +; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr]] + +; CHECK: attributes #[[outlined_func_attr]] = { +; CHECK-SAME: cold +; CHECK-SAME: minsize