RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 10:42:39 +01:00
Code Issues Projects Releases Wiki Activity

Add call branch annotation for ICP promoted direct call in SamplePGO mode.

Browse Source 
Summary: SamplePGO uses branch_weight annotation to represent callsite hotness. When ICP promotes an indirect call to direct call, we need to make sure the direct call is annotated with branch_weight in SamplePGO mode, so that downstream function inliner can use hot callsite heuristic.

Reviewers: davidxl, eraman, xur

Reviewed By: davidxl, xur

Subscribers: mehdi_amini, llvm-commits

Differential Revision: https://reviews.llvm.org/D30282

llvm-svn: 296028
This commit is contained in:
Dehao Chen 2017-02-23 22:15:18 +00:00
parent 82b48756cf
commit 70e2697ba1
8 changed files with 69 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/llvm/Passes/PassBuilder.h
View File

@ -32,6 +32,7 @@ struct PGOOptions {
std::string ProfileGenFile = "";
std::string ProfileUseFile = "";
bool RunProfileGen = false;
bool SamplePGO = false;
};
/// \brief This class provides access to building LLVM's passes.

8
include/llvm/Transforms/Instrumentation.h
View File

@ -86,7 +86,8 @@ ModulePass *createGCOVProfilerPass(const GCOVOptions &Options =
ModulePass *createPGOInstrumentationGenLegacyPass();
ModulePass *
createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""));
ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false);
ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false,
bool SamplePGO = false);
// Helper function to check if it is legal to promote indirect call \p Inst
// to a direct call of function \p F. Stores the reason in \p Reason.
@ -102,9 +103,12 @@ bool isLegalToPromote(Instruction *Inst, Function *F, const char **Reason);
// TotalCount is the profile count value that the instruction executes.
// Count is the profile count value that F is the target function.
// These two values are used to update the branch weight.
// If \p AttachProfToDirectCall is true, a prof metadata is attached to the
// new direct call to contain \p Count.
// Returns the promoted direct call instruction.
Instruction *promoteIndirectCall(Instruction *Inst, Function *F, uint64_t Count,
uint64_t TotalCount);
uint64_t TotalCount,
bool AttachProfToDirectCall);
/// Options for the frontend instrumentation based profiling pass.
struct InstrProfOptions {

5
include/llvm/Transforms/PGOInstrumentation.h
View File

@ -38,10 +38,13 @@ private:
/// The indirect function call promotion pass.
class PGOIndirectCallPromotion : public PassInfoMixin<PGOIndirectCallPromotion> {
public:
PGOIndirectCallPromotion(bool IsInLTO = false) : InLTO(IsInLTO) {}
PGOIndirectCallPromotion(bool IsInLTO = false, bool SamplePGO = false)
: InLTO(IsInLTO), SamplePGO(SamplePGO) {}
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
private:
bool InLTO;
bool SamplePGO;
};
} // End llvm namespace

8
lib/Passes/PassBuilder.cpp
View File

@ -486,13 +486,14 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
// Add all the requested passes for PGO Instrumentation, if requested.
if (PGOOpt) {
assert(PGOOpt->RunProfileGen || !PGOOpt->ProfileUseFile.empty());
assert(PGOOpt->RunProfileGen || PGOOpt->SamplePGO ||
!PGOOpt->ProfileUseFile.empty());
addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen,
PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile);
}
// Indirect call promotion that promotes intra-module targes only.
MPM.addPass(PGOIndirectCallPromotion());
MPM.addPass(PGOIndirectCallPromotion(false, PGOOpt && PGOOpt->SamplePGO));
// Require the GlobalsAA analysis for the module so we can query it within
// the CGSCC pipeline.
@ -665,7 +666,8 @@ ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
// left by the earlier promotion pass that promotes intra-module targets.
// This two-step promotion is to save the compile time. For LTO, it should
// produce the same result as if we only do promotion here.
MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */));
MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */,
PGOOpt && PGOOpt->SamplePGO));
// Propagate constants at call sites into the functions they call. This
// opens opportunities for globalopt (and inlining) by substituting function

9
lib/Transforms/IPO/PassManagerBuilder.cpp
View File

@ -431,7 +431,8 @@ void PassManagerBuilder::populateModulePassManager(
// earlier in the pass pipeline, here before globalopt. Otherwise imported
// available_externally functions look unreferenced and are removed.
if (PerformThinLTO)
MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true));
MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true,
!PGOSampleUse.empty()));
if (!DisableUnitAtATime) {
// Infer attributes about declarations if possible.
@ -458,7 +459,8 @@ void PassManagerBuilder::populateModulePassManager(
// Indirect call promotion that promotes intra-module targets only.
// For ThinLTO this is done earlier due to interactions with globalopt
// for imported functions.
MPM.add(createPGOIndirectCallPromotionLegacyPass());
MPM.add(
createPGOIndirectCallPromotionLegacyPass(false, !PGOSampleUse.empty()));
}
if (EnableNonLTOGlobalsModRef)
@ -681,7 +683,8 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
// left by the earlier promotion pass that promotes intra-module targets.
// This two-step promotion is to save the compile time. For LTO, it should
// produce the same result as if we only do promotion here.
PM.add(createPGOIndirectCallPromotionLegacyPass(true));
PM.add(
createPGOIndirectCallPromotionLegacyPass(true, !PGOSampleUse.empty()));
// Propagate constants at call sites into the functions they call. This
// opens opportunities for globalopt (and inlining) by substituting function

2
lib/Transforms/IPO/SampleProfile.cpp
View File

@ -646,7 +646,7 @@ bool SampleProfileLoader::inlineHotFunctions(Function &F) {
// We set the probability to 80% taken to indicate that the static
// call is likely taken.
DI = dyn_cast<Instruction>(
promoteIndirectCall(I, CalledFunction, 80, 100)
promoteIndirectCall(I, CalledFunction, 80, 100, false)
->stripPointerCasts());
PromotedInsns.insert(I);
} else {

56
lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
View File

@ -80,6 +80,12 @@ static cl::opt<bool> ICPLTOMode("icp-lto", cl::init(false), cl::Hidden,
cl::desc("Run indirect-call promotion in LTO "
"mode"));
// Set if the pass is called in SamplePGO mode. The difference for SamplePGO
// mode is it will add prof metadatato the created direct call.
static cl::opt<bool>
ICPSamplePGOMode("icp-samplepgo", cl::init(false), cl::Hidden,
cl::desc("Run indirect-call promotion in SamplePGO mode"));
// If the option is set to true, only call instructions will be considered for
// transformation -- invoke instructions will be ignored.
static cl::opt<bool>
@ -105,8 +111,8 @@ class PGOIndirectCallPromotionLegacyPass : public ModulePass {
public:
static char ID;
PGOIndirectCallPromotionLegacyPass(bool InLTO = false)
: ModulePass(ID), InLTO(InLTO) {
PGOIndirectCallPromotionLegacyPass(bool InLTO = false, bool SamplePGO = false)
: ModulePass(ID), InLTO(InLTO), SamplePGO(SamplePGO) {
initializePGOIndirectCallPromotionLegacyPassPass(
*PassRegistry::getPassRegistry());
}
@ -119,6 +125,10 @@ private:
// If this pass is called in LTO. We need to special handling the PGOFuncName
// for the static variables due to LTO's internalization.
bool InLTO;
// If this pass is called in SamplePGO. We need to add the prof metadata to
// the promoted direct call.
bool SamplePGO;
};
} // end anonymous namespace
@ -128,8 +138,9 @@ INITIALIZE_PASS(PGOIndirectCallPromotionLegacyPass, "pgo-icall-prom",
"direct calls.",
false, false)
ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO) {
return new PGOIndirectCallPromotionLegacyPass(InLTO);
ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO,
bool SamplePGO) {
return new PGOIndirectCallPromotionLegacyPass(InLTO, SamplePGO);
}
namespace {
@ -144,6 +155,8 @@ private:
// defines.
InstrProfSymtab *Symtab;
bool SamplePGO;
// Test if we can legally promote this direct-call of Target.
bool isPromotionLegal(Instruction *Inst, uint64_t Target, Function *&F,
const char **Reason = nullptr);
@ -175,9 +188,9 @@ private:
ICallPromotionFunc &operator=(const ICallPromotionFunc &other) = delete;
public:
ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab)
: F(Func), M(Modu), Symtab(Symtab) {
}
ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab,
bool SamplePGO)
: F(Func), M(Modu), Symtab(Symtab), SamplePGO(SamplePGO) {}
bool processFunction();
};
@ -509,10 +522,14 @@ static void insertCallRetPHI(Instruction *Inst, Instruction *CallResult,
// Ret = phi(Ret1, Ret2);
// It adds type casts for the args do not match the parameters and the return
// value. Branch weights metadata also updated.
// If \p AttachProfToDirectCall is true, a prof metadata is attached to the
// new direct call to contain \p Count. This is used by SamplePGO inliner to
// check callsite hotness.
// Returns the promoted direct call instruction.
Instruction *llvm::promoteIndirectCall(Instruction *Inst,
Function *DirectCallee, uint64_t Count,
uint64_t TotalCount) {
uint64_t TotalCount,
bool AttachProfToDirectCall) {
assert(DirectCallee != nullptr);
BasicBlock *BB = Inst->getParent();
// Just to suppress the non-debug build warning.
@ -527,6 +544,14 @@ Instruction *llvm::promoteIndirectCall(Instruction *Inst,
Instruction *NewInst =
createDirectCallInst(Inst, DirectCallee, DirectCallBB, MergeBB);
if (AttachProfToDirectCall) {
SmallVector<uint32_t, 1> Weights;
Weights.push_back(Count);
MDBuilder MDB(NewInst->getContext());
dyn_cast<Instruction>(NewInst->stripPointerCasts())
->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
}
// Move Inst from MergeBB to IndirectCallBB.
Inst->removeFromParent();
IndirectCallBB->getInstList().insert(IndirectCallBB->getFirstInsertionPt(),
@ -569,7 +594,7 @@ uint32_t ICallPromotionFunc::tryToPromote(
for (auto &C : Candidates) {
uint64_t Count = C.Count;
promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount);
promoteIndirectCall(Inst, C.TargetFunction, Count, TotalCount, SamplePGO);
assert(TotalCount >= Count);
TotalCount -= Count;
NumOfPGOICallPromotion++;
@ -610,7 +635,7 @@ bool ICallPromotionFunc::processFunction() {
}
// A wrapper function that does the actual work.
static bool promoteIndirectCalls(Module &M, bool InLTO) {
static bool promoteIndirectCalls(Module &M, bool InLTO, bool SamplePGO) {
if (DisableICP)
return false;
InstrProfSymtab Symtab;
@ -621,7 +646,7 @@ static bool promoteIndirectCalls(Module &M, bool InLTO) {
continue;
if (F.hasFnAttribute(Attribute::OptimizeNone))
continue;
ICallPromotionFunc ICallPromotion(F, &M, &Symtab);
ICallPromotionFunc ICallPromotion(F, &M, &Symtab, SamplePGO);
bool FuncChanged = ICallPromotion.processFunction();
if (ICPDUMPAFTER && FuncChanged) {
DEBUG(dbgs() << "\n== IR Dump After =="; F.print(dbgs()));
@ -638,11 +663,14 @@ static bool promoteIndirectCalls(Module &M, bool InLTO) {
bool PGOIndirectCallPromotionLegacyPass::runOnModule(Module &M) {
// Command-line option has the priority for InLTO.
return promoteIndirectCalls(M, InLTO | ICPLTOMode);
return promoteIndirectCalls(M, InLTO | ICPLTOMode,
SamplePGO | ICPSamplePGOMode);
}
PreservedAnalyses PGOIndirectCallPromotion::run(Module &M, ModuleAnalysisManager &AM) {
if (!promoteIndirectCalls(M, InLTO | ICPLTOMode))
PreservedAnalyses PGOIndirectCallPromotion::run(Module &M,
ModuleAnalysisManager &AM) {
if (!promoteIndirectCalls(M, InLTO | ICPLTOMode,
SamplePGO | ICPSamplePGOMode))
return PreservedAnalyses::all();
return PreservedAnalyses::none();

4
test/Transforms/PGOProfile/indirect_call_promotion.ll
View File

@ -1,4 +1,6 @@
; RUN: opt < %s -pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM
; RUN: opt < %s -pgo-icall-prom -S -icp-samplepgo | FileCheck %s --check-prefix=ICALL-PROM-SAMPLEPGO
; RUN: opt < %s -passes=pgo-icall-prom -S | FileCheck %s --check-prefix=ICALL-PROM
; RUN: opt < %s -pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-count-threshold=0 -icp-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
@ -40,6 +42,7 @@ entry:
; ICALL-PROM: br i1 [[CMP]], label %if.true.direct_targ, label %if.false.orig_indirect, !prof [[BRANCH_WEIGHT:![0-9]+]]
; ICALL-PROM: if.true.direct_targ:
; ICALL-PROM: [[DIRCALL_RET:%[0-9]+]] = call i32 @func4()
; ICALL-PROM-SAMPLEPGO: call i32 @func4(), !prof [[CALL_METADATA:![0-9]+]]
; ICALL-PROM: br label %if.end.icp
%call = call i32 %tmp(), !prof !1
; ICALL-PROM: if.false.orig_indirect:
@ -54,3 +57,4 @@ entry:
; ICALL-PROM: [[BRANCH_WEIGHT]] = !{!"branch_weights", i32 1030, i32 570}
; ICALL-PROM: [[NEW_VP_METADATA]] = !{!"VP", i32 0, i64 570, i64 -4377547752858689819, i64 410}
; ICALL-PROM-SAMPLEPGO: [[CALL_METADATA]] = !{!"branch_weights", i32 1030}