RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-25 12:12:47 +01:00
Code Issues Projects Releases Wiki Activity

[PGO] Extend the value profile buckets for mem op sizes.

Browse Source 
Extend the memop value profile buckets to be more flexible (could accommodate a
mix of individual values and ranges) and to cover more value ranges (from 11 to
22 buckets).

Disabled behind a flag (to be enabled separately) and the existing code to be
removed later.

Differential Revision: https://reviews.llvm.org/D81682
This commit is contained in:
Hiroshi Yamauchi 2020-06-02 12:10:15 -07:00
parent 3c47ffec1f
commit 754259b7af
10 changed files with 239 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/llvm/ProfileData/InstrProf.h
View File

@ -75,10 +75,18 @@ inline StringRef getInstrProfValueProfFuncName() {
}
/// Return the name profile runtime entry point to do value range profiling.
// FIXME: This is to be removed after switching to the new memop value
// profiling.
inline StringRef getInstrProfValueRangeProfFuncName() {
return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
}
/// Return the name profile runtime entry point to do memop size value
/// profiling.
inline StringRef getInstrProfValueProfMemOpFuncName() {
return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
}
/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }

82
include/llvm/ProfileData/InstrProfData.inc
View File

@ -157,6 +157,8 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
#ifndef VALUE_RANGE_PROF
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
#else /* VALUE_RANGE_PROF */
/* FIXME: This is to be removed after switching to the new memop value
* profiling. */
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@ -753,9 +755,14 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
/* FIXME: This is to be removed after switching to the new memop value
* profiling. */
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
@ -783,3 +790,78 @@ typedef struct InstrProfValueData {
#endif
#undef COVMAP_V2_OR_V3
#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
#ifdef __cplusplus
#define INSTR_PROF_INLINE inline
#else
#define INSTR_PROF_INLINE
#endif
/* The value range buckets (22 buckets) for the memop size value profiling looks
* like:
*
* [0, 0]
* [1, 1]
* [2, 2]
* [3, 3]
* [4, 4]
* [5, 5]
* [6, 6]
* [7, 7]
* [8, 8]
* [9, 15]
* [16, 16]
* [17, 31]
* [32, 32]
* [33, 63]
* [64, 64]
* [65, 127]
* [128, 128]
* [129, 255]
* [256, 256]
* [257, 511]
* [512, 512]
* [513, UINT64_MAX]
*
* Each range has a 'representative value' which is the lower end value of the
* range and used to store in the runtime profile data records and the VP
* metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
*/
/* Map an (observed) memop size value to the representative value of its range.
* For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
InstrProfGetRangeRepValue(uint64_t Value) {
if (Value <= 8)
// The first ranges are individually tracked. Use the value as is.
return Value;
else if (Value >= 513)
// The last range is mapped to its lowest value.
return 513;
else if (__builtin_popcountll(Value) == 1)
// If it's a power of two, use it as is.
return Value;
else
// Otherwise, take to the previous power of two + 1.
return (1 << (64 - __builtin_clzll(Value) - 1)) + 1;
}
/* Return true if the range that an (observed) memop size value belongs to has
* only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
* false, 64 -> true, 100 -> false, 513 -> false. */
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
InstrProfIsSingleValRange(uint64_t Value) {
if (Value <= 8)
// The first ranges are individually tracked.
return 1;
else if (__builtin_popcountll(Value) == 1)
// If it's a power of two, there's only one value.
return 1;
else
// Otherwise, there's more than one value in the range.
return 0;
}
#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */

2
include/llvm/Transforms/Instrumentation/InstrProfiling.h
View File

@ -68,6 +68,8 @@ private:
// vector of counter load/store pairs to be register promoted.
std::vector<LoadStorePair> PromotionCandidates;
// FIXME: These are to be removed after switching to the new memop value
// profiling.
// The start value of precise value profile range for memory intrinsic sizes.
int64_t MemOPSizeRangeStart;
// The end value of precise value profile range for memory intrinsic sizes.

2
lib/ProfileData/InstrProf.cpp
View File

@ -1111,6 +1111,8 @@ bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
return true;
}
// FIXME: This is to be removed after switching to the new memop value
// profiling.
// Parse the value profile options.
void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
int64_t &RangeLast) {

55
lib/Transforms/Instrumentation/InstrProfiling.cpp
View File

@ -57,6 +57,8 @@ using namespace llvm;
#define DEBUG_TYPE "instrprof"
// FIXME: These are to be removed after switching to the new memop value
// profiling.
// The start and end values of precise value profile range for memory
// intrinsic sizes
cl::opt<std::string> MemOPSizeRange(
@ -72,6 +74,12 @@ cl::opt<unsigned> MemOPSizeLarge(
"Value of 0 disables the large value profiling."),
cl::init(8192));
cl::opt<bool> UseOldMemOpValueProf(
"use-old-memop-value-prof",
cl::desc("Use the old memop value profiling buckets. This is "
"transitional and to be removed after switching. "),
cl::init(true));
namespace {
cl::opt<bool> DoHashBasedCounterSplit(
@ -389,6 +397,19 @@ private:
BlockFrequencyInfo *BFI;
};
enum class ValueProfilingCallType {
// Individual values are tracked. Currently used for indiret call target
// profiling.
Default,
// The old memop size value profiling. FIXME: To be removed after switching to
// the new one.
OldMemOp,
// MemOp: the (new) memop size value profiling with extended buckets.
MemOp
};
} // end anonymous namespace
PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@ -573,9 +594,9 @@ bool InstrProfiling::run(
return true;
}
static FunctionCallee
getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
bool IsRange = false) {
static FunctionCallee getOrInsertValueProfilingCall(
Module &M, const TargetLibraryInfo &TLI,
ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
LLVMContext &Ctx = M.getContext();
auto *ReturnTy = Type::getVoidTy(M.getContext());
@ -583,16 +604,22 @@ getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
if (auto AK = TLI.getExtAttrForI32Param(false))
AL = AL.addParamAttribute(M.getContext(), 2, AK);
if (!IsRange) {
if (CallType == ValueProfilingCallType::Default ||
CallType == ValueProfilingCallType::MemOp) {
Type *ParamTypes[] = {
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
#include "llvm/ProfileData/InstrProfData.inc"
};
auto *ValueProfilingCallTy =
FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
ValueProfilingCallTy, AL);
StringRef FuncName = CallType == ValueProfilingCallType::Default
? getInstrProfValueProfFuncName()
: getInstrProfValueProfMemOpFuncName();
return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
} else {
// FIXME: This code is to be removed after switching to the new memop value
// profiling.
assert(CallType == ValueProfilingCallType::OldMemOp);
Type *RangeParamTypes[] = {
#define VALUE_RANGE_PROF 1
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
@ -632,8 +659,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Index += It->second.NumValueSites[Kind];
IRBuilder<> Builder(Ind);
bool IsRange = (Ind->getValueKind()->getZExtValue() ==
llvm::InstrProfValueKind::IPVK_MemOPSize);
bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
llvm::InstrProfValueKind::IPVK_MemOPSize);
CallInst *Call = nullptr;
auto *TLI = &GetTLI(*Ind->getFunction());
@ -643,12 +670,19 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
// WinEHPrepare pass.
SmallVector<OperandBundleDef, 1> OpBundles;
Ind->getOperandBundlesAsDefs(OpBundles);
if (!IsRange) {
if (!IsMemOpSize) {
Value *Args[3] = {Ind->getTargetValue(),
Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
Builder.getInt32(Index)};
Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
OpBundles);
} else if (!UseOldMemOpValueProf) {
Value *Args[3] = {Ind->getTargetValue(),
Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
Builder.getInt32(Index)};
Call = Builder.CreateCall(
getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
Args, OpBundles);
} else {
Value *Args[6] = {
Ind->getTargetValue(),
@ -657,7 +691,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Builder.getInt64(MemOPSizeRangeStart),
Builder.getInt64(MemOPSizeRangeLast),
Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true),
Call = Builder.CreateCall(getOrInsertValueProfilingCall(
*M, *TLI, ValueProfilingCallType::OldMemOp),
Args, OpBundles);
}
if (auto AK = TLI->getExtAttrForI32Param(false))

21
lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
View File

@ -38,6 +38,8 @@
#include "llvm/Pass.h"
#include "llvm/PassRegistry.h"
#include "llvm/ProfileData/InstrProf.h"
#define INSTR_PROF_VALUE_PROF_MEMOP_API
#include "llvm/ProfileData/InstrProfData.inc"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
@ -89,17 +91,25 @@ static cl::opt<bool>
cl::desc("Scale the memop size counts using the basic "
" block count value"));
// FIXME: These are to be removed after switching to the new memop value
// profiling.
// This option sets the rangge of precise profile memop sizes.
extern cl::opt<std::string> MemOPSizeRange;
// This option sets the value that groups large memop sizes
extern cl::opt<unsigned> MemOPSizeLarge;
extern cl::opt<bool> UseOldMemOpValueProf;
cl::opt<bool>
MemOPOptMemcmpBcmp("pgo-memop-optimize-memcmp-bcmp", cl::init(true),
cl::Hidden,
cl::desc("Size-specialize memcmp and bcmp calls"));
static cl::opt<unsigned>
MemOpMaxOptSize("memop-value-prof-max-opt-size", cl::Hidden, cl::init(128),
cl::desc("Optimize the memop size <= this value"));
namespace {
class PGOMemOPSizeOptLegacyPass : public FunctionPass {
public:
@ -269,6 +279,8 @@ private:
TargetLibraryInfo &TLI;
bool Changed;
std::vector<MemOp> WorkList;
// FIXME: These are to be removed after switching to the new memop value
// profiling.
// Start of the previse range.
int64_t PreciseRangeStart;
// Last value of the previse range.
@ -277,6 +289,8 @@ private:
std::unique_ptr<InstrProfValueData[]> ValueDataArray;
bool perform(MemOp MO);
// FIXME: This is to be removed after switching to the new memop value
// profiling.
// This kind shows which group the value falls in. For PreciseValue, we have
// the profile count for that value. LargeGroup groups the values that are in
// range [LargeValue, +inf). NonLargeGroup groups the rest of values.
@ -365,8 +379,11 @@ bool MemOPSizeOpt::perform(MemOp MO) {
if (MemOPScaleCount)
C = getScaledCount(C, ActualCount, SavedTotalCount);
// Only care precise value here.
if (getMemOPSizeKind(V) != PreciseValue)
if (UseOldMemOpValueProf) {
// Only care precise value here.
if (getMemOPSizeKind(V) != PreciseValue)
continue;
} else if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize)
continue;
// ValueCounts are sorted on the count. Break at the first un-profitable

9
test/Transforms/PGOProfile/memcpy.ll
View File

@ -1,5 +1,7 @@
; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
@ -23,7 +25,8 @@ for.cond1:
for.body3:
%conv = sext i32 %add to i64
; CHECK: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
; OLDMEMOPVP: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
; NEWMEMOPVP: call void @__llvm_profile_instrument_memop(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0)
call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
%inc = add nsw i32 %j.0, 1
br label %for.cond1

9
test/Transforms/PGOProfile/memop_profile_funclet.ll
View File

@ -1,8 +1,10 @@
; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s --check-prefix=LOWER
; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
; This test is to verify that PGO runtime library calls get created with the
; appropriate operand bundle funclet information when a memory intrinsic
@ -63,7 +65,8 @@ try.cont: ; preds = %entry
; GEN-SAME: [ "funclet"(token %tmp1) ]
; LOWER: catch:
; LOWER: call void @__llvm_profile_instrument_range(
; LOWEROLDMEMOPVP: call void @__llvm_profile_instrument_range(
; LOWERNEWMEMOPVP: call void @__llvm_profile_instrument_memop(
; LOWER-SAME: [ "funclet"(token %tmp1) ]
declare dso_local void @"?may_throw@@YAXH@Z"(i32)

1
unittests/ProfileData/CMakeLists.txt
View File

@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
add_llvm_unittest(ProfileDataTests
CoverageMappingTest.cpp
InstrProfDataTest.cpp
InstrProfTest.cpp
SampleProfTest.cpp
)

68
unittests/ProfileData/InstrProfDataTest.cpp Normal file
View File

@ -0,0 +1,68 @@
//===- unittest/ProfileData/InstProfDataTest.cpp ----------------------------=//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "gtest/gtest.h"
#include <cstdint>
#define INSTR_PROF_VALUE_PROF_MEMOP_API
#include "llvm/ProfileData/InstrProfData.inc"
namespace {
TEST(InstrProfDataTest, MapValueToRangeRepValue) {
EXPECT_EQ(0ULL, InstrProfGetRangeRepValue(0));
EXPECT_EQ(1ULL, InstrProfGetRangeRepValue(1));
EXPECT_EQ(2ULL, InstrProfGetRangeRepValue(2));
EXPECT_EQ(3ULL, InstrProfGetRangeRepValue(3));
EXPECT_EQ(4ULL, InstrProfGetRangeRepValue(4));
EXPECT_EQ(5ULL, InstrProfGetRangeRepValue(5));
EXPECT_EQ(6ULL, InstrProfGetRangeRepValue(6));
EXPECT_EQ(7ULL, InstrProfGetRangeRepValue(7));
EXPECT_EQ(8ULL, InstrProfGetRangeRepValue(8));
EXPECT_EQ(9ULL, InstrProfGetRangeRepValue(9));
EXPECT_EQ(16ULL, InstrProfGetRangeRepValue(16));
EXPECT_EQ(17ULL, InstrProfGetRangeRepValue(30));
EXPECT_EQ(32ULL, InstrProfGetRangeRepValue(32));
EXPECT_EQ(33ULL, InstrProfGetRangeRepValue(54));
EXPECT_EQ(64ULL, InstrProfGetRangeRepValue(64));
EXPECT_EQ(65ULL, InstrProfGetRangeRepValue(127));
EXPECT_EQ(128ULL, InstrProfGetRangeRepValue(128));
EXPECT_EQ(129ULL, InstrProfGetRangeRepValue(200));
EXPECT_EQ(256ULL, InstrProfGetRangeRepValue(256));
EXPECT_EQ(257ULL, InstrProfGetRangeRepValue(397));
EXPECT_EQ(512ULL, InstrProfGetRangeRepValue(512));
EXPECT_EQ(513ULL, InstrProfGetRangeRepValue(2832048023));
}
TEST(InstrProfDataTest, IsInOneValueRange) {
EXPECT_EQ(true, InstrProfIsSingleValRange(0));
EXPECT_EQ(true, InstrProfIsSingleValRange(1));
EXPECT_EQ(true, InstrProfIsSingleValRange(2));
EXPECT_EQ(true, InstrProfIsSingleValRange(3));
EXPECT_EQ(true, InstrProfIsSingleValRange(4));
EXPECT_EQ(true, InstrProfIsSingleValRange(5));
EXPECT_EQ(true, InstrProfIsSingleValRange(6));
EXPECT_EQ(true, InstrProfIsSingleValRange(7));
EXPECT_EQ(true, InstrProfIsSingleValRange(8));
EXPECT_EQ(false, InstrProfIsSingleValRange(9));
EXPECT_EQ(true, InstrProfIsSingleValRange(16));
EXPECT_EQ(false, InstrProfIsSingleValRange(30));
EXPECT_EQ(true, InstrProfIsSingleValRange(32));
EXPECT_EQ(false, InstrProfIsSingleValRange(54));
EXPECT_EQ(true, InstrProfIsSingleValRange(64));
EXPECT_EQ(false, InstrProfIsSingleValRange(127));
EXPECT_EQ(true, InstrProfIsSingleValRange(128));
EXPECT_EQ(false, InstrProfIsSingleValRange(200));
EXPECT_EQ(true, InstrProfIsSingleValRange(256));
EXPECT_EQ(false, InstrProfIsSingleValRange(397));
EXPECT_EQ(true, InstrProfIsSingleValRange(512));
EXPECT_EQ(false, InstrProfIsSingleValRange(2832048023344));
}
} // end anonymous namespace