mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-25 20:23:11 +01:00
7e9ad11889
Following up D81682 and D83903, remove the code for the old value profiling buckets, which have been replaced with the new, extended buckets and disabled by default. Also syncing InstrProfData.inc between compiler-rt and llvm. Differential Revision: https://reviews.llvm.org/D88838
895 lines
33 KiB
C++
895 lines
33 KiB
C++
/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
|
|
|*
|
|
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|* See https://llvm.org/LICENSE.txt for license information.
|
|
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|*
|
|
\*===----------------------------------------------------------------------===*/
|
|
/*
|
|
* This is the master file that defines all the data structure, signature,
|
|
* constant literals that are shared across profiling runtime library,
|
|
* compiler (instrumentation), and host tools (reader/writer). The entities
|
|
* defined in this file affect the profile runtime ABI, the raw profile format,
|
|
* or both.
|
|
*
|
|
* The file has two identical copies. The master copy lives in LLVM and
|
|
* the other one sits in compiler-rt/lib/profile directory. To make changes
|
|
* in this file, first modify the master copy and copy it over to compiler-rt.
|
|
* Testing of any change in this file can start only after the two copies are
|
|
* synced up.
|
|
*
|
|
* The first part of the file includes macros that defines types, names, and
|
|
* initializers for the member fields of the core data structures. The field
|
|
* declarations for one structure is enabled by defining the field activation
|
|
* macro associated with that structure. Only one field activation record
|
|
* can be defined at one time and the rest definitions will be filtered out by
|
|
* the preprocessor.
|
|
*
|
|
* Examples of how the template is used to instantiate structure definition:
|
|
* 1. To declare a structure:
|
|
*
|
|
* struct ProfData {
|
|
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
|
|
* Type Name;
|
|
* #include "llvm/ProfileData/InstrProfData.inc"
|
|
* };
|
|
*
|
|
* 2. To construct LLVM type arrays for the struct type:
|
|
*
|
|
* Type *DataTypes[] = {
|
|
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
|
|
* LLVMType,
|
|
* #include "llvm/ProfileData/InstrProfData.inc"
|
|
* };
|
|
*
|
|
* 4. To construct constant array for the initializers:
|
|
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
|
|
* Initializer,
|
|
* Constant *ConstantVals[] = {
|
|
* #include "llvm/ProfileData/InstrProfData.inc"
|
|
* };
|
|
*
|
|
*
|
|
* The second part of the file includes definitions all other entities that
|
|
* are related to runtime ABI and format. When no field activation macro is
|
|
* defined, this file can be included to introduce the definitions.
|
|
*
|
|
\*===----------------------------------------------------------------------===*/
|
|
|
|
/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
|
|
* the compiler runtime. */
|
|
#ifndef INSTR_PROF_VISIBILITY
|
|
#define INSTR_PROF_VISIBILITY
|
|
#endif
|
|
|
|
/* INSTR_PROF_DATA start. */
|
|
/* Definition of member fields of the per-function control structure. */
|
|
#ifndef INSTR_PROF_DATA
|
|
#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
|
|
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
|
|
IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
|
|
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
|
|
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
|
|
Inc->getHash()->getZExtValue()))
|
|
INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
|
|
ConstantExpr::getBitCast(CounterPtr, \
|
|
llvm::Type::getInt64PtrTy(Ctx)))
|
|
/* This is used to map function pointers for the indirect call targets to
|
|
* function name hashes during the conversion from raw to merged profile
|
|
* data.
|
|
*/
|
|
INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
|
|
FunctionAddr)
|
|
INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
|
|
ValuesPtrExpr)
|
|
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
|
|
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
|
|
INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
|
|
ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
|
|
#undef INSTR_PROF_DATA
|
|
/* INSTR_PROF_DATA end. */
|
|
|
|
|
|
/* This is an internal data structure used by value profiler. It
|
|
* is defined here to allow serialization code sharing by LLVM
|
|
* to be used in unit test.
|
|
*
|
|
* typedef struct ValueProfNode {
|
|
* // InstrProfValueData VData;
|
|
* uint64_t Value;
|
|
* uint64_t Count;
|
|
* struct ValueProfNode *Next;
|
|
* } ValueProfNode;
|
|
*/
|
|
/* INSTR_PROF_VALUE_NODE start. */
|
|
#ifndef INSTR_PROF_VALUE_NODE
|
|
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
|
|
ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
|
|
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
|
|
ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
|
|
INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
|
|
ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
|
|
#undef INSTR_PROF_VALUE_NODE
|
|
/* INSTR_PROF_VALUE_NODE end. */
|
|
|
|
/* INSTR_PROF_RAW_HEADER start */
|
|
/* Definition of member fields of the raw profile header data structure. */
|
|
#ifndef INSTR_PROF_RAW_HEADER
|
|
#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
|
|
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
|
|
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
|
|
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
|
|
#undef INSTR_PROF_RAW_HEADER
|
|
/* INSTR_PROF_RAW_HEADER end */
|
|
|
|
/* VALUE_PROF_FUNC_PARAM start */
|
|
/* Definition of parameter types of the runtime API used to do value profiling
|
|
* for a given value site.
|
|
*/
|
|
#ifndef VALUE_PROF_FUNC_PARAM
|
|
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
|
|
#define INSTR_PROF_COMMA
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#define INSTR_PROF_COMMA ,
|
|
#endif
|
|
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
|
|
INSTR_PROF_COMMA
|
|
VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
|
|
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
|
|
#undef VALUE_PROF_FUNC_PARAM
|
|
#undef INSTR_PROF_COMMA
|
|
/* VALUE_PROF_FUNC_PARAM end */
|
|
|
|
/* VALUE_PROF_KIND start */
|
|
#ifndef VALUE_PROF_KIND
|
|
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
/* For indirect function call value profiling, the addresses of the target
|
|
* functions are profiled by the instrumented code. The target addresses are
|
|
* written in the raw profile data and converted to target function name's MD5
|
|
* hash by the profile reader during deserialization. Typically, this happens
|
|
* when the raw profile data is read during profile merging.
|
|
*
|
|
* For this remapping the ProfData is used. ProfData contains both the function
|
|
* name hash and the function address.
|
|
*/
|
|
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
|
|
/* For memory intrinsic functions size profiling. */
|
|
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
|
|
/* These two kinds must be the last to be
|
|
* declared. This is to make sure the string
|
|
* array created with the template can be
|
|
* indexed with the kind value.
|
|
*/
|
|
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
|
|
VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
|
|
|
|
#undef VALUE_PROF_KIND
|
|
/* VALUE_PROF_KIND end */
|
|
|
|
#undef COVMAP_V2_OR_V3
|
|
#ifdef COVMAP_V2
|
|
#define COVMAP_V2_OR_V3
|
|
#endif
|
|
#ifdef COVMAP_V3
|
|
#define COVMAP_V2_OR_V3
|
|
#endif
|
|
|
|
/* COVMAP_FUNC_RECORD start */
|
|
/* Definition of member fields of the function record structure in coverage
|
|
* map.
|
|
*/
|
|
#ifndef COVMAP_FUNC_RECORD
|
|
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
#ifdef COVMAP_V1
|
|
COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
|
|
NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
|
|
llvm::Type::getInt8PtrTy(Ctx)))
|
|
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
|
|
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
|
|
NameValue.size()))
|
|
#endif
|
|
#ifdef COVMAP_V2_OR_V3
|
|
COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
|
|
llvm::ConstantInt::get( \
|
|
llvm::Type::getInt64Ty(Ctx), NameHash))
|
|
#endif
|
|
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
|
|
llvm::ConstantInt::get( \
|
|
llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
|
|
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
|
|
llvm::ConstantInt::get( \
|
|
llvm::Type::getInt64Ty(Ctx), FuncHash))
|
|
#ifdef COVMAP_V3
|
|
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
|
|
llvm::ConstantInt::get( \
|
|
llvm::Type::getInt64Ty(Ctx), FilenamesRef))
|
|
COVMAP_FUNC_RECORD(const char, \
|
|
llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
|
|
CoverageMapping.size()), \
|
|
CoverageMapping,
|
|
llvm::ConstantDataArray::getRaw( \
|
|
CoverageMapping, CoverageMapping.size(), \
|
|
llvm::Type::getInt8Ty(Ctx)))
|
|
#endif
|
|
#undef COVMAP_FUNC_RECORD
|
|
/* COVMAP_FUNC_RECORD end. */
|
|
|
|
/* COVMAP_HEADER start */
|
|
/* Definition of member fields of coverage map header.
|
|
*/
|
|
#ifndef COVMAP_HEADER
|
|
#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
|
|
#else
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
|
|
llvm::ConstantInt::get(Int32Ty, NRecords))
|
|
COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
|
|
llvm::ConstantInt::get(Int32Ty, FilenamesSize))
|
|
COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
|
|
llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
|
|
COVMAP_HEADER(uint32_t, Int32Ty, Version, \
|
|
llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
|
|
#undef COVMAP_HEADER
|
|
/* COVMAP_HEADER end. */
|
|
|
|
|
|
#ifdef INSTR_PROF_SECT_ENTRY
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
INSTR_PROF_SECT_ENTRY(IPSK_data, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
|
|
INSTR_PROF_DATA_COFF, "__DATA,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
|
|
INSTR_PROF_CNTS_COFF, "__DATA,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_name, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
|
|
INSTR_PROF_NAME_COFF, "__DATA,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
|
|
INSTR_PROF_VALS_COFF, "__DATA,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
|
|
INSTR_PROF_VNODES_COFF, "__DATA,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
|
|
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
|
|
INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
|
|
INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
|
|
|
|
#undef INSTR_PROF_SECT_ENTRY
|
|
#endif
|
|
|
|
|
|
#ifdef INSTR_PROF_VALUE_PROF_DATA
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
|
|
#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
|
|
/*!
|
|
* This is the header of the data structure that defines the on-disk
|
|
* layout of the value profile data of a particular kind for one function.
|
|
*/
|
|
typedef struct ValueProfRecord {
|
|
/* The kind of the value profile record. */
|
|
uint32_t Kind;
|
|
/*
|
|
* The number of value profile sites. It is guaranteed to be non-zero;
|
|
* otherwise the record for this kind won't be emitted.
|
|
*/
|
|
uint32_t NumValueSites;
|
|
/*
|
|
* The first element of the array that stores the number of profiled
|
|
* values for each value site. The size of the array is NumValueSites.
|
|
* Since NumValueSites is greater than zero, there is at least one
|
|
* element in the array.
|
|
*/
|
|
uint8_t SiteCountArray[1];
|
|
|
|
/*
|
|
* The fake declaration is for documentation purpose only.
|
|
* Align the start of next field to be on 8 byte boundaries.
|
|
uint8_t Padding[X];
|
|
*/
|
|
|
|
/* The array of value profile data. The size of the array is the sum
|
|
* of all elements in SiteCountArray[].
|
|
InstrProfValueData ValueData[];
|
|
*/
|
|
|
|
#ifdef __cplusplus
|
|
/*!
|
|
* Return the number of value sites.
|
|
*/
|
|
uint32_t getNumValueSites() const { return NumValueSites; }
|
|
/*!
|
|
* Read data from this record and save it to Record.
|
|
*/
|
|
void deserializeTo(InstrProfRecord &Record,
|
|
InstrProfSymtab *SymTab);
|
|
/*
|
|
* In-place byte swap:
|
|
* Do byte swap for this instance. \c Old is the original order before
|
|
* the swap, and \c New is the New byte order.
|
|
*/
|
|
void swapBytes(support::endianness Old, support::endianness New);
|
|
#endif
|
|
} ValueProfRecord;
|
|
|
|
/*!
|
|
* Per-function header/control data structure for value profiling
|
|
* data in indexed format.
|
|
*/
|
|
typedef struct ValueProfData {
|
|
/*
|
|
* Total size in bytes including this field. It must be a multiple
|
|
* of sizeof(uint64_t).
|
|
*/
|
|
uint32_t TotalSize;
|
|
/*
|
|
*The number of value profile kinds that has value profile data.
|
|
* In this implementation, a value profile kind is considered to
|
|
* have profile data if the number of value profile sites for the
|
|
* kind is not zero. More aggressively, the implementation can
|
|
* choose to check the actual data value: if none of the value sites
|
|
* has any profiled values, the kind can be skipped.
|
|
*/
|
|
uint32_t NumValueKinds;
|
|
|
|
/*
|
|
* Following are a sequence of variable length records. The prefix/header
|
|
* of each record is defined by ValueProfRecord type. The number of
|
|
* records is NumValueKinds.
|
|
* ValueProfRecord Record_1;
|
|
* ValueProfRecord Record_N;
|
|
*/
|
|
|
|
#if __cplusplus
|
|
/*!
|
|
* Return the total size in bytes of the on-disk value profile data
|
|
* given the data stored in Record.
|
|
*/
|
|
static uint32_t getSize(const InstrProfRecord &Record);
|
|
/*!
|
|
* Return a pointer to \c ValueProfData instance ready to be streamed.
|
|
*/
|
|
static std::unique_ptr<ValueProfData>
|
|
serializeFrom(const InstrProfRecord &Record);
|
|
/*!
|
|
* Check the integrity of the record.
|
|
*/
|
|
Error checkIntegrity();
|
|
/*!
|
|
* Return a pointer to \c ValueProfileData instance ready to be read.
|
|
* All data in the instance are properly byte swapped. The input
|
|
* data is assumed to be in little endian order.
|
|
*/
|
|
static Expected<std::unique_ptr<ValueProfData>>
|
|
getValueProfData(const unsigned char *SrcBuffer,
|
|
const unsigned char *const SrcBufferEnd,
|
|
support::endianness SrcDataEndianness);
|
|
/*!
|
|
* Swap byte order from \c Endianness order to host byte order.
|
|
*/
|
|
void swapBytesToHost(support::endianness Endianness);
|
|
/*!
|
|
* Swap byte order from host byte order to \c Endianness order.
|
|
*/
|
|
void swapBytesFromHost(support::endianness Endianness);
|
|
/*!
|
|
* Return the total size of \c ValueProfileData.
|
|
*/
|
|
uint32_t getSize() const { return TotalSize; }
|
|
/*!
|
|
* Read data from this data and save it to \c Record.
|
|
*/
|
|
void deserializeTo(InstrProfRecord &Record,
|
|
InstrProfSymtab *SymTab);
|
|
void operator delete(void *ptr) { ::operator delete(ptr); }
|
|
#endif
|
|
} ValueProfData;
|
|
|
|
/*
|
|
* The closure is designed to abstact away two types of value profile data:
|
|
* - InstrProfRecord which is the primary data structure used to
|
|
* represent profile data in host tools (reader, writer, and profile-use)
|
|
* - value profile runtime data structure suitable to be used by C
|
|
* runtime library.
|
|
*
|
|
* Both sources of data need to serialize to disk/memory-buffer in common
|
|
* format: ValueProfData. The abstraction allows compiler-rt's raw profiler
|
|
* writer to share the same format and code with indexed profile writer.
|
|
*
|
|
* For documentation of the member methods below, refer to corresponding methods
|
|
* in class InstrProfRecord.
|
|
*/
|
|
typedef struct ValueProfRecordClosure {
|
|
const void *Record;
|
|
uint32_t (*GetNumValueKinds)(const void *Record);
|
|
uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
|
|
uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
|
|
uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
|
|
|
|
/*
|
|
* After extracting the value profile data from the value profile record,
|
|
* this method is used to map the in-memory value to on-disk value. If
|
|
* the method is null, value will be written out untranslated.
|
|
*/
|
|
uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
|
|
void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
|
|
uint32_t S);
|
|
ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
|
|
} ValueProfRecordClosure;
|
|
|
|
INSTR_PROF_VISIBILITY ValueProfRecord *
|
|
getFirstValueProfRecord(ValueProfData *VPD);
|
|
INSTR_PROF_VISIBILITY ValueProfRecord *
|
|
getValueProfRecordNext(ValueProfRecord *VPR);
|
|
INSTR_PROF_VISIBILITY InstrProfValueData *
|
|
getValueProfRecordValueData(ValueProfRecord *VPR);
|
|
INSTR_PROF_VISIBILITY uint32_t
|
|
getValueProfRecordHeaderSize(uint32_t NumValueSites);
|
|
|
|
#undef INSTR_PROF_VALUE_PROF_DATA
|
|
#endif /* INSTR_PROF_VALUE_PROF_DATA */
|
|
|
|
|
|
#ifdef INSTR_PROF_COMMON_API_IMPL
|
|
#define INSTR_PROF_DATA_DEFINED
|
|
#ifdef __cplusplus
|
|
#define INSTR_PROF_INLINE inline
|
|
#define INSTR_PROF_NULLPTR nullptr
|
|
#else
|
|
#define INSTR_PROF_INLINE
|
|
#define INSTR_PROF_NULLPTR NULL
|
|
#endif
|
|
|
|
#ifndef offsetof
|
|
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
|
|
#endif
|
|
|
|
/*!
|
|
* Return the \c ValueProfRecord header size including the
|
|
* padding bytes.
|
|
*/
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
|
|
uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
|
|
sizeof(uint8_t) * NumValueSites;
|
|
/* Round the size to multiple of 8 bytes. */
|
|
Size = (Size + 7) & ~7;
|
|
return Size;
|
|
}
|
|
|
|
/*!
|
|
* Return the total size of the value profile record including the
|
|
* header and the value data.
|
|
*/
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
uint32_t getValueProfRecordSize(uint32_t NumValueSites,
|
|
uint32_t NumValueData) {
|
|
return getValueProfRecordHeaderSize(NumValueSites) +
|
|
sizeof(InstrProfValueData) * NumValueData;
|
|
}
|
|
|
|
/*!
|
|
* Return the pointer to the start of value data array.
|
|
*/
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
|
|
return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
|
|
This->NumValueSites));
|
|
}
|
|
|
|
/*!
|
|
* Return the total number of value data for \c This record.
|
|
*/
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
|
|
uint32_t NumValueData = 0;
|
|
uint32_t I;
|
|
for (I = 0; I < This->NumValueSites; I++)
|
|
NumValueData += This->SiteCountArray[I];
|
|
return NumValueData;
|
|
}
|
|
|
|
/*!
|
|
* Use this method to advance to the next \c This \c ValueProfRecord.
|
|
*/
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
|
|
uint32_t NumValueData = getValueProfRecordNumValueData(This);
|
|
return (ValueProfRecord *)((char *)This +
|
|
getValueProfRecordSize(This->NumValueSites,
|
|
NumValueData));
|
|
}
|
|
|
|
/*!
|
|
* Return the first \c ValueProfRecord instance.
|
|
*/
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
|
|
return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
|
|
}
|
|
|
|
/* Closure based interfaces. */
|
|
|
|
/*!
|
|
* Return the total size in bytes of the on-disk value profile data
|
|
* given the data stored in Record.
|
|
*/
|
|
INSTR_PROF_VISIBILITY uint32_t
|
|
getValueProfDataSize(ValueProfRecordClosure *Closure) {
|
|
uint32_t Kind;
|
|
uint32_t TotalSize = sizeof(ValueProfData);
|
|
const void *Record = Closure->Record;
|
|
|
|
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
|
|
uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
|
|
if (!NumValueSites)
|
|
continue;
|
|
TotalSize += getValueProfRecordSize(NumValueSites,
|
|
Closure->GetNumValueData(Record, Kind));
|
|
}
|
|
return TotalSize;
|
|
}
|
|
|
|
/*!
|
|
* Extract value profile data of a function for the profile kind \c ValueKind
|
|
* from the \c Closure and serialize the data into \c This record instance.
|
|
*/
|
|
INSTR_PROF_VISIBILITY void
|
|
serializeValueProfRecordFrom(ValueProfRecord *This,
|
|
ValueProfRecordClosure *Closure,
|
|
uint32_t ValueKind, uint32_t NumValueSites) {
|
|
uint32_t S;
|
|
const void *Record = Closure->Record;
|
|
This->Kind = ValueKind;
|
|
This->NumValueSites = NumValueSites;
|
|
InstrProfValueData *DstVD = getValueProfRecordValueData(This);
|
|
|
|
for (S = 0; S < NumValueSites; S++) {
|
|
uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
|
|
This->SiteCountArray[S] = ND;
|
|
Closure->GetValueForSite(Record, DstVD, ValueKind, S);
|
|
DstVD += ND;
|
|
}
|
|
}
|
|
|
|
/*!
|
|
* Extract value profile data of a function from the \c Closure
|
|
* and serialize the data into \c DstData if it is not NULL or heap
|
|
* memory allocated by the \c Closure's allocator method. If \c
|
|
* DstData is not null, the caller is expected to set the TotalSize
|
|
* in DstData.
|
|
*/
|
|
INSTR_PROF_VISIBILITY ValueProfData *
|
|
serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
|
|
ValueProfData *DstData) {
|
|
uint32_t Kind;
|
|
uint32_t TotalSize =
|
|
DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
|
|
|
|
ValueProfData *VPD =
|
|
DstData ? DstData : Closure->AllocValueProfData(TotalSize);
|
|
|
|
VPD->TotalSize = TotalSize;
|
|
VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
|
|
ValueProfRecord *VR = getFirstValueProfRecord(VPD);
|
|
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
|
|
uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
|
|
if (!NumValueSites)
|
|
continue;
|
|
serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
|
|
VR = getValueProfRecordNext(VR);
|
|
}
|
|
return VPD;
|
|
}
|
|
|
|
#undef INSTR_PROF_COMMON_API_IMPL
|
|
#endif /* INSTR_PROF_COMMON_API_IMPL */
|
|
|
|
/*============================================================================*/
|
|
|
|
#ifndef INSTR_PROF_DATA_DEFINED
|
|
|
|
#ifndef INSTR_PROF_DATA_INC
|
|
#define INSTR_PROF_DATA_INC
|
|
|
|
/* Helper macros. */
|
|
#define INSTR_PROF_SIMPLE_QUOTE(x) #x
|
|
#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
|
|
#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
|
|
#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
|
|
|
|
/* Magic number to detect file format and endianness.
|
|
* Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
|
|
* so that utilities, like strings, don't grab it as a string. 129 is also
|
|
* invalid UTF-8, and high enough to be interesting.
|
|
* Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
|
|
* for 32-bit platforms.
|
|
*/
|
|
#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
|
|
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
|
|
(uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
|
|
#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
|
|
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
|
|
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
|
|
|
|
/* Raw profile format version (start from 1). */
|
|
#define INSTR_PROF_RAW_VERSION 5
|
|
/* Indexed profile format version (start from 1). */
|
|
#define INSTR_PROF_INDEX_VERSION 6
|
|
/* Coverage mapping format version (start from 0). */
|
|
#define INSTR_PROF_COVMAP_VERSION 3
|
|
|
|
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
|
|
* version for other variants of profile. We set the lowest bit of the upper 8
|
|
* bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
|
|
* generated profile, and 0 if this is a Clang FE generated profile.
|
|
* 1 in bit 57 indicates there are context-sensitive records in the profile.
|
|
*/
|
|
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
|
|
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
|
|
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
|
|
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
|
|
#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
|
|
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
|
|
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
|
|
|
|
/* The variable that holds the name of the profile data
|
|
* specified via command line. */
|
|
#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
|
|
|
|
/* section name strings common to all targets other
|
|
than WIN32 */
|
|
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
|
|
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
|
|
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
|
|
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
|
|
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
|
|
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
|
|
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
|
|
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
|
|
/* Windows section names. Because these section names contain dollar characters,
|
|
* they must be quoted.
|
|
*/
|
|
#define INSTR_PROF_DATA_COFF ".lprfd$M"
|
|
#define INSTR_PROF_NAME_COFF ".lprfn$M"
|
|
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
|
|
#define INSTR_PROF_VALS_COFF ".lprfv$M"
|
|
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
|
|
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
|
|
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
|
|
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
|
|
|
|
#ifdef _WIN32
|
|
/* Runtime section names and name strings. */
|
|
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
|
|
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
|
|
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
|
|
/* Array of pointers. Each pointer points to a list
|
|
* of value nodes associated with one value site.
|
|
*/
|
|
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
|
|
/* Value profile nodes section. */
|
|
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
|
|
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
|
|
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
|
|
#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
|
|
#else
|
|
/* Runtime section names and name strings. */
|
|
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
|
|
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
|
|
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
|
|
/* Array of pointers. Each pointer points to a list
|
|
* of value nodes associated with one value site.
|
|
*/
|
|
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
|
|
/* Value profile nodes section. */
|
|
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
|
|
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
|
|
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
|
|
/* Order file instrumentation. */
|
|
#define INSTR_PROF_ORDERFILE_SECT_NAME \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
|
|
#endif
|
|
|
|
#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
|
|
#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
|
|
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
|
|
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
|
|
|
|
/* Macros to define start/stop section symbol for a given
|
|
* section on Linux. For instance
|
|
* INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
|
|
* expand to __start___llvm_prof_data
|
|
*/
|
|
#define INSTR_PROF_SECT_START(Sect) \
|
|
INSTR_PROF_CONCAT(__start_,Sect)
|
|
#define INSTR_PROF_SECT_STOP(Sect) \
|
|
INSTR_PROF_CONCAT(__stop_,Sect)
|
|
|
|
/* Value Profiling API linkage name. */
|
|
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
|
|
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
|
|
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
|
|
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
|
|
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
|
|
|
|
/* InstrProfile per-function control data alignment. */
|
|
#define INSTR_PROF_DATA_ALIGNMENT 8
|
|
|
|
/* The data structure that represents a tracked value by the
|
|
* value profiler.
|
|
*/
|
|
typedef struct InstrProfValueData {
|
|
/* Profiled value. */
|
|
uint64_t Value;
|
|
/* Number of times the value appears in the training run. */
|
|
uint64_t Count;
|
|
} InstrProfValueData;
|
|
|
|
#endif /* INSTR_PROF_DATA_INC */
|
|
|
|
#ifndef INSTR_ORDER_FILE_INC
|
|
/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
|
|
#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
|
|
#define INSTR_ORDER_FILE_BUFFER_BITS 17
|
|
#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
|
|
#endif /* INSTR_ORDER_FILE_INC */
|
|
#else
|
|
#undef INSTR_PROF_DATA_DEFINED
|
|
#endif
|
|
|
|
#undef COVMAP_V2_OR_V3
|
|
|
|
#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
|
|
|
|
#ifdef __cplusplus
|
|
#define INSTR_PROF_INLINE inline
|
|
#else
|
|
#define INSTR_PROF_INLINE
|
|
#endif
|
|
|
|
/* The value range buckets (22 buckets) for the memop size value profiling looks
|
|
* like:
|
|
*
|
|
* [0, 0]
|
|
* [1, 1]
|
|
* [2, 2]
|
|
* [3, 3]
|
|
* [4, 4]
|
|
* [5, 5]
|
|
* [6, 6]
|
|
* [7, 7]
|
|
* [8, 8]
|
|
* [9, 15]
|
|
* [16, 16]
|
|
* [17, 31]
|
|
* [32, 32]
|
|
* [33, 63]
|
|
* [64, 64]
|
|
* [65, 127]
|
|
* [128, 128]
|
|
* [129, 255]
|
|
* [256, 256]
|
|
* [257, 511]
|
|
* [512, 512]
|
|
* [513, UINT64_MAX]
|
|
*
|
|
* Each range has a 'representative value' which is the lower end value of the
|
|
* range and used to store in the runtime profile data records and the VP
|
|
* metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
|
|
*/
|
|
|
|
/*
|
|
* Clz and Popcount. This code was copied from
|
|
* compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
|
|
* llvm/include/llvm/Support/MathExtras.h.
|
|
*/
|
|
#if defined(_MSC_VER) && !defined(__clang__)
|
|
|
|
#include <intrin.h>
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
int InstProfClzll(unsigned long long X) {
|
|
unsigned long LeadZeroIdx = 0;
|
|
#if !defined(_M_ARM64) && !defined(_M_X64)
|
|
// Scan the high 32 bits.
|
|
if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
|
|
return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
|
|
// from the MSB.
|
|
// Scan the low 32 bits.
|
|
if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
|
|
return (int)(63 - LeadZeroIdx);
|
|
#else
|
|
if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
|
|
#endif
|
|
return 64;
|
|
}
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
int InstProfPopcountll(unsigned long long X) {
|
|
// This code originates from https://reviews.llvm.org/rG30626254510f.
|
|
unsigned long long v = X;
|
|
v = v - ((v >> 1) & 0x5555555555555555ULL);
|
|
v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
|
|
v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
|
|
return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
|
|
}
|
|
|
|
#else
|
|
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
|
|
int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
|
|
|
|
#endif /* defined(_MSC_VER) && !defined(__clang__) */
|
|
|
|
/* Map an (observed) memop size value to the representative value of its range.
|
|
* For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
|
|
InstrProfGetRangeRepValue(uint64_t Value) {
|
|
if (Value <= 8)
|
|
// The first ranges are individually tracked. Use the value as is.
|
|
return Value;
|
|
else if (Value >= 513)
|
|
// The last range is mapped to its lowest value.
|
|
return 513;
|
|
else if (InstProfPopcountll(Value) == 1)
|
|
// If it's a power of two, use it as is.
|
|
return Value;
|
|
else
|
|
// Otherwise, take to the previous power of two + 1.
|
|
return (1 << (64 - InstProfClzll(Value) - 1)) + 1;
|
|
}
|
|
|
|
/* Return true if the range that an (observed) memop size value belongs to has
|
|
* only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
|
|
* false, 64 -> true, 100 -> false, 513 -> false. */
|
|
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
|
|
InstrProfIsSingleValRange(uint64_t Value) {
|
|
if (Value <= 8)
|
|
// The first ranges are individually tracked.
|
|
return 1;
|
|
else if (InstProfPopcountll(Value) == 1)
|
|
// If it's a power of two, there's only one value.
|
|
return 1;
|
|
else
|
|
// Otherwise, there's more than one value in the range.
|
|
return 0;
|
|
}
|
|
|
|
#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
|