diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index 56f56c37121..3f3a7e472f6 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -681,14 +681,6 @@ struct Header { uint64_t HashOffset; }; -static const uint32_t SummaryCutoffs[] = { - 10000, /* 1% */ - 100000, /* 10% */ - 200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000, - 800000, 900000, 950000, 990000, 999000, 999900, 999990, 999999}; -static const uint32_t NumSummaryCutoffs = - sizeof(SummaryCutoffs) / sizeof(*SummaryCutoffs); - // Profile summary data recorded in the profile data file in indexed // format. It is introduced in version 4. The summary data follows // right after the profile file header. diff --git a/include/llvm/ProfileData/ProfileCommon.h b/include/llvm/ProfileData/ProfileCommon.h index 71031fe73a3..e7d69431542 100644 --- a/include/llvm/ProfileData/ProfileCommon.h +++ b/include/llvm/ProfileData/ProfileCommon.h @@ -24,6 +24,9 @@ namespace llvm { namespace IndexedInstrProf { struct Summary; } +namespace sampleprof { +class FunctionSamples; +} struct InstrProfRecord; // The profile summary is one or more (Cutoff, MinCount, NumCounts) triplets. // The semantics of counts depend on the type of profile. For instrumentation @@ -55,12 +58,18 @@ protected: : DetailedSummaryCutoffs(Cutoffs), TotalCount(0), MaxCount(0), NumCounts(0) {} ProfileSummary() : TotalCount(0), MaxCount(0), NumCounts(0) {} + ProfileSummary(std::vector DetailedSummary, + uint64_t TotalCount, uint64_t MaxCount, uint32_t NumCounts) + : DetailedSummary(DetailedSummary), TotalCount(TotalCount), + MaxCount(MaxCount), NumCounts(NumCounts) {} inline void addCount(uint64_t Count); public: static const int Scale = 1000000; inline std::vector &getDetailedSummary(); void computeDetailedSummary(); + /// \brief A vector of useful cutoff values for detailed summary. + static const std::vector DefaultCutoffs; }; class InstrProfSummary : public ProfileSummary { @@ -83,6 +92,28 @@ public: uint64_t getMaxInternalBlockCount() { return MaxInternalBlockCount; } }; +class SampleProfileSummary : public ProfileSummary { + uint64_t MaxHeadSamples; + uint32_t NumFunctions; + +public: + uint32_t getNumLinesWithSamples() { return NumCounts; } + uint64_t getTotalSamples() { return TotalCount; } + uint32_t getNumFunctions() { return NumFunctions; } + uint64_t getMaxHeadSamples() { return MaxHeadSamples; } + uint64_t getMaxSamplesPerLine() { return MaxCount; } + void addRecord(const sampleprof::FunctionSamples &FS); + SampleProfileSummary(std::vector Cutoffs) + : ProfileSummary(Cutoffs), MaxHeadSamples(0), NumFunctions(0) {} + SampleProfileSummary(uint64_t TotalSamples, uint64_t MaxSamplesPerLine, + uint64_t MaxHeadSamples, int32_t NumLinesWithSamples, + uint32_t NumFunctions, + std::vector DetailedSummary) + : ProfileSummary(DetailedSummary, TotalSamples, MaxSamplesPerLine, + NumLinesWithSamples), + MaxHeadSamples(MaxHeadSamples), NumFunctions(NumFunctions) {} +}; + // This is called when a count is seen in the profile. void ProfileSummary::addCount(uint64_t Count) { TotalCount += Count; diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 6c39cf9458d..a4223b4f216 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -74,7 +74,7 @@ static inline uint64_t SPMagic() { uint64_t('2') << (64 - 56) | uint64_t(0xff); } -static inline uint64_t SPVersion() { return 102; } +static inline uint64_t SPVersion() { return 103; } /// Represents the relative location of an instruction. /// diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index 6db0fbb0e7a..eb4bcc57312 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -129,6 +129,30 @@ // VERSION (uint32_t) // File format version number computed by SPVersion() // +// SUMMARY +// TOTAL_COUNT (uint64_t) +// Total number of samples in the profile. +// MAX_COUNT (uint64_t) +// Maximum value of samples on a line. +// MAX_HEAD_SAMPLES (uint64_t) +// Maximum number of head samples. +// NUM_COUNTS (uint64_t) +// Number of lines with samples. +// NUM_FUNCTIONS (uint64_t) +// Number of functions with samples. +// NUM_DETAILED_SUMMARY_ENTRIES (size_t) +// Number of entries in detailed summary +// DETAILED_SUMMARY +// A list of detailed summary entry. Each entry consists of +// CUTOFF (uint32_t) +// Required percentile of total sample count expressed as a fraction +// multiplied by 1000000. +// MIN_COUNT (uint64_t) +// The minimum number of samples required to reach the target +// CUTOFF. +// NUM_COUNTS (uint64_t) +// Number of samples to get to the desrired percentile. +// // NAME TABLE // SIZE (uint32_t) // Number of entries in the name table. @@ -190,6 +214,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -270,6 +295,9 @@ public: static ErrorOr> create(std::unique_ptr &B, LLVMContext &C); + /// \brief Return the profile summary. + SampleProfileSummary &getSummary() { return *(Summary.get()); } + protected: /// \brief Map every function to its associated profile. /// @@ -283,6 +311,12 @@ protected: /// \brief Memory buffer holding the profile file. std::unique_ptr Buffer; + + /// \brief Profile summary information. + std::unique_ptr Summary; + + /// \brief Compute summary for this profile. + void computeSummary(); }; class SampleProfileReaderText : public SampleProfileReader { @@ -348,6 +382,12 @@ protected: /// Function name table. std::vector NameTable; + +private: + std::error_code readSummaryEntry(std::vector &Entries); + + /// \brief Read profile summary. + std::error_code readSummary(); }; typedef SmallVector InlineCallStack; diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index 029dd2ebacb..03bbf481bec 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -15,6 +15,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" @@ -42,7 +43,6 @@ public: std::error_code write(const StringMap &ProfileMap) { if (std::error_code EC = writeHeader(ProfileMap)) return EC; - for (const auto &I : ProfileMap) { StringRef FName = I.first(); const FunctionSamples &Profile = I.second; @@ -75,6 +75,12 @@ protected: /// \brief Output stream where to emit the profile to. std::unique_ptr OutputStream; + + /// \brief Profile summary. + std::unique_ptr Summary; + + /// \brief Compute summary for this profile. + void computeSummary(const StringMap &ProfileMap); }; /// \brief Sample-based profile writer (text format). @@ -113,6 +119,7 @@ protected: std::error_code writeHeader(const StringMap &ProfileMap) override; + std::error_code writeSummary(); std::error_code writeNameIdx(StringRef FName); std::error_code writeBody(StringRef FName, const FunctionSamples &S); diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp index 9c17d4e082d..b00c1392eb8 100644 --- a/lib/ProfileData/InstrProfReader.cpp +++ b/lib/ProfileData/InstrProfReader.cpp @@ -595,9 +595,8 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version, } else { // For older version of profile data, we need to compute on the fly: using namespace IndexedInstrProf; - std::vector Cutoffs(&SummaryCutoffs[0], - &SummaryCutoffs[NumSummaryCutoffs]); - this->Summary = llvm::make_unique(Cutoffs); + this->Summary = + llvm::make_unique(ProfileSummary::DefaultCutoffs); this->Summary->computeDetailedSummary(); return Cur; } diff --git a/lib/ProfileData/InstrProfWriter.cpp b/lib/ProfileData/InstrProfWriter.cpp index 19a16b82932..69b19580d4f 100644 --- a/lib/ProfileData/InstrProfWriter.cpp +++ b/lib/ProfileData/InstrProfWriter.cpp @@ -217,9 +217,7 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) { OnDiskChainedHashTableGenerator Generator; using namespace IndexedInstrProf; - std::vector Cutoffs(&SummaryCutoffs[0], - &SummaryCutoffs[NumSummaryCutoffs]); - InstrProfSummary PS(Cutoffs); + InstrProfSummary PS(ProfileSummary::DefaultCutoffs); InfoObj->TheProfileSummary = &PS; // Populate the hash table generator. @@ -249,7 +247,7 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) { OS.write(0); // Reserve space to write profile summary data. - uint32_t NumEntries = Cutoffs.size(); + uint32_t NumEntries = ProfileSummary::DefaultCutoffs.size(); uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); // Remember the summary offset. uint64_t SummaryOffset = OS.tell(); diff --git a/lib/ProfileData/ProfileSummary.cpp b/lib/ProfileData/ProfileSummary.cpp index 716c39e4354..0e2c43e5e91 100644 --- a/lib/ProfileData/ProfileSummary.cpp +++ b/lib/ProfileData/ProfileSummary.cpp @@ -11,17 +11,36 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/ProfileData/SampleProf.h" using namespace llvm; +// A set of cutoff values. Each value, when divided by ProfileSummary::Scale +// (which is 1000000) is a desired percentile of total counts. +const std::vector ProfileSummary::DefaultCutoffs( + {10000, /* 1% */ + 100000, /* 10% */ + 200000, 300000, 400000, 500000, 600000, 500000, 600000, 700000, 800000, + 900000, 950000, 990000, 999000, 999900, 999990, 999999}); + void InstrProfSummary::addRecord(const InstrProfRecord &R) { addEntryCount(R.Counts[0]); for (size_t I = 1, E = R.Counts.size(); I < E; ++I) addInternalCount(R.Counts[I]); } +// To compute the detailed summary, we consider each line containing samples as +// equivalent to a block with a count in the instrumented profile. +void SampleProfileSummary::addRecord(const sampleprof::FunctionSamples &FS) { + NumFunctions++; + if (FS.getHeadSamples() > MaxHeadSamples) + MaxHeadSamples = FS.getHeadSamples(); + for (const auto &I : FS.getBodySamples()) + addCount(I.second.getSamples()); +} + // The argument to this method is a vector of cutoff percentages and the return // value is a vector of (Cutoff, MinCount, NumCounts) triplets. void ProfileSummary::computeDetailedSummary() { diff --git a/lib/ProfileData/SampleProfReader.cpp b/lib/ProfileData/SampleProfReader.cpp index 93cd87bb82f..205e34d390f 100644 --- a/lib/ProfileData/SampleProfReader.cpp +++ b/lib/ProfileData/SampleProfReader.cpp @@ -22,6 +22,7 @@ #include "llvm/ProfileData/SampleProfReader.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" @@ -220,6 +221,8 @@ std::error_code SampleProfileReaderText::read() { } } } + if (Result == sampleprof_error::success) + computeSummary(); return Result; } @@ -400,6 +403,9 @@ std::error_code SampleProfileReaderBinary::readHeader() { else if (*Version != SPVersion()) return sampleprof_error::unsupported_version; + if (std::error_code EC = readSummary()) + return EC; + // Read the name table. auto Size = readNumber(); if (std::error_code EC = Size.getError()) @@ -415,6 +421,62 @@ std::error_code SampleProfileReaderBinary::readHeader() { return sampleprof_error::success; } +std::error_code SampleProfileReaderBinary::readSummaryEntry( + std::vector &Entries) { + auto Cutoff = readNumber(); + if (std::error_code EC = Cutoff.getError()) + return EC; + + auto MinBlockCount = readNumber(); + if (std::error_code EC = MinBlockCount.getError()) + return EC; + + auto NumBlocks = readNumber(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readSummary() { + auto TotalCount = readNumber(); + if (std::error_code EC = TotalCount.getError()) + return EC; + + auto MaxBlockCount = readNumber(); + if (std::error_code EC = MaxBlockCount.getError()) + return EC; + + auto MaxFunctionCount = readNumber(); + if (std::error_code EC = MaxFunctionCount.getError()) + return EC; + + auto NumBlocks = readNumber(); + if (std::error_code EC = NumBlocks.getError()) + return EC; + + auto NumFunctions = readNumber(); + if (std::error_code EC = NumFunctions.getError()) + return EC; + + auto NumSummaryEntries = readNumber(); + if (std::error_code EC = NumSummaryEntries.getError()) + return EC; + + std::vector Entries; + for (unsigned i = 0; i < *NumSummaryEntries; i++) { + std::error_code EC = readSummaryEntry(Entries); + if (EC != sampleprof_error::success) + return EC; + } + Summary = llvm::make_unique( + *TotalCount, *MaxBlockCount, *MaxFunctionCount, *NumBlocks, *NumFunctions, + Entries); + + return sampleprof_error::success; +} + bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) { const uint8_t *Data = reinterpret_cast(Buffer.getBufferStart()); @@ -518,6 +580,7 @@ std::error_code SampleProfileReaderGCC::readFunctionProfiles() { if (std::error_code EC = readOneFunctionProfile(Stack, true, 0)) return EC; + computeSummary(); return sampleprof_error::success; } @@ -725,3 +788,14 @@ SampleProfileReader::create(std::unique_ptr &B, LLVMContext &C) { return std::move(Reader); } + +// For text and GCC file formats, we compute the summary after reading the +// profile. Binary format has the profile summary in its header. +void SampleProfileReader::computeSummary() { + Summary.reset(new SampleProfileSummary(ProfileSummary::DefaultCutoffs)); + for (const auto &I : Profiles) { + const FunctionSamples &Profile = I.second; + Summary->addRecord(Profile); + } + Summary->computeDetailedSummary(); +} diff --git a/lib/ProfileData/SampleProfWriter.cpp b/lib/ProfileData/SampleProfWriter.cpp index 51feee5ad7d..a8c542cf200 100644 --- a/lib/ProfileData/SampleProfWriter.cpp +++ b/lib/ProfileData/SampleProfWriter.cpp @@ -120,6 +120,10 @@ std::error_code SampleProfileWriterBinary::writeHeader( encodeULEB128(SPMagic(), OS); encodeULEB128(SPVersion(), OS); + computeSummary(ProfileMap); + if (auto EC = writeSummary()) + return EC; + // Generate the name table for all the functions referenced in the profile. for (const auto &I : ProfileMap) { addName(I.first()); @@ -132,10 +136,25 @@ std::error_code SampleProfileWriterBinary::writeHeader( OS << N.first; encodeULEB128(0, OS); } - return sampleprof_error::success; } +std::error_code SampleProfileWriterBinary::writeSummary() { + auto &OS = *OutputStream; + encodeULEB128(Summary->getTotalSamples(), OS); + encodeULEB128(Summary->getMaxSamplesPerLine(), OS); + encodeULEB128(Summary->getMaxHeadSamples(), OS); + encodeULEB128(Summary->getNumLinesWithSamples(), OS); + encodeULEB128(Summary->getNumFunctions(), OS); + std::vector &Entries = Summary->getDetailedSummary(); + encodeULEB128(Entries.size(), OS); + for (auto Entry : Entries) { + encodeULEB128(Entry.Cutoff, OS); + encodeULEB128(Entry.MinCount, OS); + encodeULEB128(Entry.NumCounts, OS); + } + return sampleprof_error::success; +} std::error_code SampleProfileWriterBinary::writeBody(StringRef FName, const FunctionSamples &S) { auto &OS = *OutputStream; @@ -238,3 +257,13 @@ SampleProfileWriter::create(std::unique_ptr &OS, return std::move(Writer); } + +void SampleProfileWriter::computeSummary( + const StringMap &ProfileMap) { + Summary.reset(new SampleProfileSummary(ProfileSummary::DefaultCutoffs)); + for (const auto &I : ProfileMap) { + const FunctionSamples &Profile = I.second; + Summary->addRecord(Profile); + } + Summary->computeDetailedSummary(); +} diff --git a/test/Transforms/SampleProfile/Inputs/fnptr.binprof b/test/Transforms/SampleProfile/Inputs/fnptr.binprof index a074f53db94..420fd8f86d0 100644 Binary files a/test/Transforms/SampleProfile/Inputs/fnptr.binprof and b/test/Transforms/SampleProfile/Inputs/fnptr.binprof differ diff --git a/unittests/ProfileData/SampleProfTest.cpp b/unittests/ProfileData/SampleProfTest.cpp index cc3c2f5306e..711310785c7 100644 --- a/unittests/ProfileData/SampleProfTest.cpp +++ b/unittests/ProfileData/SampleProfTest.cpp @@ -55,6 +55,10 @@ struct SampleProfTest : ::testing::Test { FooSamples.addTotalSamples(7711); FooSamples.addHeadSamples(610); FooSamples.addBodySamples(1, 0, 610); + FooSamples.addBodySamples(2, 0, 600); + FooSamples.addBodySamples(4, 0, 60000); + FooSamples.addBodySamples(8, 0, 60351); + FooSamples.addBodySamples(10, 0, 605); StringRef BarName("_Z3bari"); FunctionSamples BarSamples; @@ -88,6 +92,32 @@ struct SampleProfTest : ::testing::Test { FunctionSamples &ReadBarSamples = ReadProfiles[BarName]; ASSERT_EQ(20301u, ReadBarSamples.getTotalSamples()); ASSERT_EQ(1437u, ReadBarSamples.getHeadSamples()); + + SampleProfileSummary &Summary = Reader->getSummary(); + ASSERT_EQ(123603u, Summary.getTotalSamples()); + ASSERT_EQ(6u, Summary.getNumLinesWithSamples()); + ASSERT_EQ(2u, Summary.getNumFunctions()); + ASSERT_EQ(1437u, Summary.getMaxHeadSamples()); + ASSERT_EQ(60351u, Summary.getMaxSamplesPerLine()); + + std::vector &Details = Summary.getDetailedSummary(); + uint32_t Cutoff = 800000; + auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) { + return PE.Cutoff == Cutoff; + }; + auto EightyPerc = std::find_if(Details.begin(), Details.end(), Predicate); + Cutoff = 900000; + auto NinetyPerc = std::find_if(Details.begin(), Details.end(), Predicate); + Cutoff = 950000; + auto NinetyFivePerc = + std::find_if(Details.begin(), Details.end(), Predicate); + Cutoff = 990000; + auto NinetyNinePerc = + std::find_if(Details.begin(), Details.end(), Predicate); + ASSERT_EQ(60000u, EightyPerc->MinCount); + ASSERT_EQ(60000u, NinetyPerc->MinCount); + ASSERT_EQ(60000u, NinetyFivePerc->MinCount); + ASSERT_EQ(610u, NinetyNinePerc->MinCount); } };