From 0ca1ed28360f8bf2083d32c2152012c9d18c8333 Mon Sep 17 00:00:00 2001 From: Wei Mi Date: Fri, 3 Apr 2020 11:57:36 -0700 Subject: [PATCH] Recommit [SampleFDO] Add flag for partial profile. Fix the error of show-prof-info.test on some platforms without zlib. The common profile usage is to collect profile from a target and then use the profile to guide the optimized build for the same target. There are some cases that no profile can be collected for a target. In those cases, although no full profile is available, it is possible to have some partial profile collected from other targets to optimize common libraries and utilities. A flag is needed to tell the partial profile from the full profile apart, so compiler can use different strategy for them. Differential Revision: https://reviews.llvm.org/D77426 --- include/llvm/IR/ProfileSummary.h | 8 +++++ include/llvm/ProfileData/SampleProf.h | 10 ++++++ include/llvm/ProfileData/SampleProfWriter.h | 8 +++++ lib/ProfileData/SampleProfReader.cpp | 33 +++++++++++++++++++- test/tools/llvm-profdata/show-prof-info.test | 12 +++++++ test/tools/llvm-profdata/show-prof-size.test | 7 ----- tools/llvm-profdata/llvm-profdata.cpp | 19 ++++++++--- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 test/tools/llvm-profdata/show-prof-info.test delete mode 100644 test/tools/llvm-profdata/show-prof-size.test diff --git a/include/llvm/IR/ProfileSummary.h b/include/llvm/IR/ProfileSummary.h index 78635ec4386..7532b44edd9 100644 --- a/include/llvm/IR/ProfileSummary.h +++ b/include/llvm/IR/ProfileSummary.h @@ -49,6 +49,12 @@ private: SummaryEntryVector DetailedSummary; uint64_t TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount; uint32_t NumCounts, NumFunctions; + /// If 'Partial' is false, it means the profile being used to optimize + /// a target is collected from the same target. + /// If 'Partial' is true, it means the profile is for common/shared + /// code. The common profile is usually merged from profiles collected + /// from running other targets. + bool Partial = false; /// Return detailed summary as metadata. Metadata *getDetailedSummaryMD(LLVMContext &Context); @@ -76,6 +82,8 @@ public: uint64_t getTotalCount() { return TotalCount; } uint64_t getMaxCount() { return MaxCount; } uint64_t getMaxInternalCount() { return MaxInternalCount; } + void setPartialProfile(bool PP) { Partial = PP; } + bool isPartialProfile() { return Partial; } }; } // end namespace llvm diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 7b1086c7491..f1ddbe3ba57 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -169,6 +169,13 @@ enum class SecNameTableFlags : uint32_t { SecFlagInValid = 0, SecFlagMD5Name = (1 << 0) }; +enum class SecProfSummaryFlags : uint32_t { + SecFlagInValid = 0, + /// SecFlagPartial means the profile is for common/shared code. + /// The common profile is usually merged from profiles collected + /// from running other targets. + SecFlagPartial = (1 << 0) +}; // Verify section specific flag is used for the correct section. template @@ -183,6 +190,9 @@ static inline void verifySecFlag(SecType Type, SecFlagType Flag) { case SecNameTable: IsFlagLegal = std::is_same(); break; + case SecProfSummary: + IsFlagLegal = std::is_same(); + break; default: break; } diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index 54500f513ad..7d0df9e44f5 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -59,6 +59,7 @@ public: virtual void setProfileSymbolList(ProfileSymbolList *PSL) {} virtual void setToCompressAllSections() {} virtual void setUseMD5() {} + virtual void setPartialProfile() {} protected: SampleProfileWriter(std::unique_ptr &OS) @@ -217,6 +218,13 @@ public: addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name); } + // Set the profile to be partial. It means the profile is for + // common/shared code. The common profile is usually merged from + // profiles collected from running other targets. + virtual void setPartialProfile() override { + addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial); + } + private: virtual void initSectionHdrLayout() override { // Note that SecFuncOffsetTable section is written after SecLBRProfile diff --git a/lib/ProfileData/SampleProfReader.cpp b/lib/ProfileData/SampleProfReader.cpp index 33efb51dca0..e27bf0771b1 100644 --- a/lib/ProfileData/SampleProfReader.cpp +++ b/lib/ProfileData/SampleProfReader.cpp @@ -478,6 +478,8 @@ std::error_code SampleProfileReaderExtBinary::readOneSection( case SecProfSummary: if (std::error_code EC = readSummary()) return EC; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) + Summary->setPartialProfile(true); break; case SecNameTable: if (std::error_code EC = readNameTableSec( @@ -831,11 +833,40 @@ uint64_t SampleProfileReaderExtBinaryBase::getFileSize() { return FileSize; } +static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) { + std::string Flags; + if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) + Flags.append("{compressed,"); + else + Flags.append("{"); + + switch (Entry.Type) { + case SecNameTable: + if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)) + Flags.append("md5,"); + break; + case SecProfSummary: + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) + Flags.append("partial,"); + break; + default: + break; + } + char &last = Flags.back(); + if (last == ',') + last = '}'; + else + Flags.append("}"); + return Flags; +} + bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { uint64_t TotalSecsSize = 0; for (auto &Entry : SecHdrTable) { OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset - << ", Size: " << Entry.Size << "\n"; + << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry) + << "\n"; + ; TotalSecsSize += getSectionSize(Entry.Type); } uint64_t HeaderSize = SecHdrTable.front().Offset; diff --git a/test/tools/llvm-profdata/show-prof-info.test b/test/tools/llvm-profdata/show-prof-info.test new file mode 100644 index 00000000000..ca91051dc1c --- /dev/null +++ b/test/tools/llvm-profdata/show-prof-info.test @@ -0,0 +1,12 @@ +REQUIRES: zlib +; RUN: llvm-profdata merge -sample -extbinary -use-md5 -compress-all-sections -partial-profile -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.1.output +; RUN: wc -c < %t.1.output > %t.txt +; RUN: llvm-profdata show -sample -show-sec-info-only %t.1.output >> %t.txt +; RUN: FileCheck %s --input-file=%t.txt +; CHECK: [[FILESIZE:.*]] +; To check llvm-profdata shows the correct flags for ProfileSummarySection. +; CHECK: ProfileSummarySection {{.*}} Flags: {compressed,partial} +; To check llvm-profdata shows the correct flags for NameTableSection. +; CHECK: NameTableSection {{.*}} Flags: {compressed,md5} +; To check llvm-profdata shows the correct file size. +; CHECK: [[FILESIZE]] diff --git a/test/tools/llvm-profdata/show-prof-size.test b/test/tools/llvm-profdata/show-prof-size.test deleted file mode 100644 index a77360c8a67..00000000000 --- a/test/tools/llvm-profdata/show-prof-size.test +++ /dev/null @@ -1,7 +0,0 @@ -; RUN: llvm-profdata merge -sample -extbinary -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.1.output -; RUN: wc -c < %t.1.output > %t.txt -; RUN: llvm-profdata show -sample -show-sec-info-only %t.1.output >> %t.txt -; RUN: FileCheck %s --input-file=%t.txt -; Check llvm-profdata shows the correct file size. -; CHECK: [[FILESIZE:.*]] -; CHECK: [[FILESIZE]] diff --git a/tools/llvm-profdata/llvm-profdata.cpp b/tools/llvm-profdata/llvm-profdata.cpp index 90ce8400c84..482bd1c842d 100644 --- a/tools/llvm-profdata/llvm-profdata.cpp +++ b/tools/llvm-profdata/llvm-profdata.cpp @@ -448,7 +448,8 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer, ProfileFormat OutputFormat, MemoryBuffer *Buffer, sampleprof::ProfileSymbolList &WriterList, - bool CompressAllSections, bool UseMD5) { + bool CompressAllSections, bool UseMD5, + bool PartialProfile) { populateProfileSymbolList(Buffer, WriterList); if (WriterList.size() > 0 && OutputFormat != PF_Ext_Binary) warn("Profile Symbol list is not empty but the output format is not " @@ -468,13 +469,19 @@ static void handleExtBinaryWriter(sampleprof::SampleProfileWriter &Writer, else Writer.setUseMD5(); } + if (PartialProfile) { + if (OutputFormat != PF_Ext_Binary) + warn("-partial-profile is ignored. Specify -extbinary to enable it"); + else + Writer.setPartialProfile(); + } } static void mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, StringRef OutputFilename, ProfileFormat OutputFormat, StringRef ProfileSymbolListFile, bool CompressAllSections, - bool UseMD5, FailureMode FailMode) { + bool UseMD5, bool PartialProfile, FailureMode FailMode) { using namespace sampleprof; StringMap ProfileMap; SmallVector, 5> Readers; @@ -531,7 +538,7 @@ mergeSampleProfile(const WeightedFileVector &Inputs, SymbolRemapper *Remapper, // Make sure Buffer lives as long as WriterList. auto Buffer = getInputFileBuf(ProfileSymbolListFile); handleExtBinaryWriter(*Writer, OutputFormat, Buffer.get(), WriterList, - CompressAllSections, UseMD5); + CompressAllSections, UseMD5, PartialProfile); Writer->write(ProfileMap); } @@ -663,6 +670,10 @@ static int merge_main(int argc, const char *argv[]) { "use-md5", cl::init(false), cl::Hidden, cl::desc("Choose to use MD5 to represent string in name table (only " "meaningful for -extbinary)")); + cl::opt PartialProfile( + "partial-profile", cl::init(false), cl::Hidden, + cl::desc("Set the profile to be a partial profile (only meaningful " + "for -extbinary)")); cl::ParseCommandLineOptions(argc, argv, "LLVM profile data merger\n"); @@ -697,7 +708,7 @@ static int merge_main(int argc, const char *argv[]) { else mergeSampleProfile(WeightedInputs, Remapper.get(), OutputFilename, OutputFormat, ProfileSymbolListFile, CompressAllSections, - UseMD5, FailureMode); + UseMD5, PartialProfile, FailureMode); return 0; }