[profile] Add binary id into profiles

This patch adds binary id into profiles to easily associate binaries
with the corresponding profiles. There is an RFC that discusses
the motivation, design and implementation in more detail:
https://lists.llvm.org/pipermail/llvm-dev/2021-June/151154.html

Differential Revision: https://reviews.llvm.org/D102039

include/llvm/ProfileData/InstrProf.h

@@ -1103,6 +1103,7 @@ namespace RawInstrProf {
 // raw header.
 // Version 5: Bit 60 of FuncHash is reserved for the flag for the context
 // sensitive records.
+// Version 6: Added binary id.
 const uint64_t Version = INSTR_PROF_RAW_VERSION;
 
 template <class IntPtrT> inline uint64_t getMagic();

include/llvm/ProfileData/InstrProfData.inc

@@ -137,6 +137,7 @@ INSTR_PROF_RAW_HEADER(uint64_t, NamesSize,  NamesSize)
 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
+INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
 #undef INSTR_PROF_RAW_HEADER
 /* INSTR_PROF_RAW_HEADER  end */
 
@@ -645,7 +646,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
         (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
 
 /* Raw profile format version (start from 1). */
-#define INSTR_PROF_RAW_VERSION 5
+#define INSTR_PROF_RAW_VERSION 6
 /* Indexed profile format version (start from 1). */
 #define INSTR_PROF_INDEX_VERSION 7
 /* Coverage mapping format version (start from 0). */

include/llvm/ProfileData/InstrProfReader.h

@@ -82,6 +82,9 @@ public:
   /// Read a single record.
   virtual Error readNextRecord(NamedInstrProfRecord &Record) = 0;
 
+  /// Print binary ids on stream OS.
+  virtual Error printBinaryIds(raw_ostream &OS) { return success(); };
+
   /// Iterator over profile data.
   InstrProfIterator begin() { return InstrProfIterator(this); }
   InstrProfIterator end() { return InstrProfIterator(); }
@@ -222,6 +225,9 @@ private:
   uint32_t ValueKindLast;
   uint32_t CurValueDataSize;
 
+  uint64_t BinaryIdsSize;
+  const uint8_t *BinaryIdsStart;
+
 public:
   RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer)
       : DataBuffer(std::move(DataBuffer)) {}
@@ -231,6 +237,7 @@ public:
   static bool hasFormat(const MemoryBuffer &DataBuffer);
   Error readHeader() override;
   Error readNextRecord(NamedInstrProfRecord &Record) override;
+  Error printBinaryIds(raw_ostream &OS) override;
 
   bool isIRLevelProfile() const override {
     return (Version & VARIANT_MASK_IR_PROF) != 0;

lib/ProfileData/InstrProfReader.cpp

@@ -24,8 +24,8 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SymbolRemappingReader.h"
 #include "llvm/Support/SwapByteOrder.h"
+#include "llvm/Support/SymbolRemappingReader.h"
 #include <algorithm>
 #include <cctype>
 #include <cstddef>
@@ -374,11 +374,13 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
   NamesSize = swap(Header.NamesSize);
   ValueKindLast = swap(Header.ValueKindLast);
+  BinaryIdsSize = swap(Header.BinaryIdsSize);
 
   auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>);
   auto PaddingSize = getNumPaddingBytes(NamesSize);
 
-  ptrdiff_t DataOffset = sizeof(RawInstrProf::Header);
+  // Profile data starts after profile header and binary ids if exist.
+  ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize;
   ptrdiff_t CountersOffset =
       DataOffset + DataSizeInBytes + PaddingBytesBeforeCounters;
   ptrdiff_t NamesOffset = CountersOffset + (sizeof(uint64_t) * CountersSize) +
@@ -392,6 +394,10 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
       Start + DataOffset);
   DataEnd = Data + DataSize;
+
+  // Binary ids start just after the header.
+  BinaryIdsStart =
+      reinterpret_cast<const uint8_t *>(&Header) + sizeof(RawInstrProf::Header);
   CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
   NamesStart = Start + NamesOffset;
   ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
@@ -506,6 +512,33 @@ Error RawInstrProfReader<IntPtrT>::readNextRecord(NamedInstrProfRecord &Record)
   return success();
 }
 
+template <class IntPtrT>
+Error RawInstrProfReader<IntPtrT>::printBinaryIds(raw_ostream &OS) {
+  if (BinaryIdsSize == 0)
+    return success();
+
+  OS << "Binary IDs: \n";
+  const uint8_t *BI = BinaryIdsStart;
+  while (BI < BinaryIdsStart + BinaryIdsSize) {
+    uint64_t BinaryIdLen = swap(*reinterpret_cast<const uint64_t *>(BI));
+    // Increment by binary id length data type size.
+    BI += sizeof(BinaryIdLen);
+    if (BI > (const uint8_t *)DataBuffer->getBufferEnd())
+      return make_error<InstrProfError>(instrprof_error::malformed);
+
+    for (uint64_t I = 0; I < BinaryIdLen; I++)
+      OS << format("%02x", BI[I]);
+    OS << "\n";
+
+    // Increment by binary id data length.
+    BI += BinaryIdLen;
+    if (BI > (const uint8_t *)DataBuffer->getBufferEnd())
+      return make_error<InstrProfError>(instrprof_error::malformed);
+  }
+
+  return success();
+}
+
 namespace llvm {
 
 template class RawInstrProfReader<uint32_t>;

test/tools/llvm-profdata/c-general.test

@@ -14,7 +14,7 @@ RUN: llvm-profdata show %p/Inputs/c-general.profraw -o - --function=switches | F
 
 SWITCHES-LABEL: Counters:
 SWITCHES-NEXT:   switches:
-SWITCHES-NEXT:     Hash: 0xa50a07f391ae4be5
+SWITCHES-NEXT:     Hash: 0x0099a0c98383683e
 SWITCHES-NEXT:     Counters: 19
 SWITCHES-NEXT:     Function count: 1
 SWITCHES-LABEL: Functions shown: 1

test/tools/llvm-profdata/malformed-ptr-to-counter-array.test

@@ -8,9 +8,10 @@
 // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
 // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
 // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
+// INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
 
 RUN: printf '\201rforpl\377' > %t.profraw
-RUN: printf '\5\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\6\0\0\0\0\0\0\0' >> %t.profraw
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
 RUN: printf '\2\0\0\0\0\0\0\0' >> %t.profraw
@@ -19,6 +20,7 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t.profraw
 RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw
 RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw
 
 // Data Section
 //

test/tools/llvm-profdata/raw-32-bits-be.test

@@ -1,5 +1,5 @@
 RUN: printf '\377lprofR\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\5' >> %t
+RUN: printf '\0\0\0\0\0\0\0\6' >> %t
 RUN: printf '\0\0\0\0\0\0\0\2' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\3' >> %t
@@ -8,6 +8,7 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
 RUN: printf '\0\0\0\0\1\0\0\0' >> %t
 RUN: printf '\0\0\0\0\2\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 
 RUN: printf '\134\370\302\114\333\030\275\254' >> %t
 RUN: printf '\0\0\0\0\0\0\0\1' >> %t

test/tools/llvm-profdata/raw-32-bits-le.test

@@ -1,5 +1,5 @@
 RUN: printf '\201Rforpl\377' > %t
-RUN: printf '\5\0\0\0\0\0\0\0' >> %t
+RUN: printf '\6\0\0\0\0\0\0\0' >> %t
 RUN: printf '\2\0\0\0\0\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 RUN: printf '\3\0\0\0\0\0\0\0' >> %t
@@ -8,6 +8,7 @@ RUN: printf '\20\0\0\0\0\0\0\0' >> %t
 RUN: printf '\0\0\0\1\0\0\0\0' >> %t
 RUN: printf '\0\0\0\2\0\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 
 RUN: printf '\254\275\030\333\114\302\370\134' >> %t
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t

test/tools/llvm-profdata/raw-64-bits-be.test

@@ -1,5 +1,5 @@
 RUN: printf '\377lprofr\201' > %t
-RUN: printf '\0\0\0\0\0\0\0\5' >> %t
+RUN: printf '\0\0\0\0\0\0\0\6' >> %t
 RUN: printf '\0\0\0\0\0\0\0\2' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\3' >> %t
@@ -8,6 +8,7 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t
 RUN: printf '\0\0\0\1\0\4\0\0' >> %t
 RUN: printf '\0\0\0\2\0\4\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 
 RUN: printf '\134\370\302\114\333\030\275\254' >> %t
 RUN: printf '\0\0\0\0\0\0\0\1' >> %t

test/tools/llvm-profdata/raw-64-bits-le.test

@@ -1,5 +1,5 @@
 RUN: printf '\201rforpl\377' > %t
-RUN: printf '\5\0\0\0\0\0\0\0' >> %t
+RUN: printf '\6\0\0\0\0\0\0\0' >> %t
 RUN: printf '\2\0\0\0\0\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 RUN: printf '\3\0\0\0\0\0\0\0' >> %t
@@ -8,6 +8,7 @@ RUN: printf '\20\0\0\0\0\0\0\0' >> %t
 RUN: printf '\0\0\4\0\1\0\0\0' >> %t
 RUN: printf '\0\0\4\0\2\0\0\0' >> %t
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t
 
 RUN: printf '\254\275\030\333\114\302\370\134' >> %t
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t

test/tools/llvm-profdata/raw-two-profiles.test

@@ -1,5 +1,5 @@
 RUN: printf '\201rforpl\377' > %t-foo.profraw
-RUN: printf '\5\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\6\0\0\0\0\0\0\0' >> %t-foo.profraw
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -8,6 +8,7 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t-foo.profraw
 RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw
 RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw
 
 RUN: printf '\254\275\030\333\114\302\370\134' >> %t-foo.profraw
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw
@@ -20,7 +21,7 @@ RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw
 RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw
 
 RUN: printf '\201rforpl\377' > %t-bar.profraw
-RUN: printf '\5\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\6\0\0\0\0\0\0\0' >> %t-bar.profraw
 RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
 RUN: printf '\2\0\0\0\0\0\0\0' >> %t-bar.profraw
@@ -29,6 +30,7 @@ RUN: printf '\10\0\0\0\0\0\0\0' >> %t-bar.profraw
 RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw
 RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw
 RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
+RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw
 
 RUN: printf '\067\265\035\031\112\165\023\344' >> %t-bar.profraw
 RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw

tools/llvm-profdata/llvm-profdata.cpp

@@ -2069,7 +2069,7 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
                             bool ShowAllFunctions, bool ShowCS,
                             uint64_t ValueCutoff, bool OnlyListBelow,
                             const std::string &ShowFunction, bool TextFormat,
-                            raw_fd_ostream &OS) {
+                            bool ShowBinaryIds, raw_fd_ostream &OS) {
   auto ReaderOrErr = InstrProfReader::create(Filename);
   std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs);
   if (ShowDetailedSummary && Cutoffs.empty()) {
@@ -2251,6 +2251,11 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
     OS << "Total count: " << PS->getTotalCount() << "\n";
     PS->printDetailedSummary(OS);
   }
+
+  if (ShowBinaryIds)
+    if (Error E = Reader->printBinaryIds(OS))
+      exitWithError(std::move(E), Filename);
+
   return 0;
 }
 
@@ -2501,6 +2506,8 @@ static int show_main(int argc, const char *argv[]) {
       cl::desc("Show the information of each section in the sample profile. "
                "The flag is only usable when the sample profile is in "
                "extbinary format"));
+  cl::opt<bool> ShowBinaryIds("binary-ids", cl::init(false),
+                              cl::desc("Show binary ids in the profile. "));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n");
 
@@ -2519,11 +2526,11 @@ static int show_main(int argc, const char *argv[]) {
     WithColor::warning() << "-function argument ignored: showing all functions\n";
 
   if (ProfileKind == instr)
-    return showInstrProfile(Filename, ShowCounts, TopNFunctions,
+    return showInstrProfile(
-                            ShowIndirectCallTargets, ShowMemOPSizes,
+        Filename, ShowCounts, TopNFunctions, ShowIndirectCallTargets,
-                            ShowDetailedSummary, DetailedSummaryCutoffs,
+        ShowMemOPSizes, ShowDetailedSummary, DetailedSummaryCutoffs,
-                            ShowAllFunctions, ShowCS, ValueCutoff,
+        ShowAllFunctions, ShowCS, ValueCutoff, OnlyListBelow, ShowFunction,
-                            OnlyListBelow, ShowFunction, TextFormat, OS);
+        TextFormat, ShowBinaryIds, OS);
   else
     return showSampleProfile(Filename, ShowCounts, ShowAllFunctions,
                              ShowDetailedSummary, ShowFunction,