[pdb] Write TPI hash values to the TPI stream.

This completes being able to write all the interesting
values of a PDB TPI stream.

Reviewed By: rnk
Differential Revision: https://reviews.llvm.org/D24370

llvm-svn: 281555

include/llvm/DebugInfo/CodeView/CVRecord.h

@@ -11,6 +11,7 @@
 #define LLVM_DEBUGINFO_CODEVIEW_RECORDITERATOR_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/DebugInfo/CodeView/CodeViewError.h"
 #include "llvm/DebugInfo/CodeView/RecordSerialization.h"
@@ -26,6 +27,7 @@ template <typename Kind> struct CVRecord {
   Kind Type;
   ArrayRef<uint8_t> Data;
   ArrayRef<uint8_t> RawData;
+  Optional<uint32_t> Hash;
 };
 }
 namespace msf {

include/llvm/DebugInfo/CodeView/TypeSerializationVisitor.h

@@ -36,27 +36,22 @@ public:
   virtual Error visitTypeEnd(CVType &Record) override {
     // Since this visitor's purpose is to serialize the record, fill out the
     // fields of `Record` with the bytes of the record.
-    if (Record.Type == TypeLeafKind::LF_FIELDLIST)
+    if (Record.Type == TypeLeafKind::LF_FIELDLIST) {
       TypeTableBuilder.writeFieldList(FieldListBuilder);
+      updateCVRecord(Record);
+    }
 
-    StringRef S = TypeTableBuilder.getRecords().back();
-    ArrayRef<uint8_t> Data(S.bytes_begin(), S.bytes_end());
-    Record.RawData = Data;
-    Record.Data = Record.RawData.drop_front(sizeof(RecordPrefix));
-    Record.Length = Data.size() - sizeof(ulittle16_t);
     return Error::success();
   }
 
 #define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
-  virtual Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,                  \
-                                 Name##Record &Record) override {              \
-    visitKnownRecordImpl(Record);                                              \
+  virtual Error visitKnownRecord(CVType &CVR, Name##Record &Record) override { \
+    visitKnownRecordImpl(CVR, Record);                                         \
     return Error::success();                                                   \
   }
 #define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
 #define MEMBER_RECORD(EnumName, EnumVal, Name)                                 \
-  virtual Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,                  \
-                                 Name##Record &Record) override {              \
+  virtual Error visitKnownRecord(CVType &CVR, Name##Record &Record) override { \
     visitMemberRecordImpl(Record);                                             \
     return Error::success();                                                   \
   }
@@ -64,15 +59,24 @@ public:
 #include "llvm/DebugInfo/CodeView/TypeRecords.def"
 
 private:
-  template <typename RecordKind> void visitKnownRecordImpl(RecordKind &Record) {
+  void updateCVRecord(CVType &Record) {
+    StringRef S = TypeTableBuilder.getRecords().back();
+    ArrayRef<uint8_t> Data(S.bytes_begin(), S.bytes_end());
+    Record.RawData = Data;
+    Record.Data = Record.RawData.drop_front(sizeof(RecordPrefix));
+    Record.Length = Data.size() - sizeof(ulittle16_t);
+  }
+  template <typename RecordKind>
+  void visitKnownRecordImpl(CVType &CVR, RecordKind &Record) {
     TypeTableBuilder.writeKnownType(Record);
+    updateCVRecord(CVR);
   }
   template <typename RecordKind>
   void visitMemberRecordImpl(RecordKind &Record) {
     FieldListBuilder.writeMemberType(Record);
   }
 
-  void visitKnownRecordImpl(FieldListRecord &FieldList) {}
+  void visitKnownRecordImpl(CVType &CVR, FieldListRecord &FieldList) {}
 
   FieldListRecordBuilder &FieldListBuilder;
   MemoryTypeTableBuilder &TypeTableBuilder;

include/llvm/DebugInfo/MSF/MSFBuilder.h

@@ -71,13 +71,13 @@ public:
   /// particular stream to occupy the original set of blocks.  If the given
   /// blocks are already allocated, or if the number of blocks specified is
   /// incorrect for the given stream size, this function will return an Error.
-  Error addStream(uint32_t Size, ArrayRef<uint32_t> Blocks);
+  Expected<uint32_t> addStream(uint32_t Size, ArrayRef<uint32_t> Blocks);
 
   /// Add a stream to the MSF file with the given size, occupying any available
   /// blocks that the builder decides to use.  This is useful when building a
   /// new PDB file from scratch and you don't care what blocks a stream occupies
   /// but you just want it to work.
-  Error addStream(uint32_t Size);
+  Expected<uint32_t> addStream(uint32_t Size);
 
   /// Update the size of an existing stream.  This will allocate or deallocate
   /// blocks as needed to match the requested size.  This can fail if `CanGrow`
@@ -113,6 +113,8 @@ public:
   /// MSF layout and can be written directly to the MSF file.
   Expected<MSFLayout> build();
 
+  BumpPtrAllocator &getAllocator() { return Allocator; }
+
 private:
   MSFBuilder(uint32_t BlockSize, uint32_t MinBlockCount, bool CanGrow,
              BumpPtrAllocator &Allocator);

include/llvm/DebugInfo/PDB/Raw/DbiStreamBuilder.h

@@ -21,6 +21,9 @@
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 
 namespace llvm {
+namespace msf {
+class MSFBuilder;
+}
 namespace pdb {
 class DbiStream;
 struct DbiStreamHeader;
@@ -28,7 +31,7 @@ class PDBFile;
 
 class DbiStreamBuilder {
 public:
-  DbiStreamBuilder(BumpPtrAllocator &Allocator);
+  DbiStreamBuilder(msf::MSFBuilder &Msf);
 
   DbiStreamBuilder(const DbiStreamBuilder &) = delete;
   DbiStreamBuilder &operator=(const DbiStreamBuilder &) = delete;
@@ -46,6 +49,8 @@ public:
   Error addModuleInfo(StringRef ObjFile, StringRef Module);
   Error addModuleSourceFile(StringRef Module, StringRef File);
 
+  Error finalizeMsfLayout();
+
   Expected<std::unique_ptr<DbiStream>> build(PDBFile &File,
                                              const msf::WritableStream &Buffer);
   Error commit(const msf::MSFLayout &Layout,
@@ -66,6 +71,7 @@ private:
     StringRef Mod;
   };
 
+  msf::MSFBuilder &Msf;
   BumpPtrAllocator &Allocator;
 
   Optional<PdbRaw_DbiVer> VerHeader;

include/llvm/DebugInfo/PDB/Raw/InfoStreamBuilder.h

@@ -20,6 +20,7 @@
 
 namespace llvm {
 namespace msf {
+class MSFBuilder;
 class StreamWriter;
 }
 namespace pdb {
@@ -27,7 +28,7 @@ class PDBFile;
 
 class InfoStreamBuilder {
 public:
-  InfoStreamBuilder();
+  InfoStreamBuilder(msf::MSFBuilder &Msf);
   InfoStreamBuilder(const InfoStreamBuilder &) = delete;
   InfoStreamBuilder &operator=(const InfoStreamBuilder &) = delete;
 
@@ -40,6 +41,8 @@ public:
 
   uint32_t calculateSerializedLength() const;
 
+  Error finalizeMsfLayout();
+
   Expected<std::unique_ptr<InfoStream>>
   build(PDBFile &File, const msf::WritableStream &Buffer);
 
@@ -47,6 +50,8 @@ public:
                const msf::WritableStream &Buffer) const;
 
 private:
+  msf::MSFBuilder &Msf;
+
   PdbRaw_ImplVer Ver;
   uint32_t Sig;
   uint32_t Age;

include/llvm/DebugInfo/PDB/Raw/RawConstants.h

@@ -63,6 +63,8 @@ enum SpecialStream : uint32_t {
   StreamTPI = 2,
   StreamDBI = 3,
   StreamIPI = 4,
+
+  kSpecialStreamCount
 };
 
 enum class DbgHeaderType : uint16_t {

include/llvm/DebugInfo/PDB/Raw/TpiHashing.h

@@ -0,0 +1,90 @@
+//===- TpiHashing.h ---------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_TPIHASHING_H
+#define LLVM_DEBUGINFO_PDB_TPIHASHING_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/DebugInfo/CodeView/TypeIndex.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h"
+#include "llvm/DebugInfo/MSF/StreamArray.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace pdb {
+class TpiHashUpdater : public codeview::TypeVisitorCallbacks {
+public:
+  TpiHashUpdater() {}
+
+#define TYPE_RECORD(EnumName, EnumVal, Name)                                   \
+  virtual Error visitKnownRecord(codeview::CVType &CVR,                        \
+                                 codeview::Name##Record &Record) override {    \
+    visitKnownRecordImpl(CVR, Record);                                         \
+    return Error::success();                                                   \
+  }
+#define TYPE_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
+#define MEMBER_RECORD(EnumName, EnumVal, Name)
+#define MEMBER_RECORD_ALIAS(EnumName, EnumVal, Name, AliasName)
+#include "llvm/DebugInfo/CodeView/TypeRecords.def"
+
+private:
+  template <typename RecordKind>
+  void visitKnownRecordImpl(codeview::CVType &CVR, RecordKind &Record) {
+    CVR.Hash = 0;
+  }
+
+  void visitKnownRecordImpl(codeview::CVType &CVR,
+                            codeview::UdtSourceLineRecord &Rec);
+  void visitKnownRecordImpl(codeview::CVType &CVR,
+                            codeview::UdtModSourceLineRecord &Rec);
+  void visitKnownRecordImpl(codeview::CVType &CVR, codeview::ClassRecord &Rec);
+  void visitKnownRecordImpl(codeview::CVType &CVR, codeview::EnumRecord &Rec);
+  void visitKnownRecordImpl(codeview::CVType &CVR, codeview::UnionRecord &Rec);
+};
+
+class TpiHashVerifier : public codeview::TypeVisitorCallbacks {
+public:
+  TpiHashVerifier(msf::FixedStreamArray<support::ulittle32_t> &HashValues,
+                  uint32_t NumHashBuckets)
+      : HashValues(HashValues), NumHashBuckets(NumHashBuckets) {}
+
+  Error visitKnownRecord(codeview::CVType &CVR,
+                         codeview::UdtSourceLineRecord &Rec) override;
+  Error visitKnownRecord(codeview::CVType &CVR,
+                         codeview::UdtModSourceLineRecord &Rec) override;
+  Error visitKnownRecord(codeview::CVType &CVR,
+                         codeview::ClassRecord &Rec) override;
+  Error visitKnownRecord(codeview::CVType &CVR,
+                         codeview::EnumRecord &Rec) override;
+  Error visitKnownRecord(codeview::CVType &CVR,
+                         codeview::UnionRecord &Rec) override;
+  Error visitTypeBegin(codeview::CVType &CVR) override;
+
+private:
+  Error verifySourceLine(codeview::TypeIndex TI);
+
+  Error errorInvalidHash() {
+    return make_error<RawError>(
+        raw_error_code::invalid_tpi_hash,
+        "Type index is 0x" +
+            utohexstr(codeview::TypeIndex::FirstNonSimpleIndex + Index));
+  }
+
+  msf::FixedStreamArray<support::ulittle32_t> HashValues;
+  codeview::CVType RawRecord;
+  uint32_t NumHashBuckets;
+  uint32_t Index = -1;
+};
+}
+}
+
+#endif

include/llvm/DebugInfo/PDB/Raw/TpiStream.h

@@ -61,7 +61,7 @@ private:
 
   codeview::CVTypeArray TypeRecords;
 
-  std::unique_ptr<msf::MappedBlockStream> HashStream;
+  std::unique_ptr<msf::ReadableStream> HashStream;
   msf::FixedStreamArray<support::ulittle32_t> HashValues;
   msf::FixedStreamArray<TypeIndexOffset> TypeIndexOffsets;
   msf::FixedStreamArray<TypeIndexOffset> HashAdjustments;

include/llvm/DebugInfo/PDB/Raw/TpiStreamBuilder.h

@@ -25,6 +25,8 @@ namespace codeview {
 class TypeRecord;
 }
 namespace msf {
+class ByteStream;
+class MSFBuilder;
 struct MSFLayout;
 class ReadableStreamRef;
 class WritableStream;
@@ -45,7 +47,7 @@ struct TpiStreamHeader;
 
 class TpiStreamBuilder {
 public:
-  explicit TpiStreamBuilder(BumpPtrAllocator &Allocator);
+  explicit TpiStreamBuilder(msf::MSFBuilder &Msf);
   ~TpiStreamBuilder();
 
   TpiStreamBuilder(const TpiStreamBuilder &) = delete;
@@ -54,6 +56,8 @@ public:
   void setVersionHeader(PdbRaw_TpiVer Version);
   void addTypeRecord(const codeview::CVType &Record);
 
+  Error finalizeMsfLayout();
+
   Expected<std::unique_ptr<TpiStream>> build(PDBFile &File,
                                              const msf::WritableStream &Buffer);
 
@@ -62,13 +66,17 @@ public:
   uint32_t calculateSerializedLength() const;
 
 private:
+  uint32_t calculateHashBufferSize() const;
   Error finalize();
 
+  msf::MSFBuilder &Msf;
   BumpPtrAllocator &Allocator;
 
   Optional<PdbRaw_TpiVer> VerHeader;
   std::vector<codeview::CVType> TypeRecords;
   msf::SequencedItemStream<codeview::CVType> TypeRecordStream;
+  uint32_t HashStreamIndex = kInvalidStreamIndex;
+  std::unique_ptr<msf::ByteStream> HashValueStream;
 
   const TpiStreamHeader *Header;
 };

lib/DebugInfo/MSF/MSFBuilder.cpp

@@ -122,7 +122,8 @@ uint32_t MSFBuilder::getTotalBlockCount() const { return FreeBlocks.size(); }
 
 bool MSFBuilder::isBlockFree(uint32_t Idx) const { return FreeBlocks[Idx]; }
 
-Error MSFBuilder::addStream(uint32_t Size, ArrayRef<uint32_t> Blocks) {
+Expected<uint32_t> MSFBuilder::addStream(uint32_t Size,
+                                         ArrayRef<uint32_t> Blocks) {
   // Add a new stream mapped to the specified blocks.  Verify that the specified
   // blocks are both necessary and sufficient for holding the requested number
   // of bytes, and verify that all requested blocks are free.
@@ -145,17 +146,17 @@ Error MSFBuilder::addStream(uint32_t Size, ArrayRef<uint32_t> Blocks) {
     FreeBlocks.reset(Block);
   }
   StreamData.push_back(std::make_pair(Size, Blocks));
-  return Error::success();
+  return StreamData.size() - 1;
 }
 
-Error MSFBuilder::addStream(uint32_t Size) {
+Expected<uint32_t> MSFBuilder::addStream(uint32_t Size) {
   uint32_t ReqBlocks = bytesToBlocks(Size, BlockSize);
   std::vector<uint32_t> NewBlocks;
   NewBlocks.resize(ReqBlocks);
   if (auto EC = allocateBlocks(ReqBlocks, NewBlocks))
-    return EC;
+    return std::move(EC);
   StreamData.push_back(std::make_pair(Size, NewBlocks));
-  return Error::success();
+  return StreamData.size() - 1;
 }
 
 Error MSFBuilder::setStreamSize(uint32_t Idx, uint32_t Size) {

lib/DebugInfo/PDB/CMakeLists.txt

@@ -45,6 +45,7 @@ add_pdb_impl_folder(Raw
   Raw/RawError.cpp
   Raw/RawSession.cpp
   Raw/SymbolStream.cpp
+  Raw/TpiHashing.cpp
   Raw/TpiStream.cpp
   Raw/TpiStreamBuilder.cpp)
 

lib/DebugInfo/PDB/Raw/DbiStreamBuilder.cpp

@@ -9,6 +9,7 @@
 
 #include "llvm/DebugInfo/PDB/Raw/DbiStreamBuilder.h"
 
+#include "llvm/DebugInfo/MSF/MSFBuilder.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/MSF/StreamWriter.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
@@ -23,9 +24,10 @@ namespace {
 class ModiSubstreamBuilder {};
 }
 
-DbiStreamBuilder::DbiStreamBuilder(BumpPtrAllocator &Allocator)
-    : Allocator(Allocator), Age(1), BuildNumber(0), PdbDllVersion(0),
-      PdbDllRbld(0), Flags(0), MachineType(PDB_Machine::x86), Header(nullptr) {}
+DbiStreamBuilder::DbiStreamBuilder(msf::MSFBuilder &Msf)
+    : Msf(Msf), Allocator(Msf.getAllocator()), Age(1), BuildNumber(0),
+      PdbDllVersion(0), PdbDllRbld(0), Flags(0), MachineType(PDB_Machine::x86),
+      Header(nullptr) {}
 
 void DbiStreamBuilder::setVersionHeader(PdbRaw_DbiVer V) { VerHeader = V; }
 
@@ -227,6 +229,13 @@ Error DbiStreamBuilder::finalize() {
   return Error::success();
 }
 
+Error DbiStreamBuilder::finalizeMsfLayout() {
+  uint32_t Length = calculateSerializedLength();
+  if (auto EC = Msf.setStreamSize(StreamDBI, Length))
+    return EC;
+  return Error::success();
+}
+
 Expected<std::unique_ptr<DbiStream>>
 DbiStreamBuilder::build(PDBFile &File, const msf::WritableStream &Buffer) {
   if (!VerHeader.hasValue())

lib/DebugInfo/PDB/Raw/InfoStreamBuilder.cpp

@@ -9,6 +9,7 @@
 
 #include "llvm/DebugInfo/PDB/Raw/InfoStreamBuilder.h"
 
+#include "llvm/DebugInfo/MSF/MSFBuilder.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/MSF/StreamWriter.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
@@ -20,8 +21,8 @@ using namespace llvm::codeview;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 
-InfoStreamBuilder::InfoStreamBuilder()
-    : Ver(PdbRaw_ImplVer::PdbImplVC70), Sig(-1), Age(0) {}
+InfoStreamBuilder::InfoStreamBuilder(msf::MSFBuilder &Msf)
+    : Msf(Msf), Ver(PdbRaw_ImplVer::PdbImplVC70), Sig(-1), Age(0) {}
 
 void InfoStreamBuilder::setVersion(PdbRaw_ImplVer V) { Ver = V; }
 
@@ -39,6 +40,13 @@ uint32_t InfoStreamBuilder::calculateSerializedLength() const {
   return sizeof(InfoStreamHeader) + NamedStreams.calculateSerializedLength();
 }
 
+Error InfoStreamBuilder::finalizeMsfLayout() {
+  uint32_t Length = calculateSerializedLength();
+  if (auto EC = Msf.setStreamSize(StreamPDB, Length))
+    return EC;
+  return Error::success();
+}
+
 Expected<std::unique_ptr<InfoStream>>
 InfoStreamBuilder::build(PDBFile &File, const msf::WritableStream &Buffer) {
   auto StreamData = MappedBlockStream::createIndexedStream(File.getMsfLayout(),

lib/DebugInfo/PDB/Raw/PDBFileBuilder.cpp

@@ -50,36 +50,33 @@ MSFBuilder &PDBFileBuilder::getMsfBuilder() { return *Msf; }
 
 InfoStreamBuilder &PDBFileBuilder::getInfoBuilder() {
   if (!Info)
-    Info = llvm::make_unique<InfoStreamBuilder>();
+    Info = llvm::make_unique<InfoStreamBuilder>(*Msf);
   return *Info;
 }
 
 DbiStreamBuilder &PDBFileBuilder::getDbiBuilder() {
   if (!Dbi)
-    Dbi = llvm::make_unique<DbiStreamBuilder>(Allocator);
+    Dbi = llvm::make_unique<DbiStreamBuilder>(*Msf);
   return *Dbi;
 }
 
 TpiStreamBuilder &PDBFileBuilder::getTpiBuilder() {
   if (!Tpi)
-    Tpi = llvm::make_unique<TpiStreamBuilder>(Allocator);
+    Tpi = llvm::make_unique<TpiStreamBuilder>(*Msf);
   return *Tpi;
 }
 
 Expected<msf::MSFLayout> PDBFileBuilder::finalizeMsfLayout() const {
   if (Info) {
-    uint32_t Length = Info->calculateSerializedLength();
-    if (auto EC = Msf->setStreamSize(StreamPDB, Length))
+    if (auto EC = Info->finalizeMsfLayout())
       return std::move(EC);
   }
   if (Dbi) {
-    uint32_t Length = Dbi->calculateSerializedLength();
-    if (auto EC = Msf->setStreamSize(StreamDBI, Length))
+    if (auto EC = Dbi->finalizeMsfLayout())
       return std::move(EC);
   }
   if (Tpi) {
-    uint32_t Length = Tpi->calculateSerializedLength();
-    if (auto EC = Msf->setStreamSize(StreamTPI, Length))
+    if (auto EC = Tpi->finalizeMsfLayout())
       return std::move(EC);
   }
 

lib/DebugInfo/PDB/Raw/TpiHashing.cpp

@@ -0,0 +1,110 @@
+//===- TpiHashing.cpp -----------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/TpiHashing.h"
+
+#include "llvm/DebugInfo/PDB/Raw/Hash.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+
+using namespace llvm;
+using namespace llvm::codeview;
+using namespace llvm::pdb;
+
+// Corresponds to `fUDTAnon`.
+template <typename T> static bool isAnonymous(T &Rec) {
+  StringRef Name = Rec.getName();
+  return Name == "<unnamed-tag>" || Name == "__unnamed" ||
+         Name.endswith("::<unnamed-tag>") || Name.endswith("::__unnamed");
+}
+
+// Computes a hash for a given TPI record.
+template <typename T>
+static uint32_t getTpiHash(T &Rec, ArrayRef<uint8_t> FullRecord) {
+  auto Opts = static_cast<uint16_t>(Rec.getOptions());
+
+  bool ForwardRef =
+      Opts & static_cast<uint16_t>(ClassOptions::ForwardReference);
+  bool Scoped = Opts & static_cast<uint16_t>(ClassOptions::Scoped);
+  bool UniqueName = Opts & static_cast<uint16_t>(ClassOptions::HasUniqueName);
+  bool IsAnon = UniqueName && isAnonymous(Rec);
+
+  if (!ForwardRef && !Scoped && !IsAnon)
+    return hashStringV1(Rec.getName());
+  if (!ForwardRef && UniqueName && !IsAnon)
+    return hashStringV1(Rec.getUniqueName());
+  return hashBufferV8(FullRecord);
+}
+
+template <typename T> static uint32_t getSourceLineHash(T &Rec) {
+  char Buf[4];
+  support::endian::write32le(Buf, Rec.getUDT().getIndex());
+  return hashStringV1(StringRef(Buf, 4));
+}
+
+void TpiHashUpdater::visitKnownRecordImpl(CVType &CVR,
+                                          UdtSourceLineRecord &Rec) {
+  CVR.Hash = getSourceLineHash(Rec);
+}
+
+void TpiHashUpdater::visitKnownRecordImpl(CVType &CVR,
+                                          UdtModSourceLineRecord &Rec) {
+  CVR.Hash = getSourceLineHash(Rec);
+}
+
+void TpiHashUpdater::visitKnownRecordImpl(CVType &CVR, ClassRecord &Rec) {
+  CVR.Hash = getTpiHash(Rec, CVR.RawData);
+}
+
+void TpiHashUpdater::visitKnownRecordImpl(CVType &CVR, EnumRecord &Rec) {
+  CVR.Hash = getTpiHash(Rec, CVR.RawData);
+}
+
+void TpiHashUpdater::visitKnownRecordImpl(CVType &CVR, UnionRecord &Rec) {
+  CVR.Hash = getTpiHash(Rec, CVR.RawData);
+}
+
+Error TpiHashVerifier::visitKnownRecord(CVType &CVR, UdtSourceLineRecord &Rec) {
+  return verifySourceLine(Rec.getUDT());
+}
+
+Error TpiHashVerifier::visitKnownRecord(CVType &CVR,
+                                        UdtModSourceLineRecord &Rec) {
+  return verifySourceLine(Rec.getUDT());
+}
+
+Error TpiHashVerifier::visitKnownRecord(CVType &CVR, ClassRecord &Rec) {
+  if (getTpiHash(Rec, CVR.RawData) % NumHashBuckets != HashValues[Index])
+    return errorInvalidHash();
+  return Error::success();
+}
+Error TpiHashVerifier::visitKnownRecord(CVType &CVR, EnumRecord &Rec) {
+  if (getTpiHash(Rec, CVR.RawData) % NumHashBuckets != HashValues[Index])
+    return errorInvalidHash();
+  return Error::success();
+}
+Error TpiHashVerifier::visitKnownRecord(CVType &CVR, UnionRecord &Rec) {
+  if (getTpiHash(Rec, CVR.RawData) % NumHashBuckets != HashValues[Index])
+    return errorInvalidHash();
+  return Error::success();
+}
+
+Error TpiHashVerifier::verifySourceLine(codeview::TypeIndex TI) {
+  char Buf[4];
+  support::endian::write32le(Buf, TI.getIndex());
+  uint32_t Hash = hashStringV1(StringRef(Buf, 4));
+  if (Hash % NumHashBuckets != HashValues[Index])
+    return errorInvalidHash();
+  return Error::success();
+}
+
+Error TpiHashVerifier::visitTypeBegin(CVType &Rec) {
+  ++Index;
+  RawRecord = Rec;
+  return Error::success();
+}

lib/DebugInfo/PDB/Raw/TpiStream.cpp

@@ -17,11 +17,11 @@
 #include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/MSF/StreamReader.h"
-#include "llvm/DebugInfo/PDB/Raw/Hash.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Raw/RawError.h"
 #include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiHashing.h"
 
 #include "llvm/Support/Endian.h"
 
@@ -37,97 +37,6 @@ TpiStream::TpiStream(const PDBFile &File,
 
 TpiStream::~TpiStream() {}
 
-// Corresponds to `fUDTAnon`.
-template <typename T> static bool isAnonymous(T &Rec) {
-  StringRef Name = Rec.getName();
-  return Name == "<unnamed-tag>" || Name == "__unnamed" ||
-      Name.endswith("::<unnamed-tag>") || Name.endswith("::__unnamed");
-}
-
-// Computes a hash for a given TPI record.
-template <typename T>
-static uint32_t getTpiHash(T &Rec, const CVRecord<TypeLeafKind> &RawRec) {
-  auto Opts = static_cast<uint16_t>(Rec.getOptions());
-
-  bool ForwardRef =
-      Opts & static_cast<uint16_t>(ClassOptions::ForwardReference);
-  bool Scoped = Opts & static_cast<uint16_t>(ClassOptions::Scoped);
-  bool UniqueName = Opts & static_cast<uint16_t>(ClassOptions::HasUniqueName);
-  bool IsAnon = UniqueName && isAnonymous(Rec);
-
-  if (!ForwardRef && !Scoped && !IsAnon)
-    return hashStringV1(Rec.getName());
-  if (!ForwardRef && UniqueName && !IsAnon)
-    return hashStringV1(Rec.getUniqueName());
-  return hashBufferV8(RawRec.RawData);
-}
-
-namespace {
-class TpiHashVerifier : public TypeVisitorCallbacks {
-public:
-  TpiHashVerifier(FixedStreamArray<support::ulittle32_t> &HashValues,
-                  uint32_t NumHashBuckets)
-      : HashValues(HashValues), NumHashBuckets(NumHashBuckets) {}
-
-  Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,
-                         UdtSourceLineRecord &Rec) override {
-    return verifySourceLine(Rec);
-  }
-
-  Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,
-                         UdtModSourceLineRecord &Rec) override {
-    return verifySourceLine(Rec);
-  }
-
-  Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,
-                         ClassRecord &Rec) override {
-    return verify(Rec);
-  }
-  Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,
-                         EnumRecord &Rec) override {
-    return verify(Rec);
-  }
-  Error visitKnownRecord(CVRecord<TypeLeafKind> &CVR,
-                         UnionRecord &Rec) override {
-    return verify(Rec);
-  }
-
-  Error visitTypeBegin(CVRecord<TypeLeafKind> &Rec) override {
-    ++Index;
-    RawRecord = Rec;
-    return Error::success();
-  }
-
-private:
-  template <typename T> Error verify(T &Rec) {
-    uint32_t Hash = getTpiHash(Rec, RawRecord);
-    if (Hash % NumHashBuckets != HashValues[Index])
-      return errorInvalidHash();
-    return Error::success();
-  }
-
-  template <typename T> Error verifySourceLine(T &Rec) {
-    char Buf[4];
-    support::endian::write32le(Buf, Rec.getUDT().getIndex());
-    uint32_t Hash = hashStringV1(StringRef(Buf, 4));
-    if (Hash % NumHashBuckets != HashValues[Index])
-      return errorInvalidHash();
-    return Error::success();
-  }
-
-  Error errorInvalidHash() {
-    return make_error<RawError>(
-        raw_error_code::invalid_tpi_hash,
-        "Type index is 0x" + utohexstr(TypeIndex::FirstNonSimpleIndex + Index));
-  }
-
-  FixedStreamArray<support::ulittle32_t> HashValues;
-  CVRecord<TypeLeafKind> RawRecord;
-  uint32_t NumHashBuckets;
-  uint32_t Index = -1;
-};
-}
-
 // Verifies that a given type record matches with a given hash value.
 // Currently we only verify SRC_LINE records.
 Error TpiStream::verifyHashValues() {
@@ -193,6 +102,9 @@ Error TpiStream::reload() {
     HSR.setOffset(Header->HashValueBuffer.Off);
     if (auto EC = HSR.readArray(HashValues, NumHashValues))
       return EC;
+    std::vector<ulittle32_t> HashValueList;
+    for (auto I : HashValues)
+      HashValueList.push_back(I);
 
     HSR.setOffset(Header->IndexOffsetBuffer.Off);
     uint32_t NumTypeIndexOffsets =

lib/DebugInfo/PDB/Raw/TpiStreamBuilder.cpp

@@ -2,6 +2,7 @@
 
 #include "llvm/DebugInfo/CodeView/TypeIndex.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
+#include "llvm/DebugInfo/MSF/MSFBuilder.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/MSF/StreamWriter.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
@@ -14,8 +15,8 @@ using namespace llvm::msf;
 using namespace llvm::pdb;
 using namespace llvm::support;
 
-TpiStreamBuilder::TpiStreamBuilder(BumpPtrAllocator &Allocator)
-    : Allocator(Allocator), Header(nullptr) {}
+TpiStreamBuilder::TpiStreamBuilder(MSFBuilder &Msf)
+    : Msf(Msf), Allocator(Msf.getAllocator()), Header(nullptr) {}
 
 TpiStreamBuilder::~TpiStreamBuilder() {}
 
@@ -35,6 +36,7 @@ Error TpiStreamBuilder::finalize() {
   TpiStreamHeader *H = Allocator.Allocate<TpiStreamHeader>();
 
   uint32_t Count = TypeRecords.size();
+  uint32_t HashBufferSize = calculateHashBufferSize();
 
   H->Version = *VerHeader;
   H->HeaderSize = sizeof(TpiStreamHeader);
@@ -42,13 +44,19 @@ Error TpiStreamBuilder::finalize() {
   H->TypeIndexEnd = H->TypeIndexBegin + Count;
   H->TypeRecordBytes = TypeRecordStream.getLength();
 
-  H->HashStreamIndex = kInvalidStreamIndex;
+  H->HashStreamIndex = HashStreamIndex;
   H->HashAuxStreamIndex = kInvalidStreamIndex;
   H->HashKeySize = sizeof(ulittle32_t);
   H->NumHashBuckets = MinTpiHashBuckets;
 
-  H->HashValueBuffer.Length = 0;
+  // Recall that hash values go into a completely different stream identified by
+  // the `HashStreamIndex` field of the `TpiStreamHeader`.  Therefore, the data
+  // begins at offset 0 of this independent stream.
+  H->HashValueBuffer.Off = 0;
+  H->HashValueBuffer.Length = HashBufferSize;
+  H->HashAdjBuffer.Off = H->HashValueBuffer.Off + H->HashValueBuffer.Length;
   H->HashAdjBuffer.Length = 0;
+  H->IndexOffsetBuffer.Off = H->HashAdjBuffer.Off + H->HashAdjBuffer.Length;
   H->IndexOffsetBuffer.Length = 0;
 
   Header = H;
@@ -56,7 +64,39 @@ Error TpiStreamBuilder::finalize() {
 }
 
 uint32_t TpiStreamBuilder::calculateSerializedLength() const {
-  return sizeof(TpiStreamHeader) + TypeRecordStream.getLength();
+  return sizeof(TpiStreamHeader) + TypeRecordStream.getLength() +
+         calculateHashBufferSize();
+}
+
+uint32_t TpiStreamBuilder::calculateHashBufferSize() const {
+  if (TypeRecords.empty() || !TypeRecords[0].Hash.hasValue())
+    return 0;
+  return TypeRecords.size() * sizeof(ulittle32_t);
+}
+
+Error TpiStreamBuilder::finalizeMsfLayout() {
+  uint32_t Length = calculateSerializedLength();
+  if (auto EC = Msf.setStreamSize(StreamTPI, Length))
+    return EC;
+
+  uint32_t HashBufferSize = calculateHashBufferSize();
+
+  if (HashBufferSize == 0)
+    return Error::success();
+
+  auto ExpectedIndex = Msf.addStream(HashBufferSize);
+  if (!ExpectedIndex)
+    return ExpectedIndex.takeError();
+  HashStreamIndex = *ExpectedIndex;
+  ulittle32_t *H = Allocator.Allocate<ulittle32_t>(TypeRecords.size());
+  MutableArrayRef<ulittle32_t> HashBuffer(H, TypeRecords.size());
+  for (uint32_t I = 0; I < TypeRecords.size(); ++I) {
+    HashBuffer[I] = *TypeRecords[I].Hash % MinTpiHashBuckets;
+  }
+  ArrayRef<uint8_t> Bytes(reinterpret_cast<const uint8_t *>(HashBuffer.data()),
+                          HashBufferSize);
+  HashValueStream = llvm::make_unique<ByteStream>(Bytes);
+  return Error::success();
 }
 
 Expected<std::unique_ptr<TpiStream>>
@@ -72,6 +112,12 @@ TpiStreamBuilder::build(PDBFile &File, const msf::WritableStream &Buffer) {
   auto Tpi = llvm::make_unique<TpiStream>(File, std::move(StreamData));
   Tpi->Header = Header;
   Tpi->TypeRecords = VarStreamArray<codeview::CVType>(TypeRecordStream);
+  if (HashValueStream) {
+    Tpi->HashStream = std::move(HashValueStream);
+    StreamReader HSR(*Tpi->HashStream);
+    if (auto EC = HSR.readArray(Tpi->HashValues, TypeRecords.size()))
+      return std::move(EC);
+  }
   return std::move(Tpi);
 }
 
@@ -91,5 +137,13 @@ Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout,
   if (auto EC = Writer.writeArray(RecordArray))
     return EC;
 
+  if (HashStreamIndex != kInvalidStreamIndex) {
+    auto HVS = WritableMappedBlockStream::createIndexedStream(Layout, Buffer,
+                                                              HashStreamIndex);
+    StreamWriter HW(*HVS);
+    if (auto EC = HW.writeStreamRef(*HashValueStream))
+      return EC;
+  }
+
   return Error::success();
 }

test/DebugInfo/PDB/pdbdump-readwrite.test

@@ -10,12 +10,12 @@ CHECK:      FileHeaders {
 CHECK-NEXT:   BlockSize: 4096
 CHECK-NEXT:   FreeBlockMap: 2
 CHECK-NEXT:   NumBlocks: 25
-CHECK-NEXT:   NumDirectoryBytes: 136
+CHECK-NEXT:   NumDirectoryBytes:
 CHECK-NEXT:   Unknown1: 0
-CHECK-NEXT:   BlockMapAddr: 24
+CHECK-NEXT:   BlockMapAddr:
 CHECK-NEXT:   NumDirectoryBlocks: 1
-CHECK-NEXT:   DirectoryBlocks: [23]
-CHECK-NEXT:   NumStreams: 17
+CHECK-NEXT:   DirectoryBlocks:
+CHECK-NEXT:   NumStreams:
 CHECK-NEXT: }
 CHECK:      PDB Stream {
 CHECK-NEXT:   Version: 20000404

test/DebugInfo/PDB/pdbdump-write.test

@@ -12,6 +12,6 @@
 ;
 ; RUN: llvm-pdbdump pdb2yaml -stream-metadata -stream-directory -pdb-stream -tpi-stream %p/Inputs/empty.pdb > %t.1
 ; RUN: llvm-pdbdump yaml2pdb -pdb=%t.2 %t.1
-; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream %p/Inputs/empty.pdb > %t.3
-; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream %t.2 > %t.4
+; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream -no-file-headers %p/Inputs/empty.pdb > %t.3
+; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream -no-file-headers %t.2 > %t.4
 ; RUN: diff %t.3 %t.4

tools/llvm-pdbdump/PdbYaml.cpp

@@ -19,6 +19,7 @@
 #include "llvm/DebugInfo/PDB/PDBExtras.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
+#include "llvm/DebugInfo/PDB/Raw/TpiHashing.h"
 
 using namespace llvm;
 using namespace llvm::pdb;
@@ -216,6 +217,7 @@ void MappingContextTraits<PdbTpiRecord, pdb::yaml::SerializationContext>::
   codeview::TypeDeserializer Deserializer;
   codeview::TypeSerializationVisitor Serializer(Context.FieldListBuilder,
                                                 Context.TypeTableBuilder);
+  pdb::TpiHashUpdater Hasher;
 
   if (IO.outputting()) {
     // For PDB to Yaml, deserialize into a high level record type, then dump it.
@@ -226,6 +228,7 @@ void MappingContextTraits<PdbTpiRecord, pdb::yaml::SerializationContext>::
     // to bytes.
     Pipeline.addCallbackToPipeline(Context.Dumper);
     Pipeline.addCallbackToPipeline(Serializer);
+    Pipeline.addCallbackToPipeline(Hasher);
   }
 
   codeview::CVTypeVisitor Visitor(Pipeline);

tools/llvm-pdbdump/llvm-pdbdump.cpp

@@ -328,37 +328,14 @@ static void yamlToPdb(StringRef Path) {
   PDBFileBuilder Builder(Allocator);
 
   ExitOnErr(Builder.initialize(YamlObj.Headers->SuperBlock));
-  ExitOnErr(Builder.getMsfBuilder().setDirectoryBlocksHint(
-      YamlObj.Headers->DirectoryBlocks));
-  if (!YamlObj.StreamSizes.hasValue()) {
-    ExitOnErr(make_error<GenericError>(
-        generic_error_code::unspecified,
-        "Cannot generate a PDB when stream sizes are not known"));
-  }
-
-  if (YamlObj.StreamMap.hasValue()) {
-    if (YamlObj.StreamMap->size() != YamlObj.StreamSizes->size()) {
-      ExitOnErr(make_error<GenericError>(generic_error_code::unspecified,
-                                         "YAML specifies different number of "
-                                         "streams in stream sizes and stream "
-                                         "map"));
-    }
-
-    auto &Sizes = *YamlObj.StreamSizes;
-    auto &Map = *YamlObj.StreamMap;
-    for (uint32_t I = 0; I < Sizes.size(); ++I) {
-      uint32_t Size = Sizes[I];
-      std::vector<uint32_t> Blocks;
-      for (auto E : Map[I].Blocks)
-        Blocks.push_back(E);
-      ExitOnErr(Builder.getMsfBuilder().addStream(Size, Blocks));
-    }
-  } else {
-    auto &Sizes = *YamlObj.StreamSizes;
-    for (auto S : Sizes) {
-      ExitOnErr(Builder.getMsfBuilder().addStream(S));
-    }
-  }
+  // Add each of the reserved streams.  We ignore stream metadata in the
+  // yaml, because we will reconstruct our own view of the streams.  For
+  // example, the YAML may say that there were 20 streams in the original
+  // PDB, but maybe we only dump a subset of those 20 streams, so we will
+  // have fewer, and the ones we do have may end up with different indices
+  // than the ones in the original PDB.  So we just start with a clean slate.
+  for (uint32_t I = 0; I < kSpecialStreamCount; ++I)
+    ExitOnErr(Builder.getMsfBuilder().addStream(0));
 
   if (YamlObj.PdbStream.hasValue()) {
     auto &InfoBuilder = Builder.getInfoBuilder();