[pdb] added support for dumping globals stream

Summary: This adds support for dumping the globals stream from PDB files using llvm-pdbdump, similar to the support we have for the publics stream.

Reviewers: ruiu, zturner

Subscribers: beanz, mgorny, modocache

Differential Revision: https://reviews.llvm.org/D25801

llvm-svn: 284861

include/llvm/DebugInfo/PDB/Raw/GlobalsStream.h

@@ -0,0 +1,45 @@
+//===- GlobalsStream.h - PDB Index of Symbols by Name ------ ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_PDB_RAW_GLOBALS_STREAM_H
+#define LLVM_DEBUGINFO_PDB_RAW_GLOBALS_STREAM_H
+
+#include "llvm/DebugInfo/MSF/MappedBlockStream.h"
+#include "llvm/DebugInfo/MSF/StreamArray.h"
+#include "llvm/DebugInfo/PDB/PDBTypes.h"
+#include "llvm/DebugInfo/PDB/Raw/RawConstants.h"
+#include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace pdb {
+class DbiStream;
+class PDBFile;
+
+class GlobalsStream {
+public:
+  explicit GlobalsStream(std::unique_ptr<msf::MappedBlockStream> Stream);
+  ~GlobalsStream();
+  Error commit();
+  msf::FixedStreamArray<support::ulittle32_t> getHashBuckets() const {
+    return HashBuckets;
+  }
+  uint32_t getNumBuckets() const { return NumBuckets; }
+  Error reload();
+
+private:
+  msf::FixedStreamArray<support::ulittle32_t> HashBuckets;
+  msf::FixedStreamArray<PSHashRecord> HashRecords;
+  uint32_t NumBuckets;
+  std::unique_ptr<msf::MappedBlockStream> Stream;
+};
+}
+}
+
+#endif

include/llvm/DebugInfo/PDB/Raw/PDBFile.h

@@ -31,6 +31,7 @@ class WritableStream;
 
 namespace pdb {
 class DbiStream;
+class GlobalsStream;
 class InfoStream;
 class NameHashTable;
 class PDBFileBuilder;
@@ -86,6 +87,7 @@ public:
 
   Expected<InfoStream &> getPDBInfoStream();
   Expected<DbiStream &> getPDBDbiStream();
+  Expected<GlobalsStream &> getPDBGlobalsStream();
   Expected<TpiStream &> getPDBTpiStream();
   Expected<TpiStream &> getPDBIpiStream();
   Expected<PublicsStream &> getPDBPublicsStream();
@@ -102,6 +104,7 @@ private:
   std::vector<uint32_t> FpmPages;
   msf::MSFLayout ContainerLayout;
 
+  std::unique_ptr<GlobalsStream> Globals;
   std::unique_ptr<InfoStream> Info;
   std::unique_ptr<DbiStream> Dbi;
   std::unique_ptr<TpiStream> Tpi;

include/llvm/DebugInfo/PDB/Raw/PublicsStream.h

@@ -22,10 +22,10 @@
 namespace llvm {
 namespace pdb {
 class DbiStream;
+struct GSIHashHeader;
 class PDBFile;
 
 class PublicsStream {
-  struct GSIHashHeader;
   struct HeaderInfo;
 
 public:

lib/DebugInfo/PDB/CMakeLists.txt

@@ -31,6 +31,8 @@ add_pdb_impl_folder(Raw
   Raw/DbiStream.cpp
   Raw/DbiStreamBuilder.cpp
   Raw/EnumTables.cpp
+  Raw/GlobalsStream.cpp
+  Raw/GSI.cpp
   Raw/Hash.cpp
   Raw/InfoStream.cpp
   Raw/InfoStreamBuilder.cpp

lib/DebugInfo/PDB/Raw/GSI.cpp

@@ -0,0 +1,93 @@
+//===- GSI.cpp - Common Functions for GlobalsStream and PublicsStream  ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "GSI.h"
+
+#include "llvm/DebugInfo/MSF/StreamArray.h"
+#include "llvm/DebugInfo/MSF/StreamReader.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
+
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace pdb {
+
+static Error checkHashHdrVersion(const GSIHashHeader *HashHdr) {
+  if (HashHdr->VerHdr != GSIHashHeader::HdrVersion)
+    return make_error<RawError>(
+        raw_error_code::feature_unsupported,
+        "Encountered unsupported globals stream version.");
+
+  return Error::success();
+}
+
+Error readGSIHashBuckets(
+    msf::FixedStreamArray<support::ulittle32_t> &HashBuckets,
+    const GSIHashHeader *HashHdr, msf::StreamReader &Reader) {
+  if (auto EC = checkHashHdrVersion(HashHdr))
+    return EC;
+
+  // Before the actual hash buckets, there is a bitmap of length determined by
+  // IPHR_HASH.
+  ArrayRef<uint8_t> Bitmap;
+  size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32);
+  uint32_t NumBitmapEntries = BitmapSizeInBits / 8;
+  if (auto EC = Reader.readBytes(Bitmap, NumBitmapEntries))
+    return joinErrors(std::move(EC),
+                      make_error<RawError>(raw_error_code::corrupt_file,
+                                           "Could not read a bitmap."));
+  uint32_t NumBuckets = 0;
+  for (uint8_t B : Bitmap)
+    NumBuckets += countPopulation(B);
+
+  // Hash buckets follow.
+  if (auto EC = Reader.readArray(HashBuckets, NumBuckets))
+    return joinErrors(std::move(EC),
+                      make_error<RawError>(raw_error_code::corrupt_file,
+                                           "Hash buckets corrupted."));
+
+  return Error::success();
+}
+
+Error readGSIHashHeader(const GSIHashHeader *&HashHdr,
+                        msf::StreamReader &Reader) {
+  if (Reader.readObject(HashHdr))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Stream does not contain a GSIHashHeader.");
+
+  if (HashHdr->VerSignature != GSIHashHeader::HdrSignature)
+    return make_error<RawError>(
+        raw_error_code::feature_unsupported,
+        "GSIHashHeader signature (0xffffffff) not found.");
+
+  return Error::success();
+}
+
+Error readGSIHashRecords(msf::FixedStreamArray<PSHashRecord> &HashRecords,
+                         const GSIHashHeader *HashHdr,
+                         msf::StreamReader &Reader) {
+  if (auto EC = checkHashHdrVersion(HashHdr))
+    return EC;
+
+  // HashHdr->HrSize specifies the number of bytes of PSHashRecords we have.
+  // Verify that we can read them all.
+  if (HashHdr->HrSize % sizeof(PSHashRecord))
+    return make_error<RawError>(raw_error_code::corrupt_file,
+                                "Invalid HR array size.");
+  uint32_t NumHashRecords = HashHdr->HrSize / sizeof(PSHashRecord);
+  if (auto EC = Reader.readArray(HashRecords, NumHashRecords))
+    return joinErrors(std::move(EC),
+                      make_error<RawError>(raw_error_code::corrupt_file,
+                                           "Error reading hash records."));
+
+  return Error::success();
+}
+}
+}

lib/DebugInfo/PDB/Raw/GSI.h

@@ -0,0 +1,70 @@
+//===- GSI.h - Common Declarations for GlobalsStream and PublicsStream ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The data structures defined in this file are based on the reference
+// implementation which is available at
+// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h
+//
+// When you are reading the reference source code, you'd find the
+// information below useful.
+//
+//  - ppdb1->m_fMinimalDbgInfo seems to be always true.
+//  - SMALLBUCKETS macro is defined.
+//
+// The reference doesn't compile, so I learned just by reading code.
+// It's not guaranteed to be correct.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_DEBUGINFO_PDB_RAW_GSI_H
+#define LLVM_LIB_DEBUGINFO_PDB_RAW_GSI_H
+
+#include "llvm/DebugInfo/MSF/StreamArray.h"
+#include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
+
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+
+namespace msf {
+class StreamReader;
+}
+
+namespace pdb {
+
+/// From https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.cpp
+static const unsigned IPHR_HASH = 4096;
+
+/// Header of the hash tables found in the globals and publics sections.
+/// Based on GSIHashHeader in
+/// https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h
+struct GSIHashHeader {
+  enum : unsigned {
+    HdrSignature = ~0U,
+    HdrVersion = 0xeffe0000 + 19990810,
+  };
+  support::ulittle32_t VerSignature;
+  support::ulittle32_t VerHdr;
+  support::ulittle32_t HrSize;
+  support::ulittle32_t NumBuckets;
+};
+
+Error readGSIHashBuckets(
+    msf::FixedStreamArray<support::ulittle32_t> &HashBuckets,
+    const GSIHashHeader *HashHdr, msf::StreamReader &Reader);
+Error readGSIHashHeader(const GSIHashHeader *&HashHdr,
+                        msf::StreamReader &Reader);
+Error readGSIHashRecords(msf::FixedStreamArray<PSHashRecord> &HashRecords,
+                         const GSIHashHeader *HashHdr,
+                         msf::StreamReader &Reader);
+}
+}
+
+#endif

lib/DebugInfo/PDB/Raw/GlobalsStream.cpp

@@ -0,0 +1,46 @@
+//===- GlobalsStream.cpp - PDB Index of Symbols by Name ---- ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/PDB/Raw/GlobalsStream.h"
+
+#include "GSI.h"
+#include "llvm/DebugInfo/PDB/Raw/PDBFile.h"
+#include "llvm/DebugInfo/PDB/Raw/RawError.h"
+#include "llvm/DebugInfo/PDB/Raw/RawTypes.h"
+
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Error.h"
+
+using namespace llvm;
+using namespace llvm::msf;
+using namespace llvm::pdb;
+
+GlobalsStream::GlobalsStream(std::unique_ptr<MappedBlockStream> Stream)
+    : Stream(std::move(Stream)) {}
+
+GlobalsStream::~GlobalsStream() {}
+
+Error GlobalsStream::reload() {
+  StreamReader Reader(*Stream);
+
+  const GSIHashHeader *HashHdr;
+  if (auto EC = readGSIHashHeader(HashHdr, Reader))
+    return EC;
+
+  if (auto EC = readGSIHashRecords(HashRecords, HashHdr, Reader))
+    return EC;
+
+  if (auto EC = readGSIHashBuckets(HashBuckets, HashHdr, Reader))
+    return EC;
+  NumBuckets = HashBuckets.size();
+
+  return Error::success();
+}
+
+Error GlobalsStream::commit() { return Error::success(); }

lib/DebugInfo/PDB/Raw/PDBFile.cpp

@@ -15,6 +15,7 @@
 #include "llvm/DebugInfo/MSF/StreamReader.h"
 #include "llvm/DebugInfo/MSF/StreamWriter.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
+#include "llvm/DebugInfo/PDB/Raw/GlobalsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/NameHashTable.h"
 #include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
@@ -217,6 +218,22 @@ llvm::ArrayRef<support::ulittle32_t> PDBFile::getDirectoryBlockArray() const {
   return ContainerLayout.DirectoryBlocks;
 }
 
+Expected<GlobalsStream &> PDBFile::getPDBGlobalsStream() {
+  if (!Globals) {
+    auto DbiS = getPDBDbiStream();
+    if (!DbiS)
+      return DbiS.takeError();
+
+    auto GlobalS = MappedBlockStream::createIndexedStream(
+        ContainerLayout, *Buffer, DbiS->getGlobalSymbolStreamIndex());
+    auto TempGlobals = llvm::make_unique<GlobalsStream>(std::move(GlobalS));
+    if (auto EC = TempGlobals->reload())
+      return std::move(EC);
+    Globals = std::move(TempGlobals);
+  }
+  return *Globals;
+}
+
 Expected<InfoStream &> PDBFile::getPDBInfoStream() {
   if (!Info) {
     auto InfoS = MappedBlockStream::createIndexedStream(ContainerLayout,

lib/DebugInfo/PDB/Raw/PublicsStream.cpp

@@ -24,6 +24,7 @@
 
 #include "llvm/DebugInfo/PDB/Raw/PublicsStream.h"
 
+#include "GSI.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
 #include "llvm/DebugInfo/CodeView/TypeRecord.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
@@ -44,8 +45,6 @@ using namespace llvm::support;
 using namespace llvm::pdb;
 
 
-static const unsigned IPHR_HASH = 4096;
-
 // This is PSGSIHDR struct defined in
 // https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/dbi/gsi.h
 struct PublicsStream::HeaderInfo {
@@ -59,18 +58,6 @@ struct PublicsStream::HeaderInfo {
   ulittle32_t NumSections;
 };
 
-// This is GSIHashHdr.
-struct PublicsStream::GSIHashHeader {
-  enum : unsigned {
-    HdrSignature = ~0U,
-    HdrVersion = 0xeffe0000 + 19990810,
-  };
-  ulittle32_t VerSignature;
-  ulittle32_t VerHdr;
-  ulittle32_t HrSize;
-  ulittle32_t NumBuckets;
-};
-
 PublicsStream::PublicsStream(PDBFile &File,
                              std::unique_ptr<MappedBlockStream> Stream)
     : Pdb(File), Stream(std::move(Stream)) {}
@@ -98,40 +85,15 @@ Error PublicsStream::reload() {
     return make_error<RawError>(raw_error_code::corrupt_file,
                                 "Publics Stream does not contain a header.");
 
-  if (Reader.readObject(HashHdr))
-    return make_error<RawError>(raw_error_code::corrupt_file,
-                                "Publics Stream does not contain a header.");
+  if (auto EC = readGSIHashHeader(HashHdr, Reader))
+    return EC;
 
-  // An array of HashRecord follows. Read them.
-  if (HashHdr->HrSize % sizeof(PSHashRecord))
-    return make_error<RawError>(raw_error_code::corrupt_file,
-                                "Invalid HR array size.");
-  uint32_t NumHashRecords = HashHdr->HrSize / sizeof(PSHashRecord);
-  if (auto EC = Reader.readArray(HashRecords, NumHashRecords))
-    return joinErrors(std::move(EC),
-                      make_error<RawError>(raw_error_code::corrupt_file,
-                                           "Could not read an HR array"));
+  if (auto EC = readGSIHashRecords(HashRecords, HashHdr, Reader))
+    return EC;
 
-  // A bitmap of a fixed length follows.
-  size_t BitmapSizeInBits = alignTo(IPHR_HASH + 1, 32);
-  uint32_t NumBitmapEntries = BitmapSizeInBits / 8;
-  if (auto EC = Reader.readBytes(Bitmap, NumBitmapEntries))
-    return joinErrors(std::move(EC),
-                      make_error<RawError>(raw_error_code::corrupt_file,
-                                           "Could not read a bitmap."));
-  for (uint8_t B : Bitmap)
-    NumBuckets += countPopulation(B);
-
-  // We don't yet understand the following data structures completely,
-  // but we at least know the types and sizes. Here we are trying
-  // to read the stream till end so that we at least can detect
-  // corrupted streams.
-
-  // Hash buckets follow.
-  if (auto EC = Reader.readArray(HashBuckets, NumBuckets))
-    return joinErrors(std::move(EC),
-                      make_error<RawError>(raw_error_code::corrupt_file,
-                                           "Hash buckets corrupted."));
+  if (auto EC = readGSIHashBuckets(HashBuckets, HashHdr, Reader))
+    return EC;
+  NumBuckets = HashBuckets.size();
 
   // Something called "address map" follows.
   uint32_t NumAddressMapEntries = Header->AddrMap / sizeof(uint32_t);

test/DebugInfo/PDB/pdbdump-headers.test

@@ -1,5 +1,5 @@
 ; RUN: llvm-pdbdump raw -headers -tpi-records -tpi-record-bytes -module-syms \
-; RUN:              -sym-record-bytes -publics -module-files \
+; RUN:              -sym-record-bytes -globals -publics -module-files \
 ; RUN:              -stream-summary -stream-blocks -ipi-records -ipi-record-bytes \
 ; RUN:              -section-contribs -section-map -section-headers -line-info \
 ; RUN:              -tpi-hash -fpo -page-stats %p/Inputs/empty.pdb | FileCheck -check-prefix=EMPTY %s
@@ -794,7 +794,12 @@
 ; EMPTY-NEXT:     SecByteLength: 4294967295
 ; EMPTY-NEXT:   }
 ; EMPTY-NEXT: ]
-; EMPTY:      Publics Stream {
+; EMPTY-NEXT: Globals Stream {
+; EMPTY-NEXT:   Stream number: 6
+; EMPTY-NEXT:   Number of buckets: 2
+; EMPTY-NEXT:   Hash Buckets: [0, 12]
+; EMPTY-NEXT: }
+; EMPTY-NEXT: Publics Stream {
 ; EMPTY-NEXT:   Stream number: 7
 ; EMPTY-NEXT:   SymHash: 556
 ; EMPTY-NEXT:   AddrMap: 8
@@ -1552,6 +1557,11 @@
 ; ALL:     SecByteLength: 4294967295
 ; ALL:   }
 ; ALL: ]
+; ALL: Globals Stream {
+; ALL:   Stream number: 6
+; ALL:   Number of buckets: 2
+; ALL:   Hash Buckets: [0, 12]
+; ALL: }
 ; ALL: Publics Stream {
 ; ALL:   Stream number: 7
 ; ALL:   SymHash: 556

tools/llvm-pdbdump/LLVMOutputStyle.cpp

@@ -18,6 +18,7 @@
 #include "llvm/DebugInfo/PDB/PDBExtras.h"
 #include "llvm/DebugInfo/PDB/Raw/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Raw/EnumTables.h"
+#include "llvm/DebugInfo/PDB/Raw/GlobalsStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ISectionContribVisitor.h"
 #include "llvm/DebugInfo/PDB/Raw/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Raw/ModInfo.h"
@@ -122,6 +123,9 @@ Error LLVMOutputStyle::dump() {
   if (auto EC = dumpSectionMap())
     return EC;
 
+  if (auto EC = dumpGlobalsStream())
+    return EC;
+
   if (auto EC = dumpPublicsStream())
     return EC;
 
@@ -343,6 +347,26 @@ void LLVMOutputStyle::dumpBitVector(StringRef Name, const BitVector &V) {
   P.printList(Name, Vec);
 }
 
+Error LLVMOutputStyle::dumpGlobalsStream() {
+  if (!opts::raw::DumpGlobals)
+    return Error::success();
+
+  DictScope D(P, "Globals Stream");
+  auto Globals = File.getPDBGlobalsStream();
+  if (!Globals)
+    return Globals.takeError();
+
+  auto Dbi = File.getPDBDbiStream();
+  if (!Dbi)
+    return Dbi.takeError();
+
+  P.printNumber("Stream number", Dbi->getGlobalSymbolStreamIndex());
+  P.printNumber("Number of buckets", Globals->getNumBuckets());
+  P.printList("Hash Buckets", Globals->getHashBuckets());
+
+  return Error::success();
+}
+
 Error LLVMOutputStyle::dumpStreamBlocks() {
   if (!opts::raw::DumpStreamBlocks)
     return Error::success();

tools/llvm-pdbdump/LLVMOutputStyle.h

@@ -31,6 +31,7 @@ private:
   Error dumpStreamSummary();
   Error dumpFreePageMap();
   Error dumpBlockRanges();
+  Error dumpGlobalsStream();
   Error dumpStreamBytes();
   Error dumpStreamBlocks();
   Error dumpInfoStream();

tools/llvm-pdbdump/llvm-pdbdump.cpp

@@ -215,6 +215,8 @@ cl::opt<bool> DumpLineInfo("line-info",
                            cl::cat(FileOptions), cl::sub(RawSubcommand));
 
 // SYMBOL OPTIONS
+cl::opt<bool> DumpGlobals("globals", cl::desc("dump globals stream data"),
+                          cl::cat(SymbolOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpModuleSyms("module-syms", cl::desc("dump module symbols"),
                              cl::cat(SymbolOptions), cl::sub(RawSubcommand));
 cl::opt<bool> DumpPublics("publics", cl::desc("dump Publics stream data"),
@@ -559,6 +561,7 @@ int main(int argc_, const char *argv_[]) {
     opts::raw::DumpModules = true;
     opts::raw::DumpModuleFiles = true;
     opts::raw::DumpModuleSyms = true;
+    opts::raw::DumpGlobals = true;
     opts::raw::DumpPublics = true;
     opts::raw::DumpSectionHeaders = true;
     opts::raw::DumpStreamSummary = true;

tools/llvm-pdbdump/llvm-pdbdump.h

@@ -43,6 +43,7 @@ struct BlockRange {
 extern llvm::Optional<BlockRange> DumpBlockRange;
 extern llvm::cl::list<uint32_t> DumpStreamData;
 
+extern llvm::cl::opt<bool> DumpGlobals;
 extern llvm::cl::opt<bool> DumpHeaders;
 extern llvm::cl::opt<bool> DumpStreamBlocks;
 extern llvm::cl::opt<bool> DumpStreamSummary;