[pdbutil] Add a command to dump the FPM.

Recently problems have been discovered in the way we write the FPM
(free page map).  In order to fix this, we first need to establish
a baseline about what a correct FPM looks like using an MSVC
generated PDB, so that we can then make our own generated PDBs
match.  And in order to do this, the dumper needs a mode where it
can dump an FPM so that we can write tests for it.

This patch adds a command to dump the FPM, as well as a test against
a known-good PDB.

llvm-svn: 309894

include/llvm/DebugInfo/MSF/MSFCommon.h

@@ -59,6 +59,23 @@ struct MSFLayout {
   std::vector<ArrayRef<support::ulittle32_t>> StreamMap;
 };
 
+/// \brief Describes the layout of a stream in an MSF layout.  A "stream" here
+/// is defined as any logical unit of data which may be arranged inside the MSF
+/// file as a sequence of (possibly discontiguous) blocks.  When we want to read
+/// from a particular MSF Stream, we fill out a stream layout structure and the
+/// reader uses it to determine which blocks in the underlying MSF file contain
+/// the data, so that it can be pieced together in the right order.
+class MSFStreamLayout {
+public:
+  uint32_t Length;
+  std::vector<support::ulittle32_t> Blocks;
+};
+
+/// \brief Determine the layout of the FPM stream, given the MSF layout.  An FPM
+/// stream spans 1 or more blocks, each at equally spaced intervals throughout
+/// the file.
+MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf);
+
 inline bool isValidBlockSize(uint32_t Size) {
   switch (Size) {
   case 512:

include/llvm/DebugInfo/MSF/MSFStreamLayout.h

@@ -1,35 +0,0 @@
-//===- MSFStreamLayout.h - Describes the layout of a stream -----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H
-#define LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H
-
-#include "llvm/Support/Endian.h"
-
-#include <cstdint>
-#include <vector>
-
-namespace llvm {
-namespace msf {
-
-/// \brief Describes the layout of a stream in an MSF layout.  A "stream" here
-/// is defined as any logical unit of data which may be arranged inside the MSF
-/// file as a sequence of (possibly discontiguous) blocks.  When we want to read
-/// from a particular MSF Stream, we fill out a stream layout structure and the
-/// reader uses it to determine which blocks in the underlying MSF file contain
-/// the data, so that it can be pieced together in the right order.
-class MSFStreamLayout {
-public:
-  uint32_t Length;
-  std::vector<support::ulittle32_t> Blocks;
-};
-} // namespace msf
-} // namespace llvm
-
-#endif // LLVM_DEBUGINFO_MSF_MSFSTREAMLAYOUT_H

include/llvm/DebugInfo/MSF/MappedBlockStream.h

@@ -12,7 +12,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/DebugInfo/MSF/MSFStreamLayout.h"
+#include "llvm/DebugInfo/MSF/MSFCommon.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStream.h"
 #include "llvm/Support/BinaryStreamRef.h"

include/llvm/DebugInfo/PDB/Native/PDBFile.h

@@ -13,7 +13,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/DebugInfo/MSF/IMSFFile.h"
 #include "llvm/DebugInfo/MSF/MSFCommon.h"
-#include "llvm/DebugInfo/MSF/MSFStreamLayout.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/BinaryStreamRef.h"
 #include "llvm/Support/Endian.h"
@@ -72,8 +71,6 @@ public:
   Error setBlockData(uint32_t BlockIndex, uint32_t Offset,
                      ArrayRef<uint8_t> Data) const override;
 
-  ArrayRef<uint32_t> getFpmPages() const { return FpmPages; }
-
   ArrayRef<support::ulittle32_t> getStreamSizes() const {
     return ContainerLayout.StreamSizes;
   }
@@ -87,6 +84,7 @@ public:
   ArrayRef<support::ulittle32_t> getDirectoryBlockArray() const;
 
   msf::MSFStreamLayout getStreamLayout(uint32_t StreamIdx) const;
+  msf::MSFStreamLayout getFpmStreamLayout() const;
 
   Error parseFileHeaders();
   Error parseStreamData();
@@ -124,7 +122,6 @@ private:
 
   std::unique_ptr<BinaryStream> Buffer;
 
-  std::vector<uint32_t> FpmPages;
   msf::MSFLayout ContainerLayout;
 
   std::unique_ptr<GlobalsStream> Globals;

lib/DebugInfo/MSF/MSFCommon.cpp

@@ -59,3 +59,18 @@ Error llvm::msf::validateSuperBlock(const SuperBlock &SB) {
 
   return Error::success();
 }
+
+MSFStreamLayout llvm::msf::getFpmStreamLayout(const MSFLayout &Msf) {
+  MSFStreamLayout FL;
+  uint32_t NumFpmIntervals = getNumFpmIntervals(Msf);
+  support::ulittle32_t FpmBlock = Msf.SB->FreeBlockMapBlock;
+  assert(FpmBlock == 1 || FpmBlock == 2);
+  while (NumFpmIntervals > 0) {
+    FL.Blocks.push_back(FpmBlock);
+    FpmBlock += msf::getFpmIntervalLength(Msf);
+    --NumFpmIntervals;
+  }
+  FL.Length = getFullFpmByteSize(Msf);
+
+  return FL;
+}

lib/DebugInfo/MSF/MappedBlockStream.cpp

@@ -11,7 +11,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/DebugInfo/MSF/MSFCommon.h"
-#include "llvm/DebugInfo/MSF/MSFStreamLayout.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
@@ -36,19 +35,6 @@ public:
 
 } // end anonymous namespace
 
-static void initializeFpmStreamLayout(const MSFLayout &Layout,
-                                      MSFStreamLayout &FpmLayout) {
-  uint32_t NumFpmIntervals = msf::getNumFpmIntervals(Layout);
-  support::ulittle32_t FpmBlock = Layout.SB->FreeBlockMapBlock;
-  assert(FpmBlock == 1 || FpmBlock == 2);
-  while (NumFpmIntervals > 0) {
-    FpmLayout.Blocks.push_back(FpmBlock);
-    FpmBlock += msf::getFpmIntervalLength(Layout);
-    --NumFpmIntervals;
-  }
-  FpmLayout.Length = msf::getFullFpmByteSize(Layout);
-}
-
 using Interval = std::pair<uint32_t, uint32_t>;
 
 static Interval intersect(const Interval &I1, const Interval &I2) {
@@ -95,8 +81,7 @@ std::unique_ptr<MappedBlockStream>
 MappedBlockStream::createFpmStream(const MSFLayout &Layout,
                                    BinaryStreamRef MsfData,
                                    BumpPtrAllocator &Allocator) {
-  MSFStreamLayout SL;
+  MSFStreamLayout SL(getFpmStreamLayout(Layout));
-  initializeFpmStreamLayout(Layout, SL);
   return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator);
 }
 
@@ -363,8 +348,7 @@ std::unique_ptr<WritableMappedBlockStream>
 WritableMappedBlockStream::createFpmStream(const MSFLayout &Layout,
                                            WritableBinaryStreamRef MsfData,
                                            BumpPtrAllocator &Allocator) {
-  MSFStreamLayout SL;
+  MSFStreamLayout SL(getFpmStreamLayout(Layout));
-  initializeFpmStreamLayout(Layout, SL);
   return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator);
 }
 

lib/DebugInfo/PDB/Native/PDBFile.cpp

@@ -238,6 +238,10 @@ MSFStreamLayout PDBFile::getStreamLayout(uint32_t StreamIdx) const {
   return Result;
 }
 
+msf::MSFStreamLayout PDBFile::getFpmStreamLayout() const {
+  return msf::getFpmStreamLayout(ContainerLayout);
+}
+
 Expected<GlobalsStream &> PDBFile::getPDBGlobalsStream() {
   if (!Globals) {
     auto DbiS = getPDBDbiStream();

test/DebugInfo/PDB/dump-fpm.test

@@ -0,0 +1,9 @@
+RUN: llvm-pdbutil bytes -fpm %p/Inputs/empty.pdb | FileCheck %s
+
+CHECK:                             Free Page Map
+CHECK-NEXT: ============================================================
+CHECK-NEXT: Block 2 (
+CHECK-NEXT:   2000: 380300FE FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF  |8...............................|
+CHECK-NEXT:   2020: FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF  |................................|
+CHECK:        2FE0: FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF  |................................|
+CHECK-NEXT: )

tools/llvm-pdbutil/BytesOutputStyle.cpp

@@ -15,6 +15,7 @@
 
 #include "llvm/DebugInfo/CodeView/Formatters.h"
 #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h"
+#include "llvm/DebugInfo/MSF/MSFCommon.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Native/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Native/InfoStream.h"
@@ -120,6 +121,11 @@ Error BytesOutputStyle::dump() {
     P.NewLine();
   }
 
+  if (opts::bytes::Fpm) {
+    dumpFpm();
+    P.NewLine();
+  }
+
   if (!opts::bytes::DumpStreamData.empty()) {
     dumpStreamBytes();
     P.NewLine();
@@ -480,6 +486,13 @@ BytesOutputStyle::initializeTypes(uint32_t StreamIdx) {
   return *TypeCollection;
 }
 
+void BytesOutputStyle::dumpFpm() {
+  printHeader(P, "Free Page Map");
+
+  msf::MSFStreamLayout FpmLayout = File.getFpmStreamLayout();
+  P.formatMsfStreamBlocks(File, FpmLayout);
+}
+
 void BytesOutputStyle::dumpStreamBytes() {
   if (StreamPurposes.empty())
     discoverStreamPurposes(File, StreamPurposes);

tools/llvm-pdbutil/BytesOutputStyle.h

@@ -35,6 +35,7 @@ private:
   void dumpNameMap();
   void dumpBlockRanges(uint32_t Min, uint32_t Max);
   void dumpByteRanges(uint32_t Min, uint32_t Max);
+  void dumpFpm();
   void dumpStreamBytes();
 
   void dumpSectionContributions();

tools/llvm-pdbutil/LinePrinter.cpp

@@ -13,7 +13,6 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/DebugInfo/MSF/MSFCommon.h"
-#include "llvm/DebugInfo/MSF/MSFStreamLayout.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
 #include "llvm/DebugInfo/PDB/UDTLayout.h"
@@ -246,6 +245,30 @@ void LinePrinter::formatMsfStreamData(StringRef Label, PDBFile &File,
   OS << ")";
 }
 
+void LinePrinter::formatMsfStreamBlocks(
+    PDBFile &File, const msf::MSFStreamLayout &StreamLayout) {
+  auto Blocks = makeArrayRef(StreamLayout.Blocks);
+  uint32_t L = StreamLayout.Length;
+
+  while (L > 0) {
+    NewLine();
+    assert(!Blocks.empty());
+    OS << formatv("Block {0} (\n", uint32_t(Blocks.front()));
+    uint32_t UsedBytes = std::min(L, File.getBlockSize());
+    ArrayRef<uint8_t> BlockData =
+        cantFail(File.getBlockData(Blocks.front(), File.getBlockSize()));
+    uint64_t BaseOffset = Blocks.front();
+    BaseOffset *= File.getBlockSize();
+    OS << format_bytes_with_ascii(BlockData, BaseOffset, 32, 4,
+                                  CurrentIndent + IndentSpaces, true);
+    NewLine();
+    OS << ")";
+    NewLine();
+    L -= UsedBytes;
+    Blocks = Blocks.drop_front();
+  }
+}
+
 bool LinePrinter::IsTypeExcluded(llvm::StringRef TypeName, uint32_t Size) {
   if (IsItemExcluded(TypeName, IncludeTypeFilters, ExcludeTypeFilters))
     return true;

tools/llvm-pdbutil/LinePrinter.h

@@ -60,6 +60,7 @@ public:
   void formatMsfStreamData(StringRef Label, PDBFile &File,
                            const msf::MSFStreamLayout &Stream,
                            BinarySubstreamRef Substream);
+  void formatMsfStreamBlocks(PDBFile &File, const msf::MSFStreamLayout &Stream);
 
   bool hasColor() const { return UseColor; }
   raw_ostream &getStream() { return OS; }

tools/llvm-pdbutil/llvm-pdbutil.cpp

@@ -342,6 +342,8 @@ cl::list<std::string>
 
 cl::opt<bool> NameMap("name-map", cl::desc("Dump bytes of PDB Name Map"),
                       cl::sub(BytesSubcommand), cl::cat(PdbBytes));
+cl::opt<bool> Fpm("fpm", cl::desc("Dump free page map"),
+                  cl::sub(BytesSubcommand), cl::cat(MsfBytes));
 
 cl::opt<bool> SectionContributions("sc", cl::desc("Dump section contributions"),
                                    cl::sub(BytesSubcommand), cl::cat(DbiBytes));

tools/llvm-pdbutil/llvm-pdbutil.h

@@ -102,6 +102,7 @@ extern llvm::Optional<NumberRange> DumpBlockRange;
 extern llvm::Optional<NumberRange> DumpByteRange;
 extern llvm::cl::list<std::string> DumpStreamData;
 extern llvm::cl::opt<bool> NameMap;
+extern llvm::cl::opt<bool> Fpm;
 
 extern llvm::cl::opt<bool> SectionContributions;
 extern llvm::cl::opt<bool> SectionMap;
@@ -123,6 +124,7 @@ extern llvm::cl::opt<bool> SplitChunks;
 namespace dump {
 
 extern llvm::cl::opt<bool> DumpSummary;
+extern llvm::cl::opt<bool> DumpFpm;
 extern llvm::cl::opt<bool> DumpStreams;
 extern llvm::cl::opt<bool> DumpStreamBlocks;
 

unittests/DebugInfo/PDB/MappedBlockStreamTest.cpp

@@ -10,7 +10,6 @@
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/MSF/IMSFFile.h"
 #include "llvm/DebugInfo/MSF/MSFError.h"
-#include "llvm/DebugInfo/MSF/MSFStreamLayout.h"
 #include "llvm/Support/BinaryByteStream.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/BinaryStreamRef.h"