From f070dd590ecacd9a62e558d4ea0791f4f1f4715c Mon Sep 17 00:00:00 2001 From: Zachary Turner Date: Fri, 27 May 2016 01:54:44 +0000 Subject: [PATCH] [codeview,pdb] Try really hard to conserve memory when reading. PDBs can be extremely large. We're already mapping the entire PDB into the process's address space, but to make matters worse the blocks of the PDB are not arranged contiguously. So, when we have something like an array or a string embedded into the stream, we have to make a copy. Since it's convenient to use traditional data structures to iterate and manipulate these records, we need the memory to be contiguous. As a result of this, we were using roughly twice as much memory as the file size of the PDB, because every stream was copied out and re-stitched together contiguously. This patch addresses this by improving the MappedBlockStream to allocate from a BumpPtrAllocator only when a read requires a discontiguous read. Furthermore, it introduces some data structures backed by a stream which can iterate over both fixed and variable length records of a PDB. Since everything is backed by a stream and not a buffer, we can read almost everything from the PDB with zero copies. Differential Revision: http://reviews.llvm.org/D20654 Reviewed By: ruiu llvm-svn: 270951 --- include/llvm/DebugInfo/CodeView/ByteStream.h | 14 +- include/llvm/DebugInfo/CodeView/StreamArray.h | 188 ++++++++++++++++++ .../llvm/DebugInfo/CodeView/StreamInterface.h | 11 +- .../llvm/DebugInfo/CodeView/StreamReader.h | 38 +++- include/llvm/DebugInfo/CodeView/StreamRef.h | 82 ++++++++ include/llvm/DebugInfo/PDB/Raw/DbiStream.h | 19 +- .../DebugInfo/PDB/Raw/MappedBlockStream.h | 11 +- include/llvm/DebugInfo/PDB/Raw/ModInfo.h | 30 ++- include/llvm/DebugInfo/PDB/Raw/ModStream.h | 7 +- .../llvm/DebugInfo/PDB/Raw/NameHashTable.h | 4 +- .../llvm/DebugInfo/PDB/Raw/PublicsStream.h | 4 +- include/llvm/DebugInfo/PDB/Raw/TpiStream.h | 9 +- lib/DebugInfo/CodeView/ByteStream.cpp | 27 +-- lib/DebugInfo/CodeView/StreamReader.cpp | 66 ++++-- lib/DebugInfo/PDB/CMakeLists.txt | 1 + lib/DebugInfo/PDB/Raw/DbiStream.cpp | 85 ++++---- lib/DebugInfo/PDB/Raw/InfoStream.cpp | 14 +- lib/DebugInfo/PDB/Raw/MappedBlockStream.cpp | 87 +++++--- lib/DebugInfo/PDB/Raw/ModInfo.cpp | 83 ++++---- lib/DebugInfo/PDB/Raw/ModStream.cpp | 8 +- lib/DebugInfo/PDB/Raw/NameHashTable.cpp | 30 ++- lib/DebugInfo/PDB/Raw/NameMap.cpp | 2 +- lib/DebugInfo/PDB/Raw/PublicsStream.cpp | 7 +- lib/DebugInfo/PDB/Raw/SymbolStream.cpp | 4 +- lib/DebugInfo/PDB/Raw/TpiStream.cpp | 13 +- tools/llvm-pdbdump/llvm-pdbdump.cpp | 10 +- 26 files changed, 615 insertions(+), 239 deletions(-) create mode 100644 include/llvm/DebugInfo/CodeView/StreamArray.h create mode 100644 include/llvm/DebugInfo/CodeView/StreamRef.h diff --git a/include/llvm/DebugInfo/CodeView/ByteStream.h b/include/llvm/DebugInfo/CodeView/ByteStream.h index 9d683b1cec6..ba162320235 100644 --- a/include/llvm/DebugInfo/CodeView/ByteStream.h +++ b/include/llvm/DebugInfo/CodeView/ByteStream.h @@ -24,20 +24,18 @@ class StreamReader; class ByteStream : public StreamInterface { public: ByteStream(); - explicit ByteStream(MutableArrayRef Bytes); - explicit ByteStream(uint32_t Length); + explicit ByteStream(MutableArrayRef Data); ~ByteStream() override; void reset(); - void initialize(MutableArrayRef Bytes); - void initialize(uint32_t Length); - Error initialize(StreamReader &Reader, uint32_t Length); + + void load(uint32_t Length); + Error load(StreamReader &Reader, uint32_t Length); Error readBytes(uint32_t Offset, MutableArrayRef Buffer) const override; - - Error getArrayRef(uint32_t Offset, ArrayRef &Buffer, - uint32_t Length) const override; + Error readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const override; uint32_t getLength() const override; diff --git a/include/llvm/DebugInfo/CodeView/StreamArray.h b/include/llvm/DebugInfo/CodeView/StreamArray.h new file mode 100644 index 00000000000..ecd3b19aa61 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/StreamArray.h @@ -0,0 +1,188 @@ +//===- StreamArray.h - Array backed by an arbitrary stream ----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H +#define LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H + +#include "llvm/DebugInfo/CodeView/StreamRef.h" + +#include + +namespace llvm { +namespace codeview { + +/// VarStreamArray represents an array of variable length records backed by a +/// stream. This could be a contiguous sequence of bytes in memory, it could +/// be a file on disk, or it could be a PDB stream where bytes are stored as +/// discontiguous blocks in a file. Usually it is desirable to treat arrays +/// as contiguous blocks of memory, but doing so with large PDB files, for +/// example, could mean allocating huge amounts of memory just to allow +/// re-ordering of stream data to be contiguous before iterating over it. By +/// abstracting this out, we need not duplicate this memory, and we can +/// iterate over arrays in arbitrarily formatted streams. +class VarStreamArrayIterator; + +class VarStreamArray { + friend class VarStreamArrayIterator; + typedef std::function LengthFuncType; + +public: + template + VarStreamArray(StreamRef Stream, const LengthFunc &Len) + : Stream(Stream), Len(Len) {} + + VarStreamArrayIterator begin() const; + VarStreamArrayIterator end() const; + +private: + StreamRef Stream; + LengthFuncType Len; // Function used to calculate legth of a record +}; + +class VarStreamArrayIterator { +public: + VarStreamArrayIterator(const VarStreamArray &Array) + : Array(&Array), IterRef(Array.Stream) { + ThisLen = Array.Len(IterRef); + } + VarStreamArrayIterator() : Array(nullptr), IterRef() {} + bool operator==(const VarStreamArrayIterator &R) const { + if (Array && R.Array) { + // Both have a valid array, make sure they're same. + assert(Array == R.Array); + return IterRef == R.IterRef; + } + + // Both iterators are at the end. + if (!Array && !R.Array) + return true; + + // One is not at the end and one is. + return false; + } + + bool operator!=(const VarStreamArrayIterator &R) { return !(*this == R); } + + StreamRef operator*() const { + ArrayRef Result; + return IterRef.keep_front(ThisLen); + } + + VarStreamArrayIterator &operator++() { + if (!Array || IterRef.getLength() == 0) + return *this; + IterRef = IterRef.drop_front(ThisLen); + if (IterRef.getLength() == 0) { + Array = nullptr; + ThisLen = 0; + } else { + ThisLen = Array->Len(IterRef); + } + return *this; + } + + VarStreamArrayIterator operator++(int) { + VarStreamArrayIterator Original = *this; + ++*this; + return Original; + } + +private: + const VarStreamArray *Array; + uint32_t ThisLen; + StreamRef IterRef; +}; + +inline VarStreamArrayIterator VarStreamArray::begin() const { + return VarStreamArrayIterator(*this); +} +inline VarStreamArrayIterator VarStreamArray::end() const { + return VarStreamArrayIterator(); +} + +template class FixedStreamArrayIterator; + +template class FixedStreamArray { + friend class FixedStreamArrayIterator; + static_assert(std::is_trivially_constructible::value, + "FixedStreamArray must be used with trivial types"); + +public: + FixedStreamArray() : Stream() {} + FixedStreamArray(StreamRef Stream) : Stream(Stream) { + assert(Stream.getLength() % sizeof(T) == 0); + } + + const T &operator[](uint32_t Index) const { + assert(Index < size()); + uint32_t Off = Index * sizeof(T); + ArrayRef Data; + if (auto EC = Stream.readBytes(Off, sizeof(T), Data)) { + assert(false && "Unexpected failure reading from stream"); + // This should never happen since we asserted that the stream length was + // an exact multiple of the element size. + consumeError(std::move(EC)); + } + return *reinterpret_cast(Data.data()); + } + + uint32_t size() const { return Stream.getLength() / sizeof(T); } + + FixedStreamArrayIterator begin() const { + return FixedStreamArrayIterator(*this, 0); + } + FixedStreamArrayIterator end() const { + return FixedStreamArrayIterator(*this); + } + +private: + StreamRef Stream; +}; + +template class FixedStreamArrayIterator { +public: + FixedStreamArrayIterator(const FixedStreamArray &Array) + : Array(Array), Index(uint32_t(-1)) {} + FixedStreamArrayIterator(const FixedStreamArray &Array, uint32_t Index) + : Array(Array), Index(Index) {} + + bool operator==(const FixedStreamArrayIterator &R) { + assert(&Array == &R.Array); + return Index == R.Index; + } + + bool operator!=(const FixedStreamArrayIterator &R) { + return !(*this == R); + } + + const T &operator*() const { return Array[Index]; } + + FixedStreamArrayIterator &operator++() { + if (Index == uint32_t(-1)) + return *this; + if (++Index >= Array.size()) + Index = uint32_t(-1); + return *this; + } + + FixedStreamArrayIterator operator++(int) { + FixedStreamArrayIterator Original = *this; + ++*this; + return Original; + } + +private: + const FixedStreamArray &Array; + uint32_t Index; +}; + +} // namespace codeview +} // namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H diff --git a/include/llvm/DebugInfo/CodeView/StreamInterface.h b/include/llvm/DebugInfo/CodeView/StreamInterface.h index 92c9041688e..fb73da35111 100644 --- a/include/llvm/DebugInfo/CodeView/StreamInterface.h +++ b/include/llvm/DebugInfo/CodeView/StreamInterface.h @@ -17,14 +17,21 @@ namespace llvm { namespace codeview { +/// StreamInterface abstracts the notion of a data stream. This way, an +/// implementation could implement trivial reading from a contiguous memory +/// buffer or, as in the case of PDB files, reading from a set of possibly +/// discontiguous blocks. The implementation is required to return references +/// to stable memory, so if this is not possible (for example in the case of +/// a PDB file with discontiguous blocks, it must keep its own pool of temp +/// storage. class StreamInterface { public: virtual ~StreamInterface() {} virtual Error readBytes(uint32_t Offset, MutableArrayRef Buffer) const = 0; - virtual Error getArrayRef(uint32_t Offset, ArrayRef &Buffer, - uint32_t Length) const = 0; + virtual Error readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const = 0; virtual uint32_t getLength() const = 0; }; diff --git a/include/llvm/DebugInfo/CodeView/StreamReader.h b/include/llvm/DebugInfo/CodeView/StreamReader.h index 34ff725235e..7d4b4afcd25 100644 --- a/include/llvm/DebugInfo/CodeView/StreamReader.h +++ b/include/llvm/DebugInfo/CodeView/StreamReader.h @@ -11,6 +11,8 @@ #define LLVM_DEBUGINFO_CODEVIEW_STREAMREADER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/CodeView/CodeViewError.h" +#include "llvm/DebugInfo/CodeView/StreamArray.h" #include "llvm/DebugInfo/CodeView/StreamInterface.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -20,18 +22,42 @@ namespace llvm { namespace codeview { +class StreamRef; + class StreamReader { public: StreamReader(const StreamInterface &S); + Error readBytes(uint32_t Size, ArrayRef &Buffer); Error readBytes(MutableArrayRef Buffer); + Error readInteger(uint16_t &Dest); Error readInteger(uint32_t &Dest); - Error readZeroString(std::string &Dest); + Error readZeroString(StringRef &Dest); + Error readFixedString(StringRef &Dest, uint32_t Length); + Error readStreamRef(StreamRef &Ref); + Error readStreamRef(StreamRef &Ref, uint32_t Length); - template Error readObject(T *Dest) { - MutableArrayRef Buffer(reinterpret_cast(Dest), - sizeof(T)); - return readBytes(Buffer); + template Error readObject(const T *&Dest) { + ArrayRef Buffer; + if (auto EC = readBytes(sizeof(T), Buffer)) + return EC; + Dest = reinterpret_cast(Buffer.data()); + return Error::success(); + } + + template + Error readArray(FixedStreamArray &Array, uint32_t NumItems) { + if (NumItems == 0) { + Array = FixedStreamArray(); + return Error::success(); + } + uint32_t Length = NumItems * sizeof(T); + if (Offset + Length > Stream.getLength()) + return make_error(cv_error_code::insufficient_buffer); + StreamRef View(Stream, Offset, Length); + Array = FixedStreamArray(View); + Offset += Length; + return Error::success(); } template Error readArray(MutableArrayRef Array) { @@ -40,8 +66,6 @@ public: return readBytes(Casted); } - Error getArrayRef(ArrayRef &Array, uint32_t Length); - void setOffset(uint32_t Off) { Offset = Off; } uint32_t getOffset() const { return Offset; } uint32_t getLength() const { return Stream.getLength(); } diff --git a/include/llvm/DebugInfo/CodeView/StreamRef.h b/include/llvm/DebugInfo/CodeView/StreamRef.h new file mode 100644 index 00000000000..1597f9a5ea0 --- /dev/null +++ b/include/llvm/DebugInfo/CodeView/StreamRef.h @@ -0,0 +1,82 @@ +//===- StreamRef.h - A copyable reference to a stream -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMREF_H +#define LLVM_DEBUGINFO_CODEVIEW_STREAMREF_H + +#include "llvm/DebugInfo/CodeView/StreamInterface.h" + +namespace llvm { +namespace codeview { + +class StreamRef : public StreamInterface { +public: + StreamRef() : Stream(nullptr), ViewOffset(0), Length(0) {} + StreamRef(const StreamInterface &Stream) + : Stream(&Stream), ViewOffset(0), Length(Stream.getLength()) {} + StreamRef(const StreamInterface &Stream, uint32_t Offset, uint32_t Length) + : Stream(&Stream), ViewOffset(Offset), Length(Length) {} + StreamRef(const StreamRef &Other) + : Stream(Other.Stream), ViewOffset(Other.ViewOffset), + Length(Other.Length) {} + + Error readBytes(uint32_t Offset, + MutableArrayRef Buffer) const override { + return Stream->readBytes(ViewOffset + Offset, Buffer); + } + + Error readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const override { + return Stream->readBytes(ViewOffset + Offset, Size, Buffer); + } + + uint32_t getLength() const override { return Length; } + StreamRef drop_front(uint32_t N) const { + if (!Stream) + return StreamRef(); + + N = std::min(N, Length); + return StreamRef(*Stream, ViewOffset + N, Length - N); + } + + StreamRef keep_front(uint32_t N) const { + if (!Stream) + return StreamRef(); + N = std::min(N, Length); + return StreamRef(*Stream, ViewOffset, N); + } + + bool operator==(const StreamRef &Other) const { + if (Stream != Other.Stream) + return false; + if (ViewOffset != Other.ViewOffset) + return false; + if (Length != Other.Length) + return false; + return true; + } + + bool operator!=(const StreamRef &Other) const { return !(*this == Other); } + + StreamRef &operator=(const StreamRef &Other) { + Stream = Other.Stream; + ViewOffset = Other.ViewOffset; + Length = Other.Length; + return *this; + } + +private: + const StreamInterface *Stream; + uint32_t ViewOffset; + uint32_t Length; +}; +} +} + +#endif // LLVM_DEBUGINFO_CODEVIEW_STREAMREF_H \ No newline at end of file diff --git a/include/llvm/DebugInfo/PDB/Raw/DbiStream.h b/include/llvm/DebugInfo/PDB/Raw/DbiStream.h index 91de6e64293..17f31f5695f 100644 --- a/include/llvm/DebugInfo/PDB/Raw/DbiStream.h +++ b/include/llvm/DebugInfo/PDB/Raw/DbiStream.h @@ -11,6 +11,8 @@ #define LLVM_DEBUGINFO_PDB_RAW_PDBDBISTREAM_H #include "llvm/DebugInfo/CodeView/ByteStream.h" +#include "llvm/DebugInfo/CodeView/StreamArray.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Raw/ModInfo.h" @@ -62,15 +64,16 @@ private: std::vector ModuleInfos; NameHashTable ECNames; - codeview::ByteStream ModInfoSubstream; - codeview::ByteStream SecContrSubstream; - codeview::ByteStream SecMapSubstream; - codeview::ByteStream FileInfoSubstream; - codeview::ByteStream TypeServerMapSubstream; - codeview::ByteStream ECSubstream; - codeview::ByteStream DbgHeader; + codeview::StreamRef ModInfoSubstream; + codeview::StreamRef SecContrSubstream; + codeview::StreamRef SecMapSubstream; + codeview::StreamRef FileInfoSubstream; + codeview::StreamRef TypeServerMapSubstream; + codeview::StreamRef ECSubstream; - std::unique_ptr Header; + codeview::FixedStreamArray DbgStreams; + + const HeaderInfo *Header; }; } } diff --git a/include/llvm/DebugInfo/PDB/Raw/MappedBlockStream.h b/include/llvm/DebugInfo/PDB/Raw/MappedBlockStream.h index 72f8b0c8667..864f8d44dd0 100644 --- a/include/llvm/DebugInfo/PDB/Raw/MappedBlockStream.h +++ b/include/llvm/DebugInfo/PDB/Raw/MappedBlockStream.h @@ -11,7 +11,9 @@ #define LLVM_DEBUGINFO_PDB_RAW_MAPPEDBLOCKSTREAM_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/DebugInfo/CodeView/StreamInterface.h" +#include "llvm/Support/Allocator.h" #include "llvm/Support/Error.h" #include #include @@ -27,14 +29,19 @@ public: Error readBytes(uint32_t Offset, MutableArrayRef Buffer) const override; - Error getArrayRef(uint32_t Offset, ArrayRef &Buffer, - uint32_t Length) const override; + Error readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const override; uint32_t getLength() const override { return StreamLength; } private: + bool tryReadContiguously(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const; + uint32_t StreamLength; std::vector BlockList; + mutable llvm::BumpPtrAllocator Pool; + mutable DenseMap CacheMap; const PDBFile &Pdb; }; diff --git a/include/llvm/DebugInfo/PDB/Raw/ModInfo.h b/include/llvm/DebugInfo/PDB/Raw/ModInfo.h index 3ed250ff42e..3b79d48a8a1 100644 --- a/include/llvm/DebugInfo/PDB/Raw/ModInfo.h +++ b/include/llvm/DebugInfo/PDB/Raw/ModInfo.h @@ -11,6 +11,7 @@ #define LLVM_DEBUGINFO_PDB_RAW_MODINFO_H #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" #include #include @@ -22,7 +23,8 @@ private: struct FileLayout; public: - ModInfo(const uint8_t *Bytes); + ModInfo(codeview::StreamRef Stream); + ModInfo(const ModInfo &Info); ~ModInfo(); bool hasECInfo() const; @@ -38,32 +40,26 @@ public: StringRef getModuleName() const; StringRef getObjFileName() const; + uint32_t getRecordLength() const; + private: + StringRef ModuleName; + StringRef ObjFileName; const FileLayout *Layout; }; struct ModuleInfoEx { - ModuleInfoEx(ModInfo Module) : Info(Module) {} + ModuleInfoEx(codeview::StreamRef Stream) : Info(Stream) {} + ModuleInfoEx(const ModuleInfoEx &Ex) + : Info(Ex.Info), SourceFiles(Ex.SourceFiles) {} ModInfo Info; std::vector SourceFiles; }; -class ModInfoIterator { -public: - ModInfoIterator(const uint8_t *Stream); - ModInfoIterator(const ModInfoIterator &Other); - - ModInfo operator*(); - ModInfoIterator &operator++(); - ModInfoIterator operator++(int); - bool operator==(const ModInfoIterator &Other); - bool operator!=(const ModInfoIterator &Other); - ModInfoIterator &operator=(const ModInfoIterator &Other); - -private: - const uint8_t *Bytes; -}; +inline uint32_t ModInfoRecordLength(const codeview::StreamInterface &Stream) { + return ModInfo(Stream).getRecordLength(); +} } // end namespace pdb } // end namespace llvm diff --git a/include/llvm/DebugInfo/PDB/Raw/ModStream.h b/include/llvm/DebugInfo/PDB/Raw/ModStream.h index 3f2b9049b46..e5bd2eba4ba 100644 --- a/include/llvm/DebugInfo/PDB/Raw/ModStream.h +++ b/include/llvm/DebugInfo/PDB/Raw/ModStream.h @@ -12,6 +12,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/DebugInfo/CodeView/ByteStream.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h" #include "llvm/Support/Error.h" @@ -36,9 +37,9 @@ private: MappedBlockStream Stream; codeview::ByteStream SymbolsSubstream; - codeview::ByteStream LinesSubstream; - codeview::ByteStream C13LinesSubstream; - codeview::ByteStream GlobalRefsSubstream; + codeview::StreamRef LinesSubstream; + codeview::StreamRef C13LinesSubstream; + codeview::StreamRef GlobalRefsSubstream; }; } } diff --git a/include/llvm/DebugInfo/PDB/Raw/NameHashTable.h b/include/llvm/DebugInfo/PDB/Raw/NameHashTable.h index daaa40d4765..2a28eaad8e2 100644 --- a/include/llvm/DebugInfo/PDB/Raw/NameHashTable.h +++ b/include/llvm/DebugInfo/PDB/Raw/NameHashTable.h @@ -12,7 +12,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" -#include "llvm/DebugInfo/CodeView/ByteStream.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" #include "llvm/Support/Error.h" #include #include @@ -39,7 +39,7 @@ public: ArrayRef name_ids() const; private: - codeview::ByteStream NamesBuffer; + codeview::StreamRef NamesBuffer; std::vector IDs; uint32_t Signature; uint32_t HashVersion; diff --git a/include/llvm/DebugInfo/PDB/Raw/PublicsStream.h b/include/llvm/DebugInfo/PDB/Raw/PublicsStream.h index f1d6be5895c..16dd7c09ac8 100644 --- a/include/llvm/DebugInfo/PDB/Raw/PublicsStream.h +++ b/include/llvm/DebugInfo/PDB/Raw/PublicsStream.h @@ -57,8 +57,8 @@ private: std::vector ThunkMap; std::vector SectionOffsets; - std::unique_ptr Header; - std::unique_ptr HashHdr; + const HeaderInfo *Header; + const GSIHashHeader *HashHdr; }; } } diff --git a/include/llvm/DebugInfo/PDB/Raw/TpiStream.h b/include/llvm/DebugInfo/PDB/Raw/TpiStream.h index 91dd891d033..4fb91103415 100644 --- a/include/llvm/DebugInfo/PDB/Raw/TpiStream.h +++ b/include/llvm/DebugInfo/PDB/Raw/TpiStream.h @@ -11,6 +11,7 @@ #define LLVM_DEBUGINFO_PDB_RAW_PDBTPISTREAM_H #include "llvm/DebugInfo/CodeView/ByteStream.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" #include "llvm/DebugInfo/CodeView/TypeStream.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h" @@ -48,11 +49,11 @@ private: HashFunctionType HashFunction; codeview::ByteStream RecordsBuffer; - codeview::ByteStream TypeIndexOffsetBuffer; - codeview::ByteStream HashValuesBuffer; - codeview::ByteStream HashAdjBuffer; + codeview::StreamRef TypeIndexOffsetBuffer; + codeview::StreamRef HashValuesBuffer; + codeview::StreamRef HashAdjBuffer; - std::unique_ptr Header; + const HeaderInfo *Header; }; } } diff --git a/lib/DebugInfo/CodeView/ByteStream.cpp b/lib/DebugInfo/CodeView/ByteStream.cpp index 879343afbfd..1ea976b6a26 100644 --- a/lib/DebugInfo/CodeView/ByteStream.cpp +++ b/lib/DebugInfo/CodeView/ByteStream.cpp @@ -17,31 +17,24 @@ using namespace llvm::codeview; ByteStream::ByteStream() {} -ByteStream::ByteStream(MutableArrayRef Bytes) { initialize(Bytes); } +ByteStream::ByteStream(MutableArrayRef Data) : Data(Data) {} -ByteStream::ByteStream(uint32_t Length) { initialize(Length); } - -ByteStream::~ByteStream() { reset(); } +ByteStream::~ByteStream() {} void ByteStream::reset() { Ownership.reset(); Data = MutableArrayRef(); } -void ByteStream::initialize(MutableArrayRef Bytes) { - reset(); - Data = Bytes; -} - -void ByteStream::initialize(uint32_t Length) { +void ByteStream::load(uint32_t Length) { reset(); if (Length > 0) Data = MutableArrayRef(new uint8_t[Length], Length); Ownership.reset(Data.data()); } -Error ByteStream::initialize(StreamReader &Reader, uint32_t Length) { - initialize(Length); +Error ByteStream::load(StreamReader &Reader, uint32_t Length) { + load(Length); auto EC = Reader.readBytes(Data); if (EC) reset(); @@ -52,15 +45,15 @@ Error ByteStream::readBytes(uint32_t Offset, MutableArrayRef Buffer) const { if (Data.size() < Buffer.size() + Offset) return make_error(cv_error_code::insufficient_buffer); - ::memcpy(Buffer.data(), Data.data() + Offset, Buffer.size()); + ::memcpy(Buffer.data() + Offset, Data.data(), Buffer.size()); return Error::success(); } -Error ByteStream::getArrayRef(uint32_t Offset, ArrayRef &Buffer, - uint32_t Length) const { - if (Data.size() < Length + Offset) +Error ByteStream::readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const { + if (Data.size() < Buffer.size() + Offset) return make_error(cv_error_code::insufficient_buffer); - Buffer = Data.slice(Offset, Length); + Buffer = Data.slice(Offset, Size); return Error::success(); } diff --git a/lib/DebugInfo/CodeView/StreamReader.cpp b/lib/DebugInfo/CodeView/StreamReader.cpp index 8a2926bc883..64985bfd0e2 100644 --- a/lib/DebugInfo/CodeView/StreamReader.cpp +++ b/lib/DebugInfo/CodeView/StreamReader.cpp @@ -10,12 +10,20 @@ #include "llvm/DebugInfo/CodeView/StreamReader.h" #include "llvm/DebugInfo/CodeView/CodeViewError.h" +#include "llvm/DebugInfo/CodeView/StreamRef.h" using namespace llvm; using namespace llvm::codeview; StreamReader::StreamReader(const StreamInterface &S) : Stream(S), Offset(0) {} +Error StreamReader::readBytes(uint32_t Size, ArrayRef &Buffer) { + if (auto EC = Stream.readBytes(Offset, Size, Buffer)) + return EC; + Offset += Size; + return Error::success(); +} + Error StreamReader::readBytes(MutableArrayRef Buffer) { if (auto EC = Stream.readBytes(Offset, Buffer)) return EC; @@ -23,29 +31,63 @@ Error StreamReader::readBytes(MutableArrayRef Buffer) { return Error::success(); } +Error StreamReader::readInteger(uint16_t &Dest) { + const support::ulittle16_t *P; + if (auto EC = readObject(P)) + return EC; + Dest = *P; + return Error::success(); +} + Error StreamReader::readInteger(uint32_t &Dest) { - support::ulittle32_t P; - if (auto EC = readObject(&P)) + const support::ulittle32_t *P; + if (auto EC = readObject(P)) return EC; - Dest = P; + Dest = *P; return Error::success(); } -Error StreamReader::readZeroString(std::string &Dest) { - Dest.clear(); - char C; +Error StreamReader::readZeroString(StringRef &Dest) { + uint32_t Length = 0; + // First compute the length of the string by reading 1 byte at a time. + uint32_t OriginalOffset = getOffset(); + const char *C; do { - if (auto EC = readObject(&C)) + if (auto EC = readObject(C)) return EC; - if (C != '\0') - Dest.push_back(C); - } while (C != '\0'); + if (*C != '\0') + ++Length; + } while (*C != '\0'); + // Now go back and request a reference for that many bytes. + uint32_t NewOffset = getOffset(); + setOffset(OriginalOffset); + + ArrayRef Data; + if (auto EC = readBytes(Length, Data)) + return EC; + Dest = StringRef(reinterpret_cast(Data.begin()), Data.size()); + + // Now set the offset back to where it was after we calculated the length. + setOffset(NewOffset); return Error::success(); } -Error StreamReader::getArrayRef(ArrayRef &Array, uint32_t Length) { - if (auto EC = Stream.getArrayRef(Offset, Array, Length)) +Error StreamReader::readFixedString(StringRef &Dest, uint32_t Length) { + ArrayRef Bytes; + if (auto EC = readBytes(Length, Bytes)) return EC; + Dest = StringRef(reinterpret_cast(Bytes.begin()), Bytes.size()); + return Error::success(); +} + +Error StreamReader::readStreamRef(StreamRef &Ref) { + return readStreamRef(Ref, bytesRemaining()); +} + +Error StreamReader::readStreamRef(StreamRef &Ref, uint32_t Length) { + if (bytesRemaining() < Length) + return make_error(cv_error_code::insufficient_buffer); + Ref = StreamRef(Stream, Offset, Length); Offset += Length; return Error::success(); } diff --git a/lib/DebugInfo/PDB/CMakeLists.txt b/lib/DebugInfo/PDB/CMakeLists.txt index 113a2ad0ec4..0938c0762c8 100644 --- a/lib/DebugInfo/PDB/CMakeLists.txt +++ b/lib/DebugInfo/PDB/CMakeLists.txt @@ -42,6 +42,7 @@ add_pdb_impl_folder(Raw Raw/SymbolStream.cpp Raw/TpiStream.cpp) +list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB/Raw") list(APPEND LIBPDB_ADDITIONAL_HEADER_DIRS "${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/PDB") add_llvm_library(LLVMDebugInfoPDB diff --git a/lib/DebugInfo/PDB/Raw/DbiStream.cpp b/lib/DebugInfo/PDB/Raw/DbiStream.cpp index 078384ebae3..35937574645 100644 --- a/lib/DebugInfo/PDB/Raw/DbiStream.cpp +++ b/lib/DebugInfo/PDB/Raw/DbiStream.cpp @@ -6,9 +6,9 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// - #include "llvm/DebugInfo/PDB/Raw/DbiStream.h" +#include "llvm/DebugInfo/CodeView/StreamArray.h" #include "llvm/DebugInfo/CodeView/StreamReader.h" #include "llvm/DebugInfo/PDB/Raw/InfoStream.h" #include "llvm/DebugInfo/PDB/Raw/ModInfo.h" @@ -73,7 +73,8 @@ struct DbiStream::HeaderInfo { ulittle32_t Reserved; // Pad to 64 bytes }; -DbiStream::DbiStream(PDBFile &File) : Pdb(File), Stream(StreamDBI, File) { +DbiStream::DbiStream(PDBFile &File) + : Pdb(File), Stream(StreamDBI, File), Header(nullptr) { static_assert(sizeof(HeaderInfo) == 64, "Invalid HeaderInfo size!"); } @@ -82,12 +83,10 @@ DbiStream::~DbiStream() {} Error DbiStream::reload() { codeview::StreamReader Reader(Stream); - Header.reset(new HeaderInfo()); - if (Stream.getLength() < sizeof(HeaderInfo)) return make_error(raw_error_code::corrupt_file, "DBI Stream does not contain a header."); - if (auto EC = Reader.readObject(Header.get())) + if (auto EC = Reader.readObject(Header)) return make_error(raw_error_code::corrupt_file, "DBI Stream does not contain a header."); @@ -137,30 +136,31 @@ Error DbiStream::reload() { return make_error(raw_error_code::corrupt_file, "DBI type server substream not aligned."); - if (auto EC = ModInfoSubstream.initialize(Reader, Header->ModiSubstreamSize)) + if (auto EC = + Reader.readStreamRef(ModInfoSubstream, Header->ModiSubstreamSize)) return EC; // Since each ModInfo in the stream is a variable length, we have to iterate // them to know how many there actually are. - auto Range = - llvm::make_range(ModInfoIterator(&ModInfoSubstream.data().front()), - ModInfoIterator(&ModInfoSubstream.data().back() + 1)); - for (auto Info : Range) - ModuleInfos.push_back(ModuleInfoEx(Info)); + codeview::VarStreamArray ModInfoArray(ModInfoSubstream, ModInfoRecordLength); + for (auto Info : ModInfoArray) { + ModuleInfos.emplace_back(Info); + } - if (auto EC = - SecContrSubstream.initialize(Reader, Header->SecContrSubstreamSize)) + if (auto EC = Reader.readStreamRef(SecContrSubstream, + Header->SecContrSubstreamSize)) return EC; - if (auto EC = SecMapSubstream.initialize(Reader, Header->SectionMapSize)) + if (auto EC = Reader.readStreamRef(SecMapSubstream, Header->SectionMapSize)) return EC; - if (auto EC = FileInfoSubstream.initialize(Reader, Header->FileInfoSize)) + if (auto EC = Reader.readStreamRef(FileInfoSubstream, Header->FileInfoSize)) return EC; if (auto EC = - TypeServerMapSubstream.initialize(Reader, Header->TypeServerSize)) + Reader.readStreamRef(TypeServerMapSubstream, Header->TypeServerSize)) return EC; - if (auto EC = ECSubstream.initialize(Reader, Header->ECSubstreamSize)) + if (auto EC = Reader.readStreamRef(ECSubstream, Header->ECSubstreamSize)) return EC; - if (auto EC = DbgHeader.initialize(Reader, Header->OptionalDbgHdrSize)) + if (auto EC = Reader.readArray(DbgStreams, Header->OptionalDbgHdrSize / + sizeof(ulittle16_t))) return EC; if (auto EC = initializeFileInfo()) @@ -247,25 +247,30 @@ Error DbiStream::initializeFileInfo() { // with the caveat that `NumSourceFiles` cannot be trusted, so // it is computed by summing `ModFileCounts`. // - const uint8_t *Buf = &FileInfoSubstream.data().front(); - auto FI = reinterpret_cast(Buf); - Buf += sizeof(FileInfoSubstreamHeader); + const FileInfoSubstreamHeader *FH; + codeview::StreamReader FISR(FileInfoSubstream); + if (auto EC = FISR.readObject(FH)) + return EC; + // The number of modules in the stream should be the same as reported by // the FileInfoSubstreamHeader. - if (FI->NumModules != ModuleInfos.size()) + if (FH->NumModules != ModuleInfos.size()) return make_error(raw_error_code::corrupt_file, "FileInfo substream count doesn't match DBI."); + codeview::FixedStreamArray ModIndexArray; + codeview::FixedStreamArray ModFileCountArray; + codeview::FixedStreamArray FileNameOffsets; + // First is an array of `NumModules` module indices. This is not used for the // same reason that `NumSourceFiles` is not used. It's an array of uint16's, // but it's possible there are more than 64k source files, which would imply // more than 64k modules (e.g. object files) as well. So we ignore this // field. - llvm::ArrayRef ModIndexArray( - reinterpret_cast(Buf), ModuleInfos.size()); - - llvm::ArrayRef ModFileCountArray(ModIndexArray.end(), - ModuleInfos.size()); + if (auto EC = FISR.readArray(ModIndexArray, ModuleInfos.size())) + return EC; + if (auto EC = FISR.readArray(ModFileCountArray, ModuleInfos.size())) + return EC; // Compute the real number of source files. uint32_t NumSourceFiles = 0; @@ -280,11 +285,13 @@ Error DbiStream::initializeFileInfo() { // them in `ModuleInfoEx`. The value written to and read from the file is // not used anyway, it is only there as a way to store the offsets for the // purposes of later accessing the names at runtime. - llvm::ArrayRef FileNameOffsets( - reinterpret_cast(ModFileCountArray.end()), - NumSourceFiles); + if (auto EC = FISR.readArray(FileNameOffsets, NumSourceFiles)) + return EC; - const char *Names = reinterpret_cast(FileNameOffsets.end()); + codeview::StreamRef NamesBufferRef; + if (auto EC = FISR.readStreamRef(NamesBufferRef)) + return EC; + codeview::StreamReader Names(NamesBufferRef); // We go through each ModuleInfo, determine the number N of source files for // that module, and then get the next N offsets from the Offsets array, using @@ -295,8 +302,10 @@ Error DbiStream::initializeFileInfo() { uint32_t NumFiles = ModFileCountArray[I]; ModuleInfos[I].SourceFiles.resize(NumFiles); for (size_t J = 0; J < NumFiles; ++J, ++NextFileIndex) { - uint32_t FileIndex = FileNameOffsets[NextFileIndex]; - ModuleInfos[I].SourceFiles[J] = StringRef(Names + FileIndex); + uint32_t FileOffset = FileNameOffsets[NextFileIndex]; + Names.setOffset(FileOffset); + if (auto EC = Names.readZeroString(ModuleInfos[I].SourceFiles[J])) + return EC; } } @@ -304,13 +313,5 @@ Error DbiStream::initializeFileInfo() { } uint32_t DbiStream::getDebugStreamIndex(DbgHeaderType Type) const { - ArrayRef DbgData; - if (auto EC = DbgHeader.getArrayRef(0, DbgData, DbgHeader.getLength())) { - consumeError(std::move(EC)); - return uint32_t(-1); - } - ArrayRef DebugStreams( - reinterpret_cast(DbgData.data()), - DbgData.size() / sizeof(ulittle16_t)); - return DebugStreams[static_cast(Type)]; + return DbgStreams[static_cast(Type)]; } diff --git a/lib/DebugInfo/PDB/Raw/InfoStream.cpp b/lib/DebugInfo/PDB/Raw/InfoStream.cpp index e7c8a831c73..64d8319efe1 100644 --- a/lib/DebugInfo/PDB/Raw/InfoStream.cpp +++ b/lib/DebugInfo/PDB/Raw/InfoStream.cpp @@ -29,19 +29,19 @@ Error InfoStream::reload() { PDB_UniqueId Guid; }; - Header H; - if (auto EC = Reader.readObject(&H)) + const Header *H; + if (auto EC = Reader.readObject(H)) return make_error(raw_error_code::corrupt_file, "PDB Stream does not contain a header."); - if (H.Version < PdbRaw_ImplVer::PdbImplVC70) + if (H->Version < PdbRaw_ImplVer::PdbImplVC70) return make_error(raw_error_code::corrupt_file, "Unsupported PDB stream version."); - Version = H.Version; - Signature = H.Signature; - Age = H.Age; - Guid = H.Guid; + Version = H->Version; + Signature = H->Signature; + Age = H->Age; + Guid = H->Guid; return NamedStreams.load(Reader); } diff --git a/lib/DebugInfo/PDB/Raw/MappedBlockStream.cpp b/lib/DebugInfo/PDB/Raw/MappedBlockStream.cpp index 09d71985031..a3db147f57e 100644 --- a/lib/DebugInfo/PDB/Raw/MappedBlockStream.cpp +++ b/lib/DebugInfo/PDB/Raw/MappedBlockStream.cpp @@ -23,6 +23,69 @@ MappedBlockStream::MappedBlockStream(uint32_t StreamIdx, const PDBFile &File) : } } +Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const { + // Make sure we aren't trying to read beyond the end of the stream. + if (Buffer.size() > StreamLength) + return make_error(raw_error_code::insufficient_buffer); + if (Offset > StreamLength - Buffer.size()) + return make_error(raw_error_code::insufficient_buffer); + + if (tryReadContiguously(Offset, Size, Buffer)) + return Error::success(); + + auto CacheIter = CacheMap.find(Offset); + if (CacheIter != CacheMap.end()) { + // In a more general solution, we would need to guarantee that the + // cached allocation is at least the requested size. In practice, since + // these are CodeView / PDB records, we know they are always formatted + // the same way and never change, so we should never be requesting two + // allocations from the same address with different sizes. + Buffer = ArrayRef(CacheIter->second, Size); + return Error::success(); + } + + // Otherwise allocate a large enough buffer in the pool, memcpy the data + // into it, and return an ArrayRef to that. + uint8_t *WriteBuffer = Pool.Allocate(Size); + + if (auto EC = readBytes(Offset, MutableArrayRef(WriteBuffer, Size))) + return EC; + CacheMap.insert(std::make_pair(Offset, WriteBuffer)); + Buffer = ArrayRef(WriteBuffer, Size); + return Error::success(); +} + +bool MappedBlockStream::tryReadContiguously(uint32_t Offset, uint32_t Size, + ArrayRef &Buffer) const { + // Attempt to fulfill the request with a reference directly into the stream. + // This can work even if the request crosses a block boundary, provided that + // all subsequent blocks are contiguous. For example, a 10k read with a 4k + // block size can be filled with a reference if, from the starting offset, + // 3 blocks in a row are contiguous. + uint32_t BlockNum = Offset / Pdb.getBlockSize(); + uint32_t OffsetInBlock = Offset % Pdb.getBlockSize(); + uint32_t BytesFromFirstBlock = + std::min(Size, Pdb.getBlockSize() - OffsetInBlock); + uint32_t NumAdditionalBlocks = + llvm::alignTo(Size - BytesFromFirstBlock, Pdb.getBlockSize()) / + Pdb.getBlockSize(); + + uint32_t RequiredContiguousBlocks = NumAdditionalBlocks + 1; + uint32_t E = BlockList[BlockNum]; + for (uint32_t I = 0; I < RequiredContiguousBlocks; ++I, ++E) { + if (BlockList[I + BlockNum] != E) + return false; + } + + uint32_t FirstBlockAddr = BlockList[BlockNum]; + StringRef Str = Pdb.getBlockData(FirstBlockAddr, Pdb.getBlockSize()); + Str = Str.drop_front(OffsetInBlock); + Buffer = + ArrayRef(reinterpret_cast(Str.data()), Size); + return true; +} + Error MappedBlockStream::readBytes(uint32_t Offset, MutableArrayRef Buffer) const { uint32_t BlockNum = Offset / Pdb.getBlockSize(); @@ -54,27 +117,5 @@ Error MappedBlockStream::readBytes(uint32_t Offset, } return Error::success(); -} - -Error MappedBlockStream::getArrayRef(uint32_t Offset, ArrayRef &Buffer, - uint32_t Length) const { - uint32_t BlockNum = Offset / Pdb.getBlockSize(); - uint32_t OffsetInBlock = Offset % Pdb.getBlockSize(); - uint32_t BytesAvailableInBlock = Pdb.getBlockSize() - OffsetInBlock; - - // If this is the last block in the stream, not all of the data is valid. - if (BlockNum == BlockList.size() - 1) { - uint32_t AllocatedBytesInBlock = StreamLength % Pdb.getBlockSize(); - if (AllocatedBytesInBlock < BytesAvailableInBlock) - BytesAvailableInBlock = AllocatedBytesInBlock; - } - if (BytesAvailableInBlock < Length) - return make_error(raw_error_code::feature_unsupported); - - uint32_t StreamBlockAddr = BlockList[BlockNum]; - StringRef Data = Pdb.getBlockData(StreamBlockAddr, Pdb.getBlockSize()); - Data = Data.substr(OffsetInBlock, Length); - - Buffer = ArrayRef(Data.bytes_begin(), Data.bytes_end()); - return Error::success(); + } diff --git a/lib/DebugInfo/PDB/Raw/ModInfo.cpp b/lib/DebugInfo/PDB/Raw/ModInfo.cpp index 362c402c09b..9ccb7edd696 100644 --- a/lib/DebugInfo/PDB/Raw/ModInfo.cpp +++ b/lib/DebugInfo/PDB/Raw/ModInfo.cpp @@ -8,6 +8,8 @@ //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/PDB/Raw/ModInfo.h" + +#include "llvm/DebugInfo/CodeView/StreamReader.h" #include "llvm/DebugInfo/PDB/Raw/PDBFile.h" #include "llvm/Support/Endian.h" @@ -16,6 +18,7 @@ using namespace llvm::pdb; using namespace llvm::support; namespace { + struct SCBytes { ulittle16_t Section; char Padding1[2]; @@ -60,17 +63,29 @@ struct ModInfo::FileLayout { // for now since it is unused. ulittle32_t SrcFileNameNI; // Name Index for src file name ulittle32_t PdbFilePathNI; // Name Index for path to compiler PDB - char VarInfo[1]; // Module name followed by Obj File Name - - StringRef getModuleName() const { return StringRef(VarInfo); } - - StringRef getObjectFileName() const { - return StringRef(getModuleName().end() + 1); - } + // Null terminated Module name + // Null terminated Obj File Name }; -ModInfo::ModInfo(const uint8_t *Bytes) - : Layout(reinterpret_cast(Bytes)) {} +ModInfo::ModInfo(codeview::StreamRef Stream) : Layout(nullptr) { + codeview::StreamReader Reader(Stream); + if (auto EC = Reader.readObject(Layout)) { + consumeError(std::move(EC)); + return; + } + if (auto EC = Reader.readZeroString(ModuleName)) { + consumeError(std::move(EC)); + return; + } + if (auto EC = Reader.readZeroString(ObjFileName)) { + consumeError(std::move(EC)); + return; + } +} + +ModInfo::ModInfo(const ModInfo &Info) + : ModuleName(Info.ModuleName), ObjFileName(Info.ObjFileName), + Layout(Info.Layout) {} ModInfo::~ModInfo() {} @@ -100,44 +115,14 @@ uint32_t ModInfo::getPdbFilePathNameIndex() const { return Layout->PdbFilePathNI; } -llvm::StringRef ModInfo::getModuleName() const { - return Layout->getModuleName(); -} - -llvm::StringRef ModInfo::getObjFileName() const { - return Layout->getObjectFileName(); -} - -ModInfoIterator::ModInfoIterator(const uint8_t *Stream) : Bytes(Stream) {} - -ModInfoIterator::ModInfoIterator(const ModInfoIterator &Other) - : Bytes(Other.Bytes) {} - -ModInfo ModInfoIterator::operator*() { return ModInfo(Bytes); } - -ModInfoIterator &ModInfoIterator::operator++() { - StringRef Obj = ModInfo(Bytes).getObjFileName(); - Bytes = Obj.bytes_end() + 1; - Bytes = reinterpret_cast(llvm::alignAddr(Bytes, 4)); - - return *this; -} - -ModInfoIterator ModInfoIterator::operator++(int) { - ModInfoIterator Copy(*this); - ++(*this); - return Copy; -} - -bool ModInfoIterator::operator==(const ModInfoIterator &Other) { - return Bytes == Other.Bytes; -} - -bool ModInfoIterator::operator!=(const ModInfoIterator &Other) { - return !(*this == Other); -} - -ModInfoIterator &ModInfoIterator::operator=(const ModInfoIterator &Other) { - Bytes = Other.Bytes; - return *this; +StringRef ModInfo::getModuleName() const { return ModuleName; } + +StringRef ModInfo::getObjFileName() const { return ObjFileName; } + +uint32_t ModInfo::getRecordLength() const { + uint32_t M = ModuleName.str().size() + 1; + uint32_t O = ObjFileName.str().size() + 1; + uint32_t Size = sizeof(FileLayout) + M + O; + Size = llvm::alignTo(Size, 4); + return Size; } diff --git a/lib/DebugInfo/PDB/Raw/ModStream.cpp b/lib/DebugInfo/PDB/Raw/ModStream.cpp index c359e7757d0..38d3f2f23e3 100644 --- a/lib/DebugInfo/PDB/Raw/ModStream.cpp +++ b/lib/DebugInfo/PDB/Raw/ModStream.cpp @@ -32,17 +32,17 @@ Error ModStream::reload() { return llvm::make_error(raw_error_code::corrupt_file, "Module has both C11 and C13 line info"); - if (auto EC = SymbolsSubstream.initialize(Reader, SymbolSize)) + if (auto EC = SymbolsSubstream.load(Reader, SymbolSize)) return EC; - if (auto EC = LinesSubstream.initialize(Reader, C11Size)) + if (auto EC = Reader.readStreamRef(LinesSubstream, C11Size)) return EC; - if (auto EC = C13LinesSubstream.initialize(Reader, C13Size)) + if (auto EC = Reader.readStreamRef(C13LinesSubstream, C13Size)) return EC; uint32_t GlobalRefsSize; if (auto EC = Reader.readInteger(GlobalRefsSize)) return EC; - if (auto EC = GlobalRefsSubstream.initialize(Reader, GlobalRefsSize)) + if (auto EC = Reader.readStreamRef(GlobalRefsSubstream, GlobalRefsSize)) return EC; if (Reader.bytesRemaining() > 0) return llvm::make_error(raw_error_code::corrupt_file, diff --git a/lib/DebugInfo/PDB/Raw/NameHashTable.cpp b/lib/DebugInfo/PDB/Raw/NameHashTable.cpp index a542a51b188..7eae7489a0e 100644 --- a/lib/DebugInfo/PDB/Raw/NameHashTable.cpp +++ b/lib/DebugInfo/PDB/Raw/NameHashTable.cpp @@ -84,28 +84,28 @@ Error NameHashTable::load(codeview::StreamReader &Stream) { support::ulittle32_t ByteSize; }; - Header H; - if (auto EC = Stream.readObject(&H)) + const Header *H; + if (auto EC = Stream.readObject(H)) return EC; - if (H.Signature != 0xEFFEEFFE) + if (H->Signature != 0xEFFEEFFE) return make_error(raw_error_code::corrupt_file, "Invalid hash table signature"); - if (H.HashVersion != 1 && H.HashVersion != 2) + if (H->HashVersion != 1 && H->HashVersion != 2) return make_error(raw_error_code::corrupt_file, "Unsupported hash version"); - Signature = H.Signature; - HashVersion = H.HashVersion; - if (auto EC = NamesBuffer.initialize(Stream, H.ByteSize)) + Signature = H->Signature; + HashVersion = H->HashVersion; + if (auto EC = Stream.readStreamRef(NamesBuffer, H->ByteSize)) return make_error(raw_error_code::corrupt_file, "Invalid hash table byte length"); - support::ulittle32_t HashCount; - if (auto EC = Stream.readObject(&HashCount)) + const support::ulittle32_t *HashCount; + if (auto EC = Stream.readObject(HashCount)) return EC; - std::vector BucketArray(HashCount); + std::vector BucketArray(*HashCount); if (auto EC = Stream.readArray(BucketArray)) return make_error(raw_error_code::corrupt_file, "Could not read bucket array"); @@ -124,7 +124,15 @@ StringRef NameHashTable::getStringForID(uint32_t ID) const { if (ID == IDs[0]) return StringRef(); - return StringRef(NamesBuffer.str().begin() + ID); + // NamesBuffer is a buffer of null terminated strings back to back. ID is + // the starting offset of the string we're looking for. So just seek into + // the desired offset and a read a null terminated stream from that offset. + StringRef Result; + codeview::StreamReader NameReader(NamesBuffer); + NameReader.setOffset(ID); + if (auto EC = NameReader.readZeroString(Result)) + consumeError(std::move(EC)); + return Result; } uint32_t NameHashTable::getIDForString(StringRef Str) const { diff --git a/lib/DebugInfo/PDB/Raw/NameMap.cpp b/lib/DebugInfo/PDB/Raw/NameMap.cpp index 777d93279c1..be4ce56ab15 100644 --- a/lib/DebugInfo/PDB/Raw/NameMap.cpp +++ b/lib/DebugInfo/PDB/Raw/NameMap.cpp @@ -113,7 +113,7 @@ Error NameMap::load(codeview::StreamReader &Stream) { uint32_t StringOffset = StringsOffset + NameOffset; uint32_t OldOffset = Stream.getOffset(); // Pump out our c-string from the stream. - std::string Str; + StringRef Str; Stream.setOffset(StringOffset); if (Stream.readZeroString(Str)) return make_error(raw_error_code::corrupt_file, diff --git a/lib/DebugInfo/PDB/Raw/PublicsStream.cpp b/lib/DebugInfo/PDB/Raw/PublicsStream.cpp index db66f536e5a..aba6a147afc 100644 --- a/lib/DebugInfo/PDB/Raw/PublicsStream.cpp +++ b/lib/DebugInfo/PDB/Raw/PublicsStream.cpp @@ -107,12 +107,11 @@ Error PublicsStream::reload() { "Publics Stream does not contain a header."); // Read PSGSIHDR and GSIHashHdr structs. - Header.reset(new HeaderInfo()); - if (Reader.readObject(Header.get())) + if (Reader.readObject(Header)) return make_error(raw_error_code::corrupt_file, "Publics Stream does not contain a header."); - HashHdr.reset(new GSIHashHeader()); - if (Reader.readObject(HashHdr.get())) + + if (Reader.readObject(HashHdr)) return make_error(raw_error_code::corrupt_file, "Publics Stream does not contain a header."); diff --git a/lib/DebugInfo/PDB/Raw/SymbolStream.cpp b/lib/DebugInfo/PDB/Raw/SymbolStream.cpp index 2037a646de7..ba4ea577d81 100644 --- a/lib/DebugInfo/PDB/Raw/SymbolStream.cpp +++ b/lib/DebugInfo/PDB/Raw/SymbolStream.cpp @@ -30,7 +30,7 @@ SymbolStream::~SymbolStream() {} Error SymbolStream::reload() { codeview::StreamReader Reader(MappedStream); - if (Stream.initialize(Reader, MappedStream.getLength())) + if (Stream.load(Reader, MappedStream.getLength())) return make_error(raw_error_code::corrupt_file, "Could not load symbol stream."); @@ -40,7 +40,7 @@ Error SymbolStream::reload() { iterator_range SymbolStream::getSymbols() const { using codeview::SymbolIterator; ArrayRef Data; - if (auto Error = Stream.getArrayRef(0, Data, Stream.getLength())) { + if (auto Error = Stream.readBytes(0, Stream.getLength(), Data)) { consumeError(std::move(Error)); return iterator_range(SymbolIterator(), SymbolIterator()); } diff --git a/lib/DebugInfo/PDB/Raw/TpiStream.cpp b/lib/DebugInfo/PDB/Raw/TpiStream.cpp index 99daf6e29fd..3345ab27571 100644 --- a/lib/DebugInfo/PDB/Raw/TpiStream.cpp +++ b/lib/DebugInfo/PDB/Raw/TpiStream.cpp @@ -68,8 +68,7 @@ Error TpiStream::reload() { return make_error(raw_error_code::corrupt_file, "TPI Stream does not contain a header."); - Header.reset(new HeaderInfo()); - if (Reader.readObject(Header.get())) + if (Reader.readObject(Header)) return make_error(raw_error_code::corrupt_file, "TPI Stream does not contain a header."); @@ -93,7 +92,7 @@ Error TpiStream::reload() { HashFunction = HashBufferV8; // The actual type records themselves come from this stream - if (auto EC = RecordsBuffer.initialize(Reader, Header->TypeRecordBytes)) + if (auto EC = RecordsBuffer.load(Reader, Header->TypeRecordBytes)) return EC; // Hash indices, hash values, etc come from the hash stream. @@ -101,16 +100,16 @@ Error TpiStream::reload() { codeview::StreamReader HSR(HS); HSR.setOffset(Header->HashValueBuffer.Off); if (auto EC = - HashValuesBuffer.initialize(HSR, Header->HashValueBuffer.Length)) + HSR.readStreamRef(HashValuesBuffer, Header->HashValueBuffer.Length)) return EC; HSR.setOffset(Header->HashAdjBuffer.Off); - if (auto EC = HashAdjBuffer.initialize(HSR, Header->HashAdjBuffer.Length)) + if (auto EC = HSR.readStreamRef(HashAdjBuffer, Header->HashAdjBuffer.Length)) return EC; HSR.setOffset(Header->IndexOffsetBuffer.Off); - if (auto EC = TypeIndexOffsetBuffer.initialize( - HSR, Header->IndexOffsetBuffer.Length)) + if (auto EC = HSR.readStreamRef(TypeIndexOffsetBuffer, + Header->IndexOffsetBuffer.Length)) return EC; return Error::success(); diff --git a/tools/llvm-pdbdump/llvm-pdbdump.cpp b/tools/llvm-pdbdump/llvm-pdbdump.cpp index b3f73cc323c..76b83cd1d00 100644 --- a/tools/llvm-pdbdump/llvm-pdbdump.cpp +++ b/tools/llvm-pdbdump/llvm-pdbdump.cpp @@ -305,7 +305,7 @@ static Error dumpStreamSummary(ScopedPrinter &P, PDBFile &File) { auto NSIter = NamedStreams.find(StreamIdx); if (ModIter != ModStreams.end()) { Value = "Module \""; - Value += ModIter->second->Info.getModuleName(); + Value += ModIter->second->Info.getModuleName().str(); Value += "\""; } else if (NSIter != NamedStreams.end()) { Value = "Named Stream \""; @@ -354,7 +354,7 @@ static Error dumpStreamData(ScopedPrinter &P, PDBFile &File) { ArrayRef Data; uint32_t BytesToReadInBlock = std::min( R.bytesRemaining(), static_cast(File.getBlockSize())); - if (auto EC = R.getArrayRef(Data, BytesToReadInBlock)) + if (auto EC = R.readBytes(BytesToReadInBlock, Data)) return EC; P.printBinaryBlock( "Data", @@ -455,9 +455,9 @@ static Error dumpDbiStream(ScopedPrinter &P, PDBFile &File, ListScope L(P, "Modules"); for (auto &Modi : DS.modules()) { DictScope DD(P); - P.printString("Name", Modi.Info.getModuleName()); + P.printString("Name", Modi.Info.getModuleName().str()); P.printNumber("Debug Stream Index", Modi.Info.getModuleStreamIndex()); - P.printString("Object File Name", Modi.Info.getObjFileName()); + P.printString("Object File Name", Modi.Info.getObjFileName().str()); P.printNumber("Num Files", Modi.Info.getNumberOfFiles()); P.printNumber("Source File Name Idx", Modi.Info.getSourceFileNameIndex()); P.printNumber("Pdb File Name Idx", Modi.Info.getPdbFilePathNameIndex()); @@ -472,7 +472,7 @@ static Error dumpDbiStream(ScopedPrinter &P, PDBFile &File, to_string(Modi.SourceFiles.size()) + " Contributing Source Files"; ListScope LL(P, FileListName); for (auto File : Modi.SourceFiles) - P.printString(File); + P.printString(File.str()); } bool HasModuleDI = (Modi.Info.getModuleStreamIndex() < File.getNumStreams());