diff --git a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h index 37293dfb8ed..72687682f60 100644 --- a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h +++ b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h @@ -81,7 +81,7 @@ using StoreFrameRangeFunction = /// Authors of JITLinkContexts can use this function to register a post-fixup /// pass that records the range of the eh-frame section. This range can /// be used after finalization to register and deregister the frame. -AtomGraphPassFunction +LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, StoreFrameRangeFunction StoreFrameRange); diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index be80d44ccf5..ea6c967f3a5 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -34,6 +34,9 @@ namespace llvm { namespace jitlink { +class Symbol; +class Section; + /// Base class for errors originating in JIT linker, e.g. missing relocation /// support. class JITLinkError : public ErrorInfo { @@ -50,27 +53,22 @@ private: std::string ErrMsg; }; -// Forward declare the Atom class. -class Atom; - -/// Edge class. Represents both object file relocations, as well as layout and -/// keep-alive constraints. +/// Represents fixups and constraints in the LinkGraph. class Edge { public: using Kind = uint8_t; - using GenericEdgeKind = enum : Kind { + enum GenericEdgeKind : Kind { Invalid, // Invalid edge value. FirstKeepAlive, // Keeps target alive. Offset/addend zero. KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness. - LayoutNext, // Layout constraint. Offset/Addend zero. FirstRelocation // First architecture specific relocation. }; using OffsetT = uint32_t; using AddendT = int64_t; - Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend) + Edge(Kind K, OffsetT Offset, Symbol &Target, AddendT Addend) : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} OffsetT getOffset() const { return Offset; } @@ -82,169 +80,516 @@ public: return K - FirstRelocation; } bool isKeepAlive() const { return K >= FirstKeepAlive; } - Atom &getTarget() const { return *Target; } - void setTarget(Atom &Target) { this->Target = &Target; } + Symbol &getTarget() const { return *Target; } + void setTarget(Symbol &Target) { this->Target = &Target; } AddendT getAddend() const { return Addend; } void setAddend(AddendT Addend) { this->Addend = Addend; } private: - Atom *Target; - OffsetT Offset; - AddendT Addend; + Symbol *Target = nullptr; + OffsetT Offset = 0; + AddendT Addend = 0; Kind K = 0; }; -using EdgeVector = std::vector; +/// Returns the string name of the given generic edge kind, or "unknown" +/// otherwise. Useful for debugging. +const char *getGenericEdgeKindName(Edge::Kind K); -const StringRef getGenericEdgeKindName(Edge::Kind K); - -/// Base Atom class. Used by absolute and undefined atoms. -class Atom { - friend class AtomGraph; +/// Base class for Addressable entities (externals, absolutes, blocks). +class Addressable { + friend class LinkGraph; protected: - /// Create a named (as yet unresolved) atom. - Atom(StringRef Name) - : Name(Name), IsDefined(false), IsLive(false), ShouldDiscard(false), - IsGlobal(false), IsAbsolute(false), IsCallable(false), - IsExported(false), IsWeak(false), HasLayoutNext(false), - IsCommon(false) {} + Addressable(JITTargetAddress Address, bool IsDefined) + : Address(Address), IsDefined(IsDefined), IsAbsolute(false) {} - /// Create an absolute symbol atom. - Atom(StringRef Name, JITTargetAddress Address) - : Name(Name), Address(Address), IsDefined(true), IsLive(false), - ShouldDiscard(false), IsGlobal(false), IsAbsolute(false), - IsCallable(false), IsExported(false), IsWeak(false), - HasLayoutNext(false), IsCommon(false) {} + Addressable(JITTargetAddress Address) + : Address(Address), IsDefined(false), IsAbsolute(true) { + assert(!(IsDefined && IsAbsolute) && + "Block cannot be both defined and absolute"); + } public: - /// Returns true if this atom has a name. - bool hasName() const { return Name != StringRef(); } + Addressable(const Addressable &) = delete; + Addressable &operator=(const Addressable &) = default; + Addressable(Addressable &&) = delete; + Addressable &operator=(Addressable &&) = default; - /// Returns the name of this atom. - StringRef getName() const { return Name; } - - /// Returns the current target address of this atom. - /// The initial target address (for atoms that have one) will be taken from - /// the input object file's virtual address space. During the layout phase - /// of JIT linking the atom's address will be updated to point to its final - /// address in the JIT'd process. JITTargetAddress getAddress() const { return Address; } - - /// Set the current target address of this atom. void setAddress(JITTargetAddress Address) { this->Address = Address; } - /// Returns true if this is a defined atom. - bool isDefined() const { return IsDefined; } - - /// Returns true if this atom is marked as live. - bool isLive() const { return IsLive; } - - /// Mark this atom as live. - /// - /// Note: Only defined and absolute atoms can be marked live. - void setLive(bool IsLive) { - assert((IsDefined || IsAbsolute || !IsLive) && - "Only defined and absolute atoms can be marked live"); - this->IsLive = IsLive; - } - - /// Returns true if this atom should be discarded during pruning. - bool shouldDiscard() const { return ShouldDiscard; } - - /// Mark this atom to be discarded. - /// - /// Note: Only defined and absolute atoms can be marked live. - void setShouldDiscard(bool ShouldDiscard) { - assert((IsDefined || IsAbsolute || !ShouldDiscard) && - "Only defined and absolute atoms can be marked live"); - this->ShouldDiscard = ShouldDiscard; - } - - /// Returns true if this definition is global (i.e. visible outside this - /// linkage unit). - /// - /// Note: This is distict from Exported, which means visibile outside the - /// JITDylib that this graph is being linked in to. - bool isGlobal() const { return IsGlobal; } - - /// Mark this atom as global. - void setGlobal(bool IsGlobal) { this->IsGlobal = IsGlobal; } - - /// Returns true if this atom represents an absolute symbol. - bool isAbsolute() const { return IsAbsolute; } - - /// Returns true if this atom is known to be callable. - /// - /// Primarily provided for easy interoperability with ORC, which uses the - /// JITSymbolFlags::Common flag to identify symbols that can be interposed - /// with stubs. - bool isCallable() const { return IsCallable; } - - /// Mark this atom as callable. - void setCallable(bool IsCallable) { - assert((IsDefined || IsAbsolute || !IsCallable) && - "Callable atoms must be defined or absolute"); - this->IsCallable = IsCallable; - } - - /// Returns true if this atom should appear in the symbol table of a final - /// linked image. - bool isExported() const { return IsExported; } - - /// Mark this atom as exported. - void setExported(bool IsExported) { - assert((!IsExported || ((IsDefined || IsAbsolute) && hasName())) && - "Exported atoms must have names"); - this->IsExported = IsExported; - } - - /// Returns true if this is a weak symbol. - bool isWeak() const { return IsWeak; } - - /// Mark this atom as weak. - void setWeak(bool IsWeak) { this->IsWeak = IsWeak; } + /// Returns true if this is a defined addressable, in which case you + /// can downcast this to a . + bool isDefined() const { return static_cast(IsDefined); } + bool isAbsolute() const { return static_cast(IsAbsolute); } private: - StringRef Name; JITTargetAddress Address = 0; - - bool IsDefined : 1; - bool IsLive : 1; - bool ShouldDiscard : 1; - - bool IsGlobal : 1; - bool IsAbsolute : 1; - bool IsCallable : 1; - bool IsExported : 1; - bool IsWeak : 1; - -protected: - // These flags only make sense for DefinedAtom, but we can minimize the size - // of DefinedAtom by defining them here. - bool HasLayoutNext : 1; - bool IsCommon : 1; + uint64_t IsDefined : 1; + uint64_t IsAbsolute : 1; }; -// Forward declare DefinedAtom. -class DefinedAtom; +using BlockOrdinal = unsigned; +using SectionOrdinal = unsigned; -raw_ostream &operator<<(raw_ostream &OS, const Atom &A); -void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, +/// An Addressable with content and edges. +class Block : public Addressable { + friend class LinkGraph; + +private: + /// Create a zero-fill defined addressable. + Block(Section &Parent, BlockOrdinal Ordinal, JITTargetAddress Size, + JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Size(Size), + Ordinal(Ordinal) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } + + /// Create a defined addressable for the given content. + Block(Section &Parent, BlockOrdinal Ordinal, StringRef Content, + JITTargetAddress Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(Parent), Data(Content.data()), + Size(Content.size()), Ordinal(Ordinal) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } + +public: + using EdgeVector = std::vector; + using edge_iterator = EdgeVector::iterator; + using const_edge_iterator = EdgeVector::const_iterator; + + Block(const Block &) = delete; + Block &operator=(const Block &) = delete; + Block(Block &&) = delete; + Block &operator=(Block &&) = delete; + + /// Return the parent section for this block. + Section &getSection() const { return Parent; } + + /// Return the ordinal for this block. + BlockOrdinal getOrdinal() const { return Ordinal; } + + /// Returns true if this is a zero-fill block. + /// + /// If true, getSize is callable but getContent is not (the content is + /// defined to be a sequence of zero bytes of length Size). + bool isZeroFill() const { return !Data; } + + /// Returns the size of this defined addressable. + size_t getSize() const { return Size; } + + /// Get the content for this block. Block must not be a zero-fill block. + StringRef getContent() const { + assert(Data && "Section does not contain content"); + return StringRef(Data, Size); + } + + /// Set the content for this block. + /// Caller is responsible for ensuring the underlying bytes are not + /// deallocated while pointed to by this block. + void setContent(StringRef Content) { + Data = Content.data(); + Size = Content.size(); + } + + /// Get the alignment for this content. + uint64_t getAlignment() const { return 1 << P2Align; } + + /// Get the alignment offset for this content. + uint64_t getAlignmentOffset() const { return AlignmentOffset; } + + /// Add an edge to this block. + void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target, + Edge::AddendT Addend) { + Edges.push_back(Edge(K, Offset, Target, Addend)); + } + + /// Return the list of edges attached to this content. + iterator_range edges() { + return make_range(Edges.begin(), Edges.end()); + } + + /// Returns the list of edges attached to this content. + iterator_range edges() const { + return make_range(Edges.begin(), Edges.end()); + } + + /// Return the size of the edges list. + size_t edges_size() const { return Edges.size(); } + + /// Returns true if the list of edges is empty. + bool edges_empty() const { return Edges.empty(); } + +private: + static constexpr uint64_t MaxAlignmentOffset = (1ULL << 57) - 1; + + uint64_t P2Align : 5; + uint64_t AlignmentOffset : 57; + Section &Parent; + const char *Data = nullptr; + size_t Size = 0; + BlockOrdinal Ordinal = 0; + std::vector Edges; +}; + +/// Describes symbol linkage. This can be used to make resolve definition +/// clashes. +enum class Linkage : uint8_t { + Strong, + Weak, +}; + +/// For errors and debugging output. +const char *getLinkageName(Linkage L); + +/// Defines the scope in which this symbol should be visible: +/// Default -- Visible in the public interface of the linkage unit. +/// Hidden -- Visible within the linkage unit, but not exported from it. +/// Local -- Visible only within the LinkGraph. +enum class Scope : uint8_t { Default, Hidden, Local }; + +/// For debugging output. +const char *getScopeName(Scope S); + +raw_ostream &operator<<(raw_ostream &OS, const Block &B); + +/// Symbol representation. +/// +/// Symbols represent locations within Addressable objects. +/// They can be either Named or Anonymous. +/// Anonymous symbols have neither linkage nor visibility, and must point at +/// ContentBlocks. +/// Named symbols may be in one of four states: +/// - Null: Default initialized. Assignable, but otherwise unusable. +/// - Defined: Has both linkage and visibility and points to a ContentBlock +/// - Common: Has both linkage and visibility, points to a null Addressable. +/// - External: Has neither linkage nor visibility, points to an external +/// Addressable. +/// +class Symbol { + friend class LinkGraph; + +private: + Symbol(Addressable &Base, JITTargetAddress Offset, StringRef Name, + JITTargetAddress Size, Linkage L, Scope S, bool IsLive, + bool IsCallable) + : Name(Name), Base(&Base), Offset(Offset), Size(Size) { + setLinkage(L); + setScope(S); + setLive(IsLive); + setCallable(IsCallable); + } + + static Symbol &constructCommon(void *SymStorage, Block &Base, StringRef Name, + JITTargetAddress Size, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Common symbol name cannot be empty"); + assert(Base.isDefined() && + "Cannot create common symbol from undefined block"); + assert(static_cast(Base).getSize() == Size && + "Common symbol size should match underlying block size"); + auto *Sym = reinterpret_cast(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Weak, S, IsLive, false); + return *Sym; + } + + static Symbol &constructExternal(void *SymStorage, Addressable &Base, + StringRef Name, JITTargetAddress Size) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create external symbol from defined block"); + assert(!Name.empty() && "External symbol name cannot be empty"); + auto *Sym = reinterpret_cast(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Strong, Scope::Default, + false, false); + return *Sym; + } + + static Symbol &constructAbsolute(void *SymStorage, Addressable &Base, + StringRef Name, JITTargetAddress Size, + Linkage L, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create absolute symbol from a defined block"); + auto *Sym = reinterpret_cast(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, L, S, IsLive, false); + return *Sym; + } + + static Symbol &constructAnonDef(void *SymStorage, Block &Base, + JITTargetAddress Offset, + JITTargetAddress Size, bool IsCallable, + bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + auto *Sym = reinterpret_cast(SymStorage); + new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong, + Scope::Local, IsLive, IsCallable); + return *Sym; + } + + static Symbol &constructNamedDef(void *SymStorage, Block &Base, + JITTargetAddress Offset, StringRef Name, + JITTargetAddress Size, Linkage L, Scope S, + bool IsLive, bool IsCallable) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Name cannot be empty"); + auto *Sym = reinterpret_cast(SymStorage); + new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable); + return *Sym; + } + +public: + /// Create a null Symbol. This allows Symbols to be default initialized for + /// use in containers (e.g. as map values). Null symbols are only useful for + /// assigning to. + Symbol() = default; + + // Symbols are not movable or copyable. + Symbol(const Symbol &) = delete; + Symbol &operator=(const Symbol &) = delete; + Symbol(Symbol &&) = delete; + Symbol &operator=(Symbol &&) = delete; + + /// Returns true if this symbol has a name. + bool hasName() const { return !Name.empty(); } + + /// Returns the name of this symbol (empty if the symbol is anonymous). + StringRef getName() const { + assert((!Name.empty() || getScope() == Scope::Local) && + "Anonymous symbol has non-local scope"); + return Name; + } + + /// Returns true if this Symbol has content (potentially) defined within this + /// object file (i.e. is anything but an external or absolute symbol). + bool isDefined() const { + assert(Base && "Attempt to access null symbol"); + return Base->isDefined(); + } + + /// Returns true if this symbol is live (i.e. should be treated as a root for + /// dead stripping). + bool isLive() const { + assert(Base && "Attempting to access null symbol"); + return IsLive; + } + + /// Set this symbol's live bit. + void setLive(bool IsLive) { this->IsLive = IsLive; } + + /// Returns true is this symbol is callable. + bool isCallable() const { return IsCallable; } + + /// Set this symbol's callable bit. + void setCallable(bool IsCallable) { this->IsCallable = IsCallable; } + + /// Returns true if the underlying addressable is an unresolved external. + bool isExternal() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && !Base->isAbsolute(); + } + + /// Returns true if the underlying addressable is an absolute symbol. + bool isAbsolute() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && Base->isAbsolute(); + } + + /// Return the addressable that this symbol points to. + Addressable &getAddressable() { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } + + /// Return the addressable that thsi symbol points to. + const Addressable &getAddressable() const { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } + + /// Return the Block for this Symbol (Symbol must be defined). + Block &getBlock() { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast(*Base); + } + + /// Return the Block for this Symbol (Symbol must be defined). + const Block &getBlock() const { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast(*Base); + } + + /// Returns the offset for this symbol within the underlying addressable. + JITTargetAddress getOffset() const { return Offset; } + + /// Returns the address of this symbol. + JITTargetAddress getAddress() const { return Base->getAddress() + Offset; } + + /// Returns the size of this symbol. + JITTargetAddress getSize() const { return Size; } + + /// Returns true if this symbol is backed by a zero-fill block. + /// This method may only be called on defined symbols. + bool isSymbolZeroFill() const { return getBlock().isZeroFill(); } + + /// Returns the content in the underlying block covered by this symbol. + /// This method may only be called on defined non-zero-fill symbols. + StringRef getSymbolContent() const { + return getBlock().getContent().substr(Offset, Size); + } + + /// Get the linkage for this Symbol. + Linkage getLinkage() const { return static_cast(L); } + + /// Set the linkage for this Symbol. + void setLinkage(Linkage L) { + assert((L == Linkage::Strong || (Base->isDefined() && !Name.empty())) && + "Linkage can only be applied to defined named symbols"); + this->L = static_cast(L); + } + + /// Get the visibility for this Symbol. + Scope getScope() const { return static_cast(S); } + + /// Set the visibility for this Symbol. + void setScope(Scope S) { + assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && + "Invalid visibility for symbol type"); + this->S = static_cast(S); + } + +private: + void makeExternal(Addressable &A) { + assert(!A.isDefined() && "Attempting to make external with defined block"); + Base = &A; + Offset = 0; + setLinkage(Linkage::Strong); + setScope(Scope::Default); + IsLive = 0; + // note: Size and IsCallable fields left unchanged. + } + + static constexpr uint64_t MaxOffset = (1ULL << 59) - 1; + + // FIXME: A char* or SymbolStringPtr may pack better. + StringRef Name; + Addressable *Base = nullptr; + uint64_t Offset : 59; + uint64_t L : 1; + uint64_t S : 2; + uint64_t IsLive : 1; + uint64_t IsCallable : 1; + JITTargetAddress Size = 0; +}; + +raw_ostream &operator<<(raw_ostream &OS, const Symbol &A); + +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, StringRef EdgeKindName); -/// Represents a section address range via a pair of DefinedAtom pointers to -/// the first and last atoms in the section. +/// Represents an object file section. +class Section { + friend class LinkGraph; + +private: + Section(StringRef Name, sys::Memory::ProtectionFlags Prot, + SectionOrdinal SecOrdinal) + : Name(Name), Prot(Prot), SecOrdinal(SecOrdinal) {} + + using SymbolSet = DenseSet; + using BlockSet = DenseSet; + +public: + using symbol_iterator = SymbolSet::iterator; + using const_symbol_iterator = SymbolSet::const_iterator; + + using block_iterator = BlockSet::iterator; + using const_block_iterator = BlockSet::const_iterator; + + ~Section(); + + /// Returns the name of this section. + StringRef getName() const { return Name; } + + /// Returns the protection flags for this section. + sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } + + /// Returns the ordinal for this section. + SectionOrdinal getOrdinal() const { return SecOrdinal; } + + /// Returns an iterator over the symbols defined in this section. + iterator_range symbols() { + return make_range(Symbols.begin(), Symbols.end()); + } + + /// Returns an iterator over the symbols defined in this section. + iterator_range symbols() const { + return make_range(Symbols.begin(), Symbols.end()); + } + + /// Return the number of symbols in this section. + SymbolSet::size_type symbols_size() { return Symbols.size(); } + + /// Return true if this section contains no symbols. + bool symbols_empty() const { return Symbols.empty(); } + + /// Returns the ordinal for the next block. + BlockOrdinal getNextBlockOrdinal() { return NextBlockOrdinal++; } + +private: + void addSymbol(Symbol &Sym) { + assert(!Symbols.count(&Sym) && "Symbol is already in this section"); + Symbols.insert(&Sym); + } + + void removeSymbol(Symbol &Sym) { + assert(Symbols.count(&Sym) && "symbol is not in this section"); + Symbols.erase(&Sym); + } + + StringRef Name; + sys::Memory::ProtectionFlags Prot; + SectionOrdinal SecOrdinal = 0; + BlockOrdinal NextBlockOrdinal = 0; + SymbolSet Symbols; +}; + +/// Represents a section address range via a pair of Block pointers +/// to the first and last Blocks in the section. class SectionRange { public: SectionRange() = default; - SectionRange(DefinedAtom *First, DefinedAtom *Last) - : First(First), Last(Last) {} - DefinedAtom *getFirstAtom() const { + SectionRange(const Section &Sec) { + if (Sec.symbols_empty()) + return; + First = Last = *Sec.symbols().begin(); + for (auto *Sym : Sec.symbols()) { + if (Sym->getAddress() < First->getAddress()) + First = Sym; + if (Sym->getAddress() > Last->getAddress()) + Last = Sym; + } + } + Symbol *getFirstSymbol() const { assert((!Last || First) && "First can not be null if end is non-null"); return First; } - DefinedAtom *getLastAtom() const { + Symbol *getLastSymbol() const { assert((First || !Last) && "Last can not be null if start is non-null"); return Last; } @@ -252,287 +597,112 @@ public: assert((First || !Last) && "Last can not be null if start is non-null"); return !First; } - JITTargetAddress getStart() const; - JITTargetAddress getEnd() const; - uint64_t getSize() const; + JITTargetAddress getStart() const { + return First ? First->getBlock().getAddress() : 0; + } + JITTargetAddress getEnd() const { + return Last ? Last->getBlock().getAddress() + Last->getBlock().getSize() + : 0; + } + uint64_t getSize() const { return getEnd() - getStart(); } private: - DefinedAtom *First = nullptr; - DefinedAtom *Last = nullptr; + Symbol *First = nullptr; + Symbol *Last = nullptr; }; -/// Represents an object file section. -class Section { - friend class AtomGraph; - -private: - Section(StringRef Name, uint32_t Alignment, sys::Memory::ProtectionFlags Prot, - unsigned Ordinal, bool IsZeroFill) - : Name(Name), Alignment(Alignment), Prot(Prot), Ordinal(Ordinal), - IsZeroFill(IsZeroFill) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of 2"); - } - - using DefinedAtomSet = DenseSet; - -public: - using atom_iterator = DefinedAtomSet::iterator; - using const_atom_iterator = DefinedAtomSet::const_iterator; - - ~Section(); - StringRef getName() const { return Name; } - uint32_t getAlignment() const { return Alignment; } - sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } - unsigned getSectionOrdinal() const { return Ordinal; } - size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; } - - bool isZeroFill() const { return IsZeroFill; } - - /// Returns an iterator over the atoms in the section (in no particular - /// order). - iterator_range atoms() { - return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); - } - - /// Returns an iterator over the atoms in the section (in no particular - /// order). - iterator_range atoms() const { - return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); - } - - /// Return the number of atoms in this section. - DefinedAtomSet::size_type atoms_size() { return DefinedAtoms.size(); } - - /// Return true if this section contains no atoms. - bool atoms_empty() const { return DefinedAtoms.empty(); } - - /// Returns the range of this section as the pair of atoms with the lowest - /// and highest target address. This operation is expensive, as it - /// must traverse all atoms in the section. - /// - /// Note: If the section is empty, both values will be null. The section - /// address will evaluate to null, and the size to zero. If the section - /// contains a single atom both values will point to it, the address will - /// evaluate to the address of that atom, and the size will be the size of - /// that atom. - SectionRange getRange() const; - -private: - void addAtom(DefinedAtom &DA) { - assert(!DefinedAtoms.count(&DA) && "Atom is already in this section"); - DefinedAtoms.insert(&DA); - } - - void removeAtom(DefinedAtom &DA) { - assert(DefinedAtoms.count(&DA) && "Atom is not in this section"); - DefinedAtoms.erase(&DA); - } - - StringRef Name; - uint32_t Alignment = 0; - sys::Memory::ProtectionFlags Prot; - unsigned Ordinal = 0; - unsigned NextAtomOrdinal = 0; - bool IsZeroFill = false; - DefinedAtomSet DefinedAtoms; -}; - -/// Defined atom class. Suitable for use by defined named and anonymous -/// atoms. -class DefinedAtom : public Atom { - friend class AtomGraph; - -private: - DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment) - : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()), - Alignment(Alignment) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); - } - - DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address, - uint32_t Alignment) - : Atom(Name, Address), Parent(Parent), - Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) { - assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); - } - -public: - using edge_iterator = EdgeVector::iterator; - - Section &getSection() const { return Parent; } - - uint64_t getSize() const { return Size; } - - StringRef getContent() const { - assert(!Parent.isZeroFill() && "Trying to get content for zero-fill atom"); - assert(Size <= std::numeric_limits::max() && - "Content size too large"); - return {ContentPtr, static_cast(Size)}; - } - void setContent(StringRef Content) { - assert(!Parent.isZeroFill() && "Calling setContent on zero-fill atom?"); - ContentPtr = Content.data(); - Size = Content.size(); - } - - bool isZeroFill() const { return Parent.isZeroFill(); } - - void setZeroFill(uint64_t Size) { - assert(Parent.isZeroFill() && !ContentPtr && - "Can't set zero-fill length of a non zero-fill atom"); - this->Size = Size; - } - - uint64_t getZeroFillSize() const { - assert(Parent.isZeroFill() && - "Can't get zero-fill length of a non zero-fill atom"); - return Size; - } - - uint32_t getAlignment() const { return Alignment; } - - bool hasLayoutNext() const { return HasLayoutNext; } - void setLayoutNext(DefinedAtom &Next) { - assert(!HasLayoutNext && "Atom already has layout-next constraint"); - HasLayoutNext = true; - Edges.push_back(Edge(Edge::LayoutNext, 0, Next, 0)); - } - DefinedAtom &getLayoutNext() { - assert(HasLayoutNext && "Atom does not have a layout-next constraint"); - DefinedAtom *Next = nullptr; - for (auto &E : edges()) - if (E.getKind() == Edge::LayoutNext) { - assert(E.getTarget().isDefined() && - "layout-next target atom must be a defined atom"); - Next = static_cast(&E.getTarget()); - break; - } - assert(Next && "Missing LayoutNext edge"); - return *Next; - } - - bool isCommon() const { return IsCommon; } - - void addEdge(Edge::Kind K, Edge::OffsetT Offset, Atom &Target, - Edge::AddendT Addend) { - assert(K != Edge::LayoutNext && - "Layout edges should be added via setLayoutNext"); - Edges.push_back(Edge(K, Offset, Target, Addend)); - } - - iterator_range edges() { - return make_range(Edges.begin(), Edges.end()); - } - size_t edges_size() const { return Edges.size(); } - bool edges_empty() const { return Edges.empty(); } - - unsigned getOrdinal() const { return Ordinal; } - -private: - void setCommon(uint64_t Size) { - assert(ContentPtr == 0 && "Atom already has content?"); - IsCommon = true; - setZeroFill(Size); - } - - EdgeVector Edges; - uint64_t Size = 0; - Section &Parent; - const char *ContentPtr = nullptr; - unsigned Ordinal = 0; - uint32_t Alignment = 0; -}; - -inline JITTargetAddress SectionRange::getStart() const { - return First ? First->getAddress() : 0; -} - -inline JITTargetAddress SectionRange::getEnd() const { - return Last ? Last->getAddress() + Last->getSize() : 0; -} - -inline uint64_t SectionRange::getSize() const { return getEnd() - getStart(); } - -inline SectionRange Section::getRange() const { - if (atoms_empty()) - return SectionRange(); - DefinedAtom *First = *DefinedAtoms.begin(), *Last = *DefinedAtoms.begin(); - for (auto *DA : atoms()) { - if (DA->getAddress() < First->getAddress()) - First = DA; - if (DA->getAddress() > Last->getAddress()) - Last = DA; - } - return SectionRange(First, Last); -} - -class AtomGraph { +class LinkGraph { private: using SectionList = std::vector>; - using AddressToAtomMap = std::map; - using NamedAtomMap = DenseMap; - using ExternalAtomSet = DenseSet; + using ExternalSymbolSet = DenseSet; + using BlockSet = DenseSet; + + template + Addressable &createAddressable(ArgTs &&... Args) { + Addressable *A = + reinterpret_cast(Allocator.Allocate()); + new (A) Addressable(std::forward(Args)...); + return *A; + } + + void destroyAddressable(Addressable &A) { + A.~Addressable(); + Allocator.Deallocate(&A); + } + + template Block &createBlock(ArgTs &&... Args) { + Block *B = reinterpret_cast(Allocator.Allocate()); + new (B) Block(std::forward(Args)...); + return *B; + } + + void destroyBlock(Block &B) { + B.~Block(); + Allocator.Deallocate(&B); + } + + void destroySymbol(Symbol &S) { + S.~Symbol(); + Allocator.Deallocate(&S); + } public: - using external_atom_iterator = ExternalAtomSet::iterator; + using external_symbol_iterator = ExternalSymbolSet::iterator; + + using block_iterator = BlockSet::iterator; using section_iterator = pointee_iterator; using const_section_iterator = pointee_iterator; - template - class defined_atom_iterator_impl + template + class defined_symbol_iterator_impl : public iterator_facade_base< - defined_atom_iterator_impl, + defined_symbol_iterator_impl, std::forward_iterator_tag, T> { public: - defined_atom_iterator_impl() = default; + defined_symbol_iterator_impl() = default; - defined_atom_iterator_impl(SecItrT SI, SecItrT SE) - : SI(SI), SE(SE), - AI(SI != SE ? SI->atoms().begin() : Section::atom_iterator()) { - moveToNextAtomOrEnd(); + defined_symbol_iterator_impl(SectionItrT SecI, SectionItrT SecE) + : SecI(SecI), SecE(SecE), + SymI(SecI != SecE ? SecI->symbols().begin() : SymbolItrT()) { + moveToNextSymbolOrEnd(); } - bool operator==(const defined_atom_iterator_impl &RHS) const { - return (SI == RHS.SI) && (AI == RHS.AI); + bool operator==(const defined_symbol_iterator_impl &RHS) const { + return (SecI == RHS.SecI) && (SymI == RHS.SymI); } T operator*() const { - assert(AI != SI->atoms().end() && "Dereferencing end?"); - return *AI; + assert(SymI != SecI->symbols().end() && "Dereferencing end?"); + return *SymI; } - defined_atom_iterator_impl operator++() { - ++AI; - moveToNextAtomOrEnd(); + defined_symbol_iterator_impl operator++() { + ++SymI; + moveToNextSymbolOrEnd(); return *this; } private: - void moveToNextAtomOrEnd() { - while (SI != SE && AI == SI->atoms().end()) { - ++SI; - if (SI == SE) - AI = Section::atom_iterator(); - else - AI = SI->atoms().begin(); + void moveToNextSymbolOrEnd() { + while (SecI != SecE && SymI == SecI->symbols().end()) { + ++SecI; + SymI = SecI == SecE ? SymbolItrT() : SecI->symbols().begin(); } } - SecItrT SI, SE; - AtomItrT AI; + SectionItrT SecI, SecE; + SymbolItrT SymI; }; - using defined_atom_iterator = - defined_atom_iterator_impl; + using defined_symbol_iterator = + defined_symbol_iterator_impl; - using const_defined_atom_iterator = - defined_atom_iterator_impl; + using const_defined_symbol_iterator = defined_symbol_iterator_impl< + const_section_iterator, Section::const_symbol_iterator, const Symbol *>; - AtomGraph(std::string Name, unsigned PointerSize, + LinkGraph(std::string Name, unsigned PointerSize, support::endianness Endianness) : Name(std::move(Name)), PointerSize(PointerSize), Endianness(Endianness) {} @@ -544,84 +714,87 @@ public: /// Returns the pointer size for use in this graph. unsigned getPointerSize() const { return PointerSize; } - /// Returns the endianness of atom-content in this graph. + /// Returns the endianness of content in this graph. support::endianness getEndianness() const { return Endianness; } /// Create a section with the given name, protection flags, and alignment. - Section &createSection(StringRef Name, uint32_t Alignment, - sys::Memory::ProtectionFlags Prot, bool IsZeroFill) { - std::unique_ptr
Sec( - new Section(Name, Alignment, Prot, Sections.size(), IsZeroFill)); + Section &createSection(StringRef Name, sys::Memory::ProtectionFlags Prot) { + std::unique_ptr
Sec(new Section(Name, Prot, Sections.size())); Sections.push_back(std::move(Sec)); return *Sections.back(); } - /// Add an external atom representing an undefined symbol in this graph. - Atom &addExternalAtom(StringRef Name) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - Atom *A = reinterpret_cast( - AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); - new (A) Atom(Name); - ExternalAtoms.insert(A); - NamedAtoms[Name] = A; - return *A; + /// Create a content block. + Block &createContentBlock(Section &Parent, StringRef Content, + uint64_t Address, uint64_t Alignment, + uint64_t AlignmentOffset) { + auto &B = createBlock(Parent, Parent.getNextBlockOrdinal(), Content, + Address, Alignment, AlignmentOffset); + Blocks.insert(&B); + return B; } - /// Add an external atom representing an absolute symbol. - Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - Atom *A = reinterpret_cast( - AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); - new (A) Atom(Name, Addr); - AbsoluteAtoms.insert(A); - NamedAtoms[Name] = A; - return *A; + /// Create a zero-fill block. + Block &createZeroFillBlock(Section &Parent, uint64_t Size, uint64_t Address, + uint64_t Alignment, uint64_t AlignmentOffset) { + auto &B = createBlock(Parent, Parent.getNextBlockOrdinal(), Size, Address, + Alignment, AlignmentOffset); + Blocks.insert(&B); + return B; } - /// Add an anonymous defined atom to the graph. - /// - /// Anonymous atoms have content but no name. They must have an address. - DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address, - uint32_t Alignment) { - DefinedAtom *A = reinterpret_cast( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Address, Alignment); - Parent.addAtom(*A); - getAddrToAtomMap()[A->getAddress()] = A; - return *A; + /// Add an external symbol. + /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose + /// size is not known, you should substitute '0'. + Symbol &addExternalSymbol(StringRef Name, uint64_t Size) { + auto &Sym = Symbol::constructExternal( + Allocator.Allocate(), createAddressable(0, false), Name, Size); + ExternalSymbols.insert(&Sym); + return Sym; } - /// Add a defined atom to the graph. - /// - /// Allocates and constructs a DefinedAtom instance with the given parent, - /// name, address, and alignment. - DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name, - JITTargetAddress Address, uint32_t Alignment) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - DefinedAtom *A = reinterpret_cast( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Name, Address, Alignment); - Parent.addAtom(*A); - getAddrToAtomMap()[A->getAddress()] = A; - NamedAtoms[Name] = A; - return *A; + /// Add an absolute symbol. + Symbol &addAbsoluteSymbol(StringRef Name, JITTargetAddress Address, + uint64_t Size, Linkage L, Scope S, bool IsLive) { + auto &Sym = Symbol::constructAbsolute(Allocator.Allocate(), + createAddressable(Address), Name, + Size, L, S, IsLive); + AbsoluteSymbols.insert(&Sym); + return Sym; } - /// Add a common symbol atom to the graph. - /// - /// Adds a common-symbol atom to the graph with the given parent, name, - /// address, alignment and size. - DefinedAtom &addCommonAtom(Section &Parent, StringRef Name, - JITTargetAddress Address, uint32_t Alignment, - uint64_t Size) { - assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); - DefinedAtom *A = reinterpret_cast( - AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); - new (A) DefinedAtom(Parent, Name, Address, Alignment); - A->setCommon(Size); - Parent.addAtom(*A); - NamedAtoms[Name] = A; - return *A; + /// Convenience method for adding a weak zero-fill symbol. + Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section, + JITTargetAddress Address, uint64_t Size, + uint64_t Alignment, bool IsLive) { + auto &Sym = Symbol::constructCommon( + Allocator.Allocate(), + createBlock(Section, Section.getNextBlockOrdinal(), Address, Size, + Alignment, 0), + Name, Size, S, IsLive); + Section.addSymbol(Sym); + return Sym; + } + + /// Add an anonymous symbol. + Symbol &addAnonymousSymbol(Block &Content, JITTargetAddress Offset, + JITTargetAddress Size, bool IsCallable, + bool IsLive) { + auto &Sym = Symbol::constructAnonDef(Allocator.Allocate(), Content, + Offset, Size, IsCallable, IsLive); + Content.getSection().addSymbol(Sym); + return Sym; + } + + /// Add a named symbol. + Symbol &addDefinedSymbol(Block &Content, JITTargetAddress Offset, + StringRef Name, JITTargetAddress Size, Linkage L, + Scope S, bool IsCallable, bool IsLive) { + auto &Sym = + Symbol::constructNamedDef(Allocator.Allocate(), Content, Offset, + Name, Size, L, S, IsLive, IsCallable); + Content.getSection().addSymbol(Sym); + return Sym; } iterator_range sections() { @@ -638,135 +811,79 @@ public: return nullptr; } - iterator_range external_atoms() { - return make_range(ExternalAtoms.begin(), ExternalAtoms.end()); + iterator_range external_symbols() { + return make_range(ExternalSymbols.begin(), ExternalSymbols.end()); } - iterator_range absolute_atoms() { - return make_range(AbsoluteAtoms.begin(), AbsoluteAtoms.end()); + iterator_range absolute_symbols() { + return make_range(AbsoluteSymbols.begin(), AbsoluteSymbols.end()); } - iterator_range defined_atoms() { - return make_range(defined_atom_iterator(Sections.begin(), Sections.end()), - defined_atom_iterator(Sections.end(), Sections.end())); + iterator_range defined_symbols() { + return make_range(defined_symbol_iterator(Sections.begin(), Sections.end()), + defined_symbol_iterator(Sections.end(), Sections.end())); } - iterator_range defined_atoms() const { + iterator_range defined_symbols() const { return make_range( - const_defined_atom_iterator(Sections.begin(), Sections.end()), - const_defined_atom_iterator(Sections.end(), Sections.end())); + const_defined_symbol_iterator(Sections.begin(), Sections.end()), + const_defined_symbol_iterator(Sections.end(), Sections.end())); } - /// Returns the atom with the given name, which must exist in this graph. - Atom &getAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - assert(I != NamedAtoms.end() && "Name not in NamedAtoms map"); - return *I->second; + iterator_range blocks() { + return make_range(Blocks.begin(), Blocks.end()); } - /// Returns the atom with the given name, which must exist in this graph and - /// be a DefinedAtom. - DefinedAtom &getDefinedAtomByName(StringRef Name) { - auto &A = getAtomByName(Name); - assert(A.isDefined() && "Atom is not a defined atom"); - return static_cast(A); - } - - /// Search for the given atom by name. - /// Returns the atom (if found) or an error (if no atom with this name - /// exists). - Expected findAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - if (I == NamedAtoms.end()) - return make_error("No atom named " + Name); - return *I->second; - } - - /// Search for the given defined atom by name. - /// Returns the defined atom (if found) or an error (if no atom with this - /// name exists, or if one exists but is not a defined atom). - Expected findDefinedAtomByName(StringRef Name) { - auto I = NamedAtoms.find(Name); - if (I == NamedAtoms.end()) - return make_error("No atom named " + Name); - if (!I->second->isDefined()) - return make_error("Atom " + Name + - " exists but is not a " - "defined atom"); - return static_cast(*I->second); - } - - /// Returns the atom covering the given address, or an error if no such atom - /// exists. - /// - /// Returns null if no atom exists at the given address. - DefinedAtom *getAtomByAddress(JITTargetAddress Address) { - refreshAddrToAtomCache(); - - // If there are no defined atoms, bail out early. - if (AddrToAtomCache->empty()) - return nullptr; - - // Find the atom *after* the given address. - auto I = AddrToAtomCache->upper_bound(Address); - - // If this address falls before any known atom, bail out. - if (I == AddrToAtomCache->begin()) - return nullptr; - - // The atom we're looking for is the one before the atom we found. - --I; - - // Otherwise range check the atom that was found. - assert(!I->second->getContent().empty() && "Atom content not set"); - if (Address >= I->second->getAddress() + I->second->getContent().size()) - return nullptr; - - return I->second; - } - - /// Like getAtomByAddress, but returns an Error if the given address is not - /// covered by an atom, rather than a null pointer. - Expected findAtomByAddress(JITTargetAddress Address) { - if (auto *DA = getAtomByAddress(Address)) - return *DA; - return make_error("No atom at address " + - formatv("{0:x16}", Address)); - } - - // Remove the given external atom from the graph. - void removeExternalAtom(Atom &A) { - assert(!A.isDefined() && !A.isAbsolute() && "A is not an external atom"); - assert(ExternalAtoms.count(&A) && "A is not in the external atoms set"); - ExternalAtoms.erase(&A); - A.~Atom(); - } - - /// Remove the given absolute atom from the graph. - void removeAbsoluteAtom(Atom &A) { - assert(A.isAbsolute() && "A is not an absolute atom"); - assert(AbsoluteAtoms.count(&A) && "A is not in the absolute atoms set"); - AbsoluteAtoms.erase(&A); - A.~Atom(); - } - - /// Remove the given defined atom from the graph. - void removeDefinedAtom(DefinedAtom &DA) { - if (AddrToAtomCache) { - assert(AddrToAtomCache->count(DA.getAddress()) && - "Cache exists, but does not contain atom"); - AddrToAtomCache->erase(DA.getAddress()); + /// Turn a defined symbol into an external one. + void makeExternal(Symbol &Sym) { + if (Sym.getAddressable().isAbsolute()) { + assert(AbsoluteSymbols.count(&Sym) && + "Sym is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + } else { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Section &Sec = Sym.getBlock().getSection(); + Sec.removeSymbol(Sym); } - if (DA.hasName()) { - assert(NamedAtoms.count(DA.getName()) && "Named atom not in map"); - NamedAtoms.erase(DA.getName()); - } - DA.getSection().removeAtom(DA); - DA.~DefinedAtom(); + Sym.makeExternal(createAddressable(false)); + ExternalSymbols.insert(&Sym); } - /// Invalidate the atom-to-address map. - void invalidateAddrToAtomMap() { AddrToAtomCache = None; } + /// Removes an external symbol. Also removes the underlying Addressable. + void removeExternalSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && !Sym.isAbsolute() && + "Sym is not an external symbol"); + assert(ExternalSymbols.count(&Sym) && "Symbol is not in the externals set"); + ExternalSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + destroySymbol(Sym); + destroyAddressable(Base); + } + + /// Remove an absolute symbol. Also removes the underlying Addressable. + void removeAbsoluteSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && Sym.isAbsolute() && + "Sym is not an absolute symbol"); + assert(AbsoluteSymbols.count(&Sym) && + "Symbol is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + destroySymbol(Sym); + destroyAddressable(Base); + } + + /// Removes defined symbols. Does not remove the underlying block. + void removeDefinedSymbol(Symbol &Sym) { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Sym.getBlock().getSection().removeSymbol(Sym); + destroySymbol(Sym); + } + + /// Remove a block. + void removeBlock(Block &B) { + Blocks.erase(&B); + destroyBlock(B); + } /// Dump the graph. /// @@ -778,87 +895,84 @@ public: std::function()); private: - AddressToAtomMap &getAddrToAtomMap() { - refreshAddrToAtomCache(); - return *AddrToAtomCache; - } - - const AddressToAtomMap &getAddrToAtomMap() const { - refreshAddrToAtomCache(); - return *AddrToAtomCache; - } - - void refreshAddrToAtomCache() const { - if (!AddrToAtomCache) { - AddrToAtomCache = AddressToAtomMap(); - for (auto *DA : defined_atoms()) - (*AddrToAtomCache)[DA->getAddress()] = const_cast(DA); - } - } - - // Put the BumpPtrAllocator first so that we don't free any of the atoms in - // it until all of their destructors have been run. - BumpPtrAllocator AtomAllocator; + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; std::string Name; unsigned PointerSize; support::endianness Endianness; + BlockSet Blocks; SectionList Sections; - NamedAtomMap NamedAtoms; - ExternalAtomSet ExternalAtoms; - ExternalAtomSet AbsoluteAtoms; - mutable Optional AddrToAtomCache; + ExternalSymbolSet ExternalSymbols; + ExternalSymbolSet AbsoluteSymbols; }; -/// A function for mutating AtomGraphs. -using AtomGraphPassFunction = std::function; +/// A function for mutating LinkGraphs. +using LinkGraphPassFunction = std::function; -/// A list of atom graph passes. -using AtomGraphPassList = std::vector; +/// A list of LinkGraph passes. +using LinkGraphPassList = std::vector; -/// An atom graph pass configuration, consisting of a list of pre-prune, +/// An LinkGraph pass configuration, consisting of a list of pre-prune, /// post-prune, and post-fixup passes. struct PassConfiguration { /// Pre-prune passes. /// /// These passes are called on the graph after it is built, and before any - /// atoms have been pruned. + /// symbols have been pruned. /// - /// Notable use cases: Marking atoms live or should-discard. - AtomGraphPassList PrePrunePasses; + /// Notable use cases: Marking symbols live or should-discard. + LinkGraphPassList PrePrunePasses; /// Post-prune passes. /// - /// These passes are called on the graph after dead and should-discard atoms - /// have been removed, but before fixups are applied. + /// These passes are called on the graph after dead stripping, but before + /// fixups are applied. /// - /// Notable use cases: Building GOT, stub, and TLV atoms. - AtomGraphPassList PostPrunePasses; + /// Notable use cases: Building GOT, stub, and TLV symbols. + LinkGraphPassList PostPrunePasses; /// Post-fixup passes. /// - /// These passes are called on the graph after atom contents has been copied + /// These passes are called on the graph after block contents has been copied /// to working memory, and fixups applied. /// /// Notable use cases: Testing and validation. - AtomGraphPassList PostFixupPasses; + LinkGraphPassList PostFixupPasses; }; /// A map of symbol names to resolved addresses. using AsyncLookupResult = DenseMap; -/// A function to call with a resolved symbol map (See AsyncLookupResult) or an -/// error if resolution failed. -using JITLinkAsyncLookupContinuation = - std::function LR)>; +/// A function object to call with a resolved symbol map (See AsyncLookupResult) +/// or an error if resolution failed. +class JITLinkAsyncLookupContinuation { +public: + virtual ~JITLinkAsyncLookupContinuation() {} + virtual void run(Expected LR) = 0; -/// An asynchronous symbol lookup. Performs a search (possibly asynchronously) -/// for the given symbols, calling the given continuation with either the result -/// (if the lookup succeeds), or an error (if the lookup fails). -using JITLinkAsyncLookupFunction = - std::function &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation)>; +private: + virtual void anchor(); +}; + +/// Create a lookup continuation from a function object. +template +std::unique_ptr +createLookupContinuation(Continuation Cont) { + + class Impl final : public JITLinkAsyncLookupContinuation { + public: + Impl(Continuation C) : C(std::move(C)) {} + void run(Expected LR) override { C(std::move(LR)); } + + private: + Continuation C; + }; + + return std::make_unique(std::move(Cont)); +}; /// Holds context for a single jitLink invocation. class JITLinkContext { @@ -881,13 +995,13 @@ public: /// lookup continutation which it must call with a result to continue the /// linking process. virtual void lookup(const DenseSet &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation) = 0; + std::unique_ptr LC) = 0; - /// Called by JITLink once all defined atoms in the graph have been assigned - /// their final memory locations in the target process. At this point he - /// atom graph can be, inspected to build a symbol table however the atom + /// Called by JITLink once all defined symbols in the graph have been assigned + /// their final memory locations in the target process. At this point the + /// LinkGraph can be inspected to build a symbol table, however the block /// content will not generally have been copied to the target location yet. - virtual void notifyResolved(AtomGraph &G) = 0; + virtual void notifyResolved(LinkGraph &G) = 0; /// Called by JITLink to notify the context that the object has been /// finalized (i.e. emitted to memory and memory permissions set). If all of @@ -904,20 +1018,20 @@ public: /// Returns the mark-live pass to be used for this link. If no pass is /// returned (the default) then the target-specific linker implementation will - /// choose a conservative default (usually marking all atoms live). + /// choose a conservative default (usually marking all symbols live). /// This function is only called if shouldAddDefaultTargetPasses returns true, /// otherwise the JITContext is responsible for adding a mark-live pass in /// modifyPassConfig. - virtual AtomGraphPassFunction getMarkLivePass(const Triple &TT) const; + virtual LinkGraphPassFunction getMarkLivePass(const Triple &TT) const; /// Called by JITLink to modify the pass pipeline prior to linking. /// The default version performs no modification. virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config); }; -/// Marks all atoms in a graph live. This can be used as a default, conservative -/// mark-live implementation. -Error markAllAtomsLive(AtomGraph &G); +/// Marks all symbols in a graph live. This can be used as a default, +/// conservative mark-live implementation. +Error markAllSymbolsLive(LinkGraph &G); /// Basic JITLink implementation. /// diff --git a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h index 9d0b37fe4a4..ac5a593bb77 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -33,20 +33,19 @@ public: class SegmentRequest { public: SegmentRequest() = default; - SegmentRequest(size_t ContentSize, unsigned ContentAlign, - uint64_t ZeroFillSize, unsigned ZeroFillAlign) - : ContentSize(ContentSize), ZeroFillSize(ZeroFillSize), - ContentAlign(ContentAlign), ZeroFillAlign(ZeroFillAlign) {} + SegmentRequest(uint64_t Alignment, size_t ContentSize, + uint64_t ZeroFillSize) + : Alignment(Alignment), ContentSize(ContentSize), + ZeroFillSize(ZeroFillSize) { + assert(isPowerOf2_32(Alignment) && "Alignment must be power of 2"); + } + uint64_t getAlignment() const { return Alignment; } size_t getContentSize() const { return ContentSize; } - unsigned getContentAlignment() const { return ContentAlign; } uint64_t getZeroFillSize() const { return ZeroFillSize; } - unsigned getZeroFillAlignment() const { return ZeroFillAlign; } - private: + uint64_t Alignment = 0; size_t ContentSize = 0; uint64_t ZeroFillSize = 0; - unsigned ContentAlign = 0; - unsigned ZeroFillAlign = 0; }; using SegmentsRequestMap = DenseMap; diff --git a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h index 1271ad962b3..b47a798c760 100644 --- a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -20,24 +20,23 @@ namespace jitlink { template class BasicGOTAndStubsBuilder { public: - BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {} void run() { - // We're going to be adding new atoms, but we don't want to iterate over - // the newly added ones, so just copy the existing atoms out. - std::vector DAs(G.defined_atoms().begin(), - G.defined_atoms().end()); + // We're going to be adding new blocks, but we don't want to iterate over + // the newly added ones, so just copy the existing blocks out. + std::vector Blocks(G.blocks().begin(), G.blocks().end()); - for (auto *DA : DAs) - for (auto &E : DA->edges()) + for (auto *B : Blocks) + for (auto &E : B->edges()) if (impl().isGOTEdge(E)) - impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget())); + impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); else if (impl().isExternalBranchEdge(E)) - impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget())); + impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); } protected: - Atom &getGOTEntryAtom(Atom &Target) { + Symbol &getGOTEntrySymbol(Symbol &Target) { assert(Target.hasName() && "GOT edge cannot point to anonymous target"); auto GOTEntryI = GOTEntries.find(Target.getName()); @@ -49,31 +48,31 @@ protected: GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; } - assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); return *GOTEntryI->second; } - Atom &getStubAtom(Atom &Target) { + Symbol &getStubSymbol(Symbol &Target) { assert(Target.hasName() && "External branch edge can not point to an anonymous target"); auto StubI = Stubs.find(Target.getName()); if (StubI == Stubs.end()) { - auto &StubAtom = impl().createStub(Target); - StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + auto &StubSymbol = impl().createStub(Target); + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; } - assert(StubI != Stubs.end() && "Count not get stub atom"); + assert(StubI != Stubs.end() && "Count not get stub symbol"); return *StubI->second; } - AtomGraph &G; + LinkGraph &G; private: BuilderImpl &impl() { return static_cast(*this); } - DenseMap GOTEntries; - DenseMap Stubs; + DenseMap GOTEntries; + DenseMap Stubs; }; } // end namespace jitlink diff --git a/lib/ExecutionEngine/JITLink/CMakeLists.txt b/lib/ExecutionEngine/JITLink/CMakeLists.txt index e81648311cf..ad3427fdfe3 100644 --- a/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ b/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -5,7 +5,7 @@ add_llvm_library(LLVMJITLink EHFrameSupport.cpp MachO.cpp MachO_x86_64.cpp - MachOAtomGraphBuilder.cpp + MachOLinkGraphBuilder.cpp DEPENDS intrinsics_gen diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index f373f2d92b0..355a000bb98 100644 --- a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -17,65 +17,57 @@ namespace llvm { namespace jitlink { -EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, - JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) - : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), - EHFrameAddress(EHFrameAddress), - EHFrameReader(EHFrameContent, G.getEndianness()), - FDEToCIERelocKind(FDEToCIERelocKind), - FDEToTargetRelocKind(FDEToTargetRelocKind) {} +EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress, + StringRef EHFrameContent, + unsigned PointerSize, + support::endianness Endianness) + : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent), + PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {} -Error EHFrameParser::atomize() { +Error EHFrameBinaryParser::addToGraph() { while (!EHFrameReader.empty()) { size_t RecordOffset = EHFrameReader.getOffset(); + JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; LLVM_DEBUG({ dbgs() << "Processing eh-frame record at " - << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) - << " (offset " << RecordOffset << ")\n"; + << format("0x%016" PRIx64, RecordAddress) << " (offset " + << RecordOffset << ")\n"; }); - size_t CIELength = 0; - uint32_t CIELengthField; - if (auto Err = EHFrameReader.readInteger(CIELengthField)) + size_t RecordLength = 0; + uint32_t RecordLengthField; + if (auto Err = EHFrameReader.readInteger(RecordLengthField)) return Err; - // Process CIE length/extended-length fields to build the atom. + // Process CIE/FDE length/extended-length fields to build the blocks. // // The value of these fields describe the length of the *rest* of the CIE // (not including data up to the end of the field itself) so we have to - // bump CIELength to include the data up to the end of the field: 4 bytes + // bump RecordLength to include the data up to the end of the field: 4 bytes // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. - if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + if (RecordLengthField == 0) // Length 0 means end of __eh_frame section. break; // If the regular length field's value is 0xffffffff, use extended length. - if (CIELengthField == 0xffffffff) { - uint64_t CIEExtendedLengthField; - if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + if (RecordLengthField == 0xffffffff) { + uint64_t ExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(ExtendedLengthField)) return Err; - if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + if (ExtendedLengthField > EHFrameReader.bytesRemaining()) return make_error("CIE record extends past the end of " "the __eh_frame section"); - if (CIEExtendedLengthField + 12 > std::numeric_limits::max()) + if (ExtendedLengthField + 12 > std::numeric_limits::max()) return make_error("CIE record too large to process"); - CIELength = CIEExtendedLengthField + 12; + RecordLength = ExtendedLengthField + 12; } else { - if (CIELengthField > EHFrameReader.bytesRemaining()) + if (RecordLengthField > EHFrameReader.bytesRemaining()) return make_error("CIE record extends past the end of " "the __eh_frame section"); - CIELength = CIELengthField + 4; + RecordLength = RecordLengthField + 4; } - LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); - - // Add an atom for this record. - CurRecordAtom = &G.addAnonymousAtom( - EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); - CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n"); // Read the CIE Pointer. size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); @@ -85,21 +77,24 @@ Error EHFrameParser::atomize() { // Based on the CIE pointer value, parse this as a CIE or FDE record. if (CIEPointer == 0) { - if (auto Err = processCIE()) + if (auto Err = processCIE(RecordOffset, RecordLength)) return Err; } else { - if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress, + CIEPointer)) return Err; } - EHFrameReader.setOffset(RecordOffset + CIELength); + EHFrameReader.setOffset(RecordOffset + RecordLength); } return Error::success(); } -Expected -EHFrameParser::parseAugmentationString() { +void EHFrameBinaryParser::anchor() {} + +Expected +EHFrameBinaryParser::parseAugmentationString() { AugmentationInfo AugInfo; uint8_t NextChar; uint8_t *NextField = &AugInfo.Fields[0]; @@ -139,14 +134,14 @@ EHFrameParser::parseAugmentationString() { return std::move(AugInfo); } -Expected EHFrameParser::readAbsolutePointer() { +Expected EHFrameBinaryParser::readAbsolutePointer() { static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), "Result must be able to hold a uint64_t"); JITTargetAddress Addr; - if (G.getPointerSize() == 8) { + if (PointerSize == 8) { if (auto Err = EHFrameReader.readInteger(Addr)) return std::move(Err); - } else if (G.getPointerSize() == 4) { + } else if (PointerSize == 4) { uint32_t Addr32; if (auto Err = EHFrameReader.readInteger(Addr32)) return std::move(Err); @@ -156,14 +151,19 @@ Expected EHFrameParser::readAbsolutePointer() { return Addr; } -Error EHFrameParser::processCIE() { +Error EHFrameBinaryParser::processCIE(size_t RecordOffset, + size_t RecordLength) { // Use the dwarf namespace for convenient access to pointer encoding // constants. using namespace dwarf; LLVM_DEBUG(dbgs() << " Record is CIE\n"); - CIEInformation CIEInfo(*CurRecordAtom); + auto &CIESymbol = + createCIERecord(EHFrameAddress + RecordOffset, + EHFrameContent.substr(RecordOffset, RecordLength)); + + CIEInformation CIEInfo(CIESymbol); uint8_t Version = 0; if (auto Err = EHFrameReader.readInteger(Version)) @@ -179,7 +179,7 @@ Error EHFrameParser::processCIE() { // Skip the EH Data field if present. if (AugInfo->EHDataFieldPresent) - if (auto Err = EHFrameReader.skip(G.getPointerSize())) + if (auto Err = EHFrameReader.skip(PointerSize)) return Err; // Read and sanity check the code alignment factor. @@ -226,7 +226,7 @@ Error EHFrameParser::processCIE() { return make_error( "Unsupported LSDA pointer encoding " + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); break; } case 'P': { @@ -239,7 +239,7 @@ Error EHFrameParser::processCIE() { "Unspported personality pointer " "encoding " + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); uint32_t PersonalityPointerAddress; if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) return Err; @@ -254,7 +254,7 @@ Error EHFrameParser::processCIE() { "Unsupported FDE address pointer " "encoding " + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); break; } default: @@ -267,15 +267,16 @@ Error EHFrameParser::processCIE() { return make_error("Read past the end of the augmentation " "data while parsing fields"); - assert(!CIEInfos.count(CurRecordAtom->getAddress()) && + assert(!CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); - CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo); + CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); return Error::success(); } -Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, - uint32_t CIEPointer) { +Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength, + JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { LLVM_DEBUG(dbgs() << " Record is FDE\n"); LLVM_DEBUG({ @@ -286,16 +287,11 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); if (CIEInfoItr == CIEInfos.end()) return make_error( - "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) + + "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) + " points to non-existant CIE at " + formatv("{0:x16}", CIEPointerAddress - CIEPointer)); auto &CIEInfo = CIEInfoItr->second; - // The CIEPointer looks good. Add a relocation. - CurRecordAtom->addEdge(FDEToCIERelocKind, - CIEPointerAddress - CurRecordAtom->getAddress(), - *CIEInfo.CIEAtom, 0); - // Read and sanity check the PC-start pointer and size. JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); @@ -305,83 +301,68 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; LLVM_DEBUG({ - dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; }); - auto *TargetAtom = G.getAtomByAddress(PCBegin); + auto *TargetSymbol = getSymbolAtAddress(PCBegin); - if (!TargetAtom) + if (!TargetSymbol) return make_error("FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point at atom"); + " does not point at symbol"); - if (TargetAtom->getAddress() != PCBegin) + if (TargetSymbol->getAddress() != PCBegin) return make_error( "FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point to start of atom at " + - formatv("{0:x16}", TargetAtom->getAddress())); + " does not point to start of symbol at " + + formatv("{0:x16}", TargetSymbol->getAddress())); - LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); - - // The PC-start pointer and size look good. Add relocations. - CurRecordAtom->addEdge(FDEToTargetRelocKind, - PCBeginAddress - CurRecordAtom->getAddress(), - *TargetAtom, 0); - - // Add a keep-alive relocation from the function to the FDE to ensure it is - // not dead stripped. - TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n"); // Skip over the PC range size field. - if (auto Err = EHFrameReader.skip(G.getPointerSize())) + if (auto Err = EHFrameReader.skip(PointerSize)) return Err; + Symbol *LSDASymbol = nullptr; + JITTargetAddress LSDAAddress = 0; if (CIEInfo.FDEsHaveLSDAField) { uint64_t AugmentationDataSize; if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) return Err; - if (AugmentationDataSize != G.getPointerSize()) + if (AugmentationDataSize != PointerSize) return make_error( "Unexpected FDE augmentation data size (expected " + - Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) + - ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress())); - JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) + + ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset)); + LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); auto LSDADelta = readAbsolutePointer(); if (!LSDADelta) return LSDADelta.takeError(); JITTargetAddress LSDA = LSDAAddress + *LSDADelta; - auto *LSDAAtom = G.getAtomByAddress(LSDA); + LSDASymbol = getSymbolAtAddress(LSDA); - if (!LSDAAtom) + if (!LSDASymbol) return make_error("FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point at atom"); + " does not point at symbol"); - if (LSDAAtom->getAddress() != LSDA) + if (LSDASymbol->getAddress() != LSDA) return make_error( "FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point to start of atom at " + - formatv("{0:x16}", LSDAAtom->getAddress())); + " does not point to start of symbol at " + + formatv("{0:x16}", LSDASymbol->getAddress())); - LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); - - // LSDA looks good. Add relocations. - CurRecordAtom->addEdge(FDEToTargetRelocKind, - LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, - 0); + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n"); } - return Error::success(); -} + JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; + auto FDESymbol = createFDERecord( + RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength), + *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol, + PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress); -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) { - return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, - FDEToCIERelocKind, FDEToTargetRelocKind) - .atomize(); + return FDESymbol.takeError(); } // Determine whether we can register EH tables. @@ -523,7 +504,7 @@ InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() { InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {} -AtomGraphPassFunction +LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, StoreFrameRangeFunction StoreRangeAddress) { const char *EHFrameSectionName = nullptr; @@ -533,14 +514,14 @@ createEHFrameRecorderPass(const Triple &TT, EHFrameSectionName = ".eh_frame"; auto RecordEHFrame = - [EHFrameSectionName, - StoreFrameRange = std::move(StoreRangeAddress)](AtomGraph &G) -> Error { - // Search for a non-empty eh-frame and record the address of the first atom - // in it. + [EHFrameSectionName, + StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first + // symbol in it. JITTargetAddress Addr = 0; size_t Size = 0; if (auto *S = G.findSectionByName(EHFrameSectionName)) { - auto R = S->getRange(); + auto R = SectionRange(*S); Addr = R.getStart(); Size = R.getSize(); } diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index d679edef7ea..6f9f68ad838 100644 --- a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,18 +21,31 @@ namespace llvm { namespace jitlink { -/// A generic parser for eh-frame sections. +/// A generic binary parser for eh-frame sections. /// -/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and -/// FDEToTarget relocation kinds. -class EHFrameParser { +/// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph. +/// +/// This parser assumes that the user has already verified that the EH-frame's +/// address range does not overlap any other section/symbol, so that generated +/// CIE/FDE records do not overlap other sections/symbols. +class EHFrameBinaryParser { public: - EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, - JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind); - Error atomize(); + EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent, + unsigned PointerSize, support::endianness Endianness); + virtual ~EHFrameBinaryParser() {} + + Error addToGraph(); private: + virtual void anchor(); + virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0; + virtual Symbol &createCIERecord(JITTargetAddress RecordAddr, + StringRef RecordContent) = 0; + virtual Expected + createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent, + Symbol &CIE, size_t CIEOffset, Symbol &Func, + size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0; + struct AugmentationInfo { bool AugmentationDataPresent = false; bool EHDataFieldPresent = false; @@ -41,31 +54,24 @@ private: Expected parseAugmentationString(); Expected readAbsolutePointer(); - Error processCIE(); - Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + Error processCIE(size_t RecordOffset, size_t RecordLength); + Error processFDE(size_t RecordOffset, size_t RecordLength, + JITTargetAddress CIEPointerOffset, uint32_t CIEPointer); struct CIEInformation { CIEInformation() = default; - CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {} - DefinedAtom *CIEAtom = nullptr; + CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} + Symbol *CIESymbol = nullptr; bool FDEsHaveLSDAField = false; }; - AtomGraph &G; - Section &EHFrameSection; - StringRef EHFrameContent; JITTargetAddress EHFrameAddress; + StringRef EHFrameContent; + unsigned PointerSize; BinaryStreamReader EHFrameReader; - DefinedAtom *CurRecordAtom = nullptr; DenseMap CIEInfos; - Edge::Kind FDEToCIERelocKind; - Edge::Kind FDEToTargetRelocKind; }; -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); - } // end namespace jitlink } // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/JITLink.cpp b/lib/ExecutionEngine/JITLink/JITLink.cpp index 9d0a7459dc0..54324f74521 100644 --- a/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -56,95 +56,143 @@ std::error_code JITLinkError::convertToErrorCode() const { return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); } -const StringRef getGenericEdgeKindName(Edge::Kind K) { +const char *getGenericEdgeKindName(Edge::Kind K) { switch (K) { case Edge::Invalid: return "INVALID RELOCATION"; case Edge::KeepAlive: return "Keep-Alive"; - case Edge::LayoutNext: - return "Layout-Next"; default: llvm_unreachable("Unrecognized relocation kind"); } } -raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { +const char *getLinkageName(Linkage L) { + switch (L) { + case Linkage::Strong: + return "strong"; + case Linkage::Weak: + return "weak"; + } +} + +const char *getScopeName(Scope S) { + switch (S) { + case Scope::Default: + return "default"; + case Scope::Hidden: + return "hidden"; + case Scope::Local: + return "local"; + } +} + +raw_ostream &operator<<(raw_ostream &OS, const Block &B) { + return OS << formatv("{0:x16}", B.getAddress()) << " -- " + << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": " + << (B.isZeroFill() ? "zero-fill" : "content") + << ", align = " << B.getAlignment() + << ", align-ofs = " << B.getAlignmentOffset() + << ", section = " << B.getSection().getName(); +} + +raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { OS << "<"; - if (A.getName().empty()) - OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + if (Sym.getName().empty()) + OS << "*anon*"; else - OS << A.getName(); - OS << " ["; - if (A.isDefined()) { - auto &DA = static_cast(A); - OS << " section=" << DA.getSection().getName(); - if (DA.isLive()) - OS << " live"; - if (DA.shouldDiscard()) - OS << " should-discard"; - } else - OS << " external"; - OS << " ]>"; + OS << Sym.getName(); + OS << ": flags = "; + switch (Sym.getLinkage()) { + case Linkage::Strong: + OS << 'S'; + break; + case Linkage::Weak: + OS << 'W'; + break; + } + switch (Sym.getScope()) { + case Scope::Default: + OS << 'D'; + break; + case Scope::Hidden: + OS << 'H'; + break; + case Scope::Local: + OS << 'L'; + break; + } + OS << (Sym.isLive() ? '+' : '-') + << ", size = " << formatv("{0:x8}", Sym.getSize()) + << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " (" + << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + " + << formatv("{0:x8}", Sym.getOffset()); + if (Sym.isDefined()) + OS << " " << Sym.getBlock().getSection().getName(); + OS << ")>"; return OS; } -void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, StringRef EdgeKindName) { - OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) - << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName - << " -> " << E.getTarget() << " + " << E.getAddend(); + OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": " + << formatv("{0:x16}", B.getAddress()) << " + " << E.getOffset() << " -- " + << EdgeKindName << " -> " << E.getTarget() << " + " << E.getAddend(); } Section::~Section() { - for (auto *DA : DefinedAtoms) - DA->~DefinedAtom(); + for (auto *Sym : Symbols) + Sym->~Symbol(); } -void AtomGraph::dump(raw_ostream &OS, +void LinkGraph::dump(raw_ostream &OS, std::function EdgeKindToName) { if (!EdgeKindToName) EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; - OS << "Defined atoms:\n"; - for (auto *DA : defined_atoms()) { - OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + OS << "Symbols:\n"; + for (auto *Sym : defined_symbols()) { + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; - for (auto &E : DA->edges()) { - OS << " "; - StringRef EdgeName = (E.getKind() < Edge::FirstRelocation - ? getGenericEdgeKindName(E.getKind()) - : EdgeKindToName(E.getKind())); + if (Sym->isDefined()) { + for (auto &E : Sym->getBlock().edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); - if (!EdgeName.empty()) - printEdge(OS, *DA, E, EdgeName); - else { - auto EdgeNumberString = std::to_string(E.getKind()); - printEdge(OS, *DA, E, EdgeNumberString); + if (!EdgeName.empty()) + printEdge(OS, Sym->getBlock(), E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, Sym->getBlock(), E, EdgeNumberString); + } + OS << "\n"; } - OS << "\n"; } } - OS << "Absolute atoms:\n"; - for (auto *A : absolute_atoms()) - OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + OS << "Absolute symbols:\n"; + for (auto *Sym : absolute_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; - OS << "External atoms:\n"; - for (auto *A : external_atoms()) - OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + OS << "External symbols:\n"; + for (auto *Sym : external_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; } +void JITLinkAsyncLookupContinuation::anchor() {} + JITLinkContext::~JITLinkContext() {} bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { return true; } -AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { - return AtomGraphPassFunction(); +LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return LinkGraphPassFunction(); } Error JITLinkContext::modifyPassConfig(const Triple &TT, @@ -152,9 +200,9 @@ Error JITLinkContext::modifyPassConfig(const Triple &TT, return Error::success(); } -Error markAllAtomsLive(AtomGraph &G) { - for (auto *DA : G.defined_atoms()) - DA->setLive(true); +Error markAllSymbolsLive(LinkGraph &G) { + for (auto *Sym : G.defined_symbols()) + Sym->setLive(true); return Error::success(); } diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 877107ffe25..d4270b5aa79 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "JITLinkGeneric.h" -#include "EHFrameSupportImpl.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/MemoryBuffer.h" @@ -25,7 +24,7 @@ JITLinkerBase::~JITLinkerBase() {} void JITLinkerBase::linkPhase1(std::unique_ptr Self) { - // Build the atom graph. + // Build the link graph. if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) G = std::move(*GraphOrErr); else @@ -33,33 +32,33 @@ void JITLinkerBase::linkPhase1(std::unique_ptr Self) { assert(G && "Graph should have been created by buildGraph above"); // Prune and optimize the graph. - if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + if (auto Err = runPasses(Passes.PrePrunePasses)) return Ctx->notifyFailed(std::move(Err)); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n"; dumpGraph(dbgs()); }); prune(*G); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n"; dumpGraph(dbgs()); }); // Run post-pruning passes. - if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + if (auto Err = runPasses(Passes.PostPrunePasses)) return Ctx->notifyFailed(std::move(Err)); - // Sort atoms into segments. - layOutAtoms(); + // Sort blocks into segments. + auto Layout = layOutBlocks(); // Allocate memory for segments. if (auto Err = allocateSegments(Layout)) return Ctx->notifyFailed(std::move(Err)); - // Notify client that the defined atoms have been assigned addresses. + // Notify client that the defined symbols have been assigned addresses. Ctx->notifyResolved(*G); auto ExternalSymbols = getExternalSymbolNames(); @@ -74,42 +73,42 @@ void JITLinkerBase::linkPhase1(std::unique_ptr Self) { // [Self=std::move(Self)](Expected Result) { // Self->linkPhase2(std::move(Self), std::move(Result)); // }); - // - // FIXME: Use move capture once we have c++14. auto *TmpCtx = Ctx.get(); - auto *UnownedSelf = Self.release(); - auto Phase2Continuation = - [UnownedSelf](Expected LookupResult) { - std::unique_ptr Self(UnownedSelf); - UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); - }; - TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); + TmpCtx->lookup(std::move(ExternalSymbols), + createLookupContinuation( + [S = std::move(Self), L = std::move(Layout)]( + Expected LookupResult) mutable { + auto &TmpSelf = *S; + TmpSelf.linkPhase2(std::move(S), std::move(LookupResult), + std::move(L)); + })); } void JITLinkerBase::linkPhase2(std::unique_ptr Self, - Expected LR) { + Expected LR, + SegmentLayoutMap Layout) { // If the lookup failed, bail out. if (!LR) return deallocateAndBailOut(LR.takeError()); - // Assign addresses to external atoms. + // Assign addresses to external addressables. applyLookupResult(*LR); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; dumpGraph(dbgs()); }); - // Copy atom content to working memory and fix up. - if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + // Copy block content to working memory and fix up. + if (auto Err = copyAndFixUpBlocks(Layout, *Alloc)) return deallocateAndBailOut(std::move(Err)); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n"; dumpGraph(dbgs()); }); - if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + if (auto Err = runPasses(Passes.PostFixupPasses)) return deallocateAndBailOut(std::move(Err)); // FIXME: Use move capture once we have c++14. @@ -128,82 +127,38 @@ void JITLinkerBase::linkPhase3(std::unique_ptr Self, Error Err) { Ctx->notifyFinalized(std::move(Alloc)); } -Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { +Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) { for (auto &P : Passes) - if (auto Err = P(G)) + if (auto Err = P(*G)) return Err; return Error::success(); } -void JITLinkerBase::layOutAtoms() { - // Group sections by protections, and whether or not they're zero-fill. - for (auto &S : G->sections()) { +JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { - // Skip empty sections. - if (S.atoms_empty()) - continue; + SegmentLayoutMap Layout; - auto &SL = Layout[S.getProtectionFlags()]; - if (S.isZeroFill()) - SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + /// Partition blocks based on permissions and content vs. zero-fill. + for (auto *B : G->blocks()) { + auto &SegLists = Layout[B->getSection().getProtectionFlags()]; + if (!B->isZeroFill()) + SegLists.ContentBlocks.push_back(B); else - SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + SegLists.ZeroFillBlocks.push_back(B); } - // Sort sections within the layout by ordinal. - { - auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, - const SegmentLayout::SectionLayout &RHS) { - return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); - }; - for (auto &KV : Layout) { - auto &SL = KV.second; - std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), - CompareByOrdinal); - std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), - CompareByOrdinal); - } - } - - // Add atoms to the sections. + /// Sort blocks within each list. for (auto &KV : Layout) { - auto &SL = KV.second; - for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { - for (auto &SI : *SIList) { - // First build the set of layout-heads (i.e. "heads" of layout-next - // chains) by copying the section atoms, then eliminating any that - // appear as layout-next targets. - DenseSet LayoutHeads; - for (auto *DA : SI.S->atoms()) - LayoutHeads.insert(DA); - for (auto *DA : SI.S->atoms()) - if (DA->hasLayoutNext()) - LayoutHeads.erase(&DA->getLayoutNext()); + auto CompareBlocks = [](const Block *LHS, const Block *RHS) { + if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal()) + return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal(); + return LHS->getOrdinal() < RHS->getOrdinal(); + }; - // Next, sort the layout heads by address order. - std::vector OrderedLayoutHeads; - OrderedLayoutHeads.reserve(LayoutHeads.size()); - for (auto *DA : LayoutHeads) - OrderedLayoutHeads.push_back(DA); - - // Now sort the list of layout heads by address. - std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); - - // Now populate the SI.Atoms field by appending each of the chains. - for (auto *DA : OrderedLayoutHeads) { - SI.Atoms.push_back(DA); - while (DA->hasLayoutNext()) { - auto &Next = DA->getLayoutNext(); - SI.Atoms.push_back(&Next); - DA = &Next; - } - } - } - } + auto &SegLists = KV.second; + llvm::sort(SegLists.ContentBlocks, CompareBlocks); + llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks); } LLVM_DEBUG({ @@ -213,18 +168,16 @@ void JITLinkerBase::layOutAtoms() { << static_cast(KV.first) << ":\n"; auto &SL = KV.second; for (auto &SIEntry : - {std::make_pair(&SL.ContentSections, "content sections"), - std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { - auto &SIList = *SIEntry.first; + {std::make_pair(&SL.ContentBlocks, "content block"), + std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) { dbgs() << " " << SIEntry.second << ":\n"; - for (auto &SI : SIList) { - dbgs() << " " << SI.S->getName() << ":\n"; - for (auto *DA : SI.Atoms) - dbgs() << " " << *DA << "\n"; - } + for (auto *B : *SIEntry.first) + dbgs() << " " << *B << "\n"; } } }); + + return Layout; } Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { @@ -234,61 +187,36 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { JITLinkMemoryManager::SegmentsRequestMap Segments; for (auto &KV : Layout) { auto &Prot = KV.first; - auto &SegLayout = KV.second; + auto &SegLists = KV.second; + + uint64_t SegAlign = 1; // Calculate segment content size. size_t SegContentSize = 0; - uint32_t SegContentAlign = 1; - for (auto &SI : SegLayout.ContentSections) { - assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); - assert(!SI.Atoms.empty() && "Section layouts must not be empty"); - - // Bump to section alignment before processing atoms. - SegContentSize = alignTo(SegContentSize, SI.S->getAlignment()); - SegContentAlign = std::max(SegContentAlign, SI.S->getAlignment()); - - for (auto *DA : SI.Atoms) { - SegContentSize = alignTo(SegContentSize, DA->getAlignment()); - SegContentSize += DA->getSize(); - SegContentAlign = std::max(SegContentAlign, DA->getAlignment()); - } + for (auto *B : SegLists.ContentBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegContentSize = alignToBlock(SegContentSize, *B); + SegContentSize += B->getSize(); } - // Calculate segment zero-fill size. - uint64_t SegZeroFillSize = 0; - uint32_t SegZeroFillAlign = 1; + uint64_t SegZeroFillStart = SegContentSize; + uint64_t SegZeroFillEnd = SegZeroFillStart; - for (auto &SI : SegLayout.ZeroFillSections) { - assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); - assert(!SI.Atoms.empty() && "Section layouts must not be empty"); - - // Bump to section alignment before processing atoms. - SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment()); - SegZeroFillAlign = std::max(SegZeroFillAlign, SI.S->getAlignment()); - - for (auto *DA : SI.Atoms) { - SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); - SegZeroFillSize += DA->getSize(); - SegZeroFillAlign = std::max(SegZeroFillAlign, SI.S->getAlignment()); - } + for (auto *B : SegLists.ZeroFillBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B); + SegZeroFillEnd += B->getSize(); } - assert(isPowerOf2_32(SegContentAlign) && - "Expected content alignment to be power of 2"); - assert(isPowerOf2_32(SegZeroFillAlign) && - "Expected zero-fill alignment to be power of 2"); - // Round content alignment up to segment alignment. - SegContentAlign = std::max(SegContentAlign, SegZeroFillAlign); - - Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, - SegZeroFillAlign}; + Segments[Prot] = {SegAlign, SegContentSize, + SegZeroFillEnd - SegZeroFillStart}; LLVM_DEBUG({ dbgs() << (&KV == &*Layout.begin() ? "" : "; ") - << static_cast(Prot) << ": " - << SegContentSize << " content bytes (alignment " - << SegContentAlign << ") + " << SegZeroFillSize - << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + << static_cast(Prot) + << ": alignment = " << SegAlign + << ", content size = " << SegContentSize + << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart); }); } LLVM_DEBUG(dbgs() << " }\n"); @@ -307,22 +235,19 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } }); - // Update atom target addresses. + // Update block target addresses. for (auto &KV : Layout) { auto &Prot = KV.first; auto &SL = KV.second; - JITTargetAddress AtomTargetAddr = + JITTargetAddress NextBlockAddr = Alloc->getTargetMemory(static_cast(Prot)); - for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) - for (auto &SI : *SIList) { - AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment()); - for (auto *DA : SI.Atoms) { - AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); - DA->setAddress(AtomTargetAddr); - AtomTargetAddr += DA->getSize(); - } + for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks}) + for (auto *B : *SIList) { + NextBlockAddr = alignToBlock(NextBlockAddr, *B); + B->setAddress(NextBlockAddr); + NextBlockAddr += B->getSize(); } } @@ -330,34 +255,35 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } DenseSet JITLinkerBase::getExternalSymbolNames() const { - // Identify unresolved external atoms. + // Identify unresolved external symbols. DenseSet UnresolvedExternals; - for (auto *DA : G->external_atoms()) { - assert(DA->getAddress() == 0 && + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && "External has already been assigned an address"); - assert(DA->getName() != StringRef() && DA->getName() != "" && + assert(Sym->getName() != StringRef() && Sym->getName() != "" && "Externals must be named"); - UnresolvedExternals.insert(DA->getName()); + UnresolvedExternals.insert(Sym->getName()); } return UnresolvedExternals; } void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { - for (auto &KV : Result) { - Atom &A = G->getAtomByName(KV.first); - assert(A.getAddress() == 0 && "Atom already resolved"); - A.setAddress(KV.second.getAddress()); + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && "Symbol already resolved"); + assert(!Sym->isDefined() && "Symbol being resolved is already defined"); + assert(Result.count(Sym->getName()) && "Missing resolution for symbol"); + Sym->getAddressable().setAddress(Result[Sym->getName()].getAddress()); } LLVM_DEBUG({ dbgs() << "Externals after applying lookup result:\n"; - for (auto *A : G->external_atoms()) - dbgs() << " " << A->getName() << ": " - << formatv("{0:x16}", A->getAddress()) << "\n"; + for (auto *Sym : G->external_symbols()) + dbgs() << " " << Sym->getName() << ": " + << formatv("{0:x16}", Sym->getAddress()) << "\n"; }); - assert(llvm::all_of(G->external_atoms(), - [](Atom *A) { return A->getAddress() != 0; }) && - "All atoms should have been resolved by this point"); + assert(llvm::all_of(G->external_symbols(), + [](Symbol *Sym) { return Sym->getAddress() != 0; }) && + "All symbols should have been resolved by this point"); } void JITLinkerBase::deallocateAndBailOut(Error Err) { @@ -371,96 +297,60 @@ void JITLinkerBase::dumpGraph(raw_ostream &OS) { G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); } -void prune(AtomGraph &G) { - std::vector Worklist; - DenseMap> EdgesToUpdate; +void prune(LinkGraph &G) { + std::vector Worklist; + DenseSet VisitedBlocks; - // Build the initial worklist from all atoms initially live. - for (auto *DA : G.defined_atoms()) { - if (!DA->isLive() || DA->shouldDiscard()) - continue; + // Build the initial worklist from all symbols initially live. + for (auto *Sym : G.defined_symbols()) + if (Sym->isLive()) + Worklist.push_back(Sym); - for (auto &E : DA->edges()) { - if (!E.getTarget().isDefined()) - continue; - - auto &EDT = static_cast(E.getTarget()); - - if (EDT.shouldDiscard()) - EdgesToUpdate[&EDT].push_back(&E); - else if (E.isKeepAlive() && !EDT.isLive()) - Worklist.push_back(&EDT); - } - } - - // Propagate live flags to all atoms reachable from the initial live set. + // Propagate live flags to all symbols reachable from the initial live set. while (!Worklist.empty()) { - DefinedAtom &NextLive = *Worklist.back(); + auto *Sym = Worklist.back(); Worklist.pop_back(); - assert(!NextLive.shouldDiscard() && - "should-discard nodes should never make it into the worklist"); + auto &B = Sym->getBlock(); - // If this atom has already been marked as live, or is marked to be - // discarded, then skip it. - if (NextLive.isLive()) + // Skip addressables that we've visited before. + if (VisitedBlocks.count(&B)) continue; - // Otherwise set it as live and add any non-live atoms that it points to - // to the worklist. - NextLive.setLive(true); + VisitedBlocks.insert(&B); - for (auto &E : NextLive.edges()) { - if (!E.getTarget().isDefined()) - continue; - - auto &EDT = static_cast(E.getTarget()); - - if (EDT.shouldDiscard()) - EdgesToUpdate[&EDT].push_back(&E); - else if (E.isKeepAlive() && !EDT.isLive()) - Worklist.push_back(&EDT); + for (auto &E : Sym->getBlock().edges()) { + if (E.getTarget().isDefined() && !E.getTarget().isLive()) { + E.getTarget().setLive(true); + Worklist.push_back(&E.getTarget()); + } } } - // Collect atoms to remove, then remove them from the graph. - std::vector AtomsToRemove; - for (auto *DA : G.defined_atoms()) - if (DA->shouldDiscard() || !DA->isLive()) - AtomsToRemove.push_back(DA); - - LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); - for (auto *DA : AtomsToRemove) { - LLVM_DEBUG(dbgs() << " " << *DA << "... "); - - // Check whether we need to replace this atom with an external atom. - // - // We replace if all of the following hold: - // (1) The atom is marked should-discard, - // (2) it has live edges (i.e. edges from live atoms) pointing to it. - // - // Otherwise we simply delete the atom. - - G.removeDefinedAtom(*DA); - - auto EdgesToUpdateItr = EdgesToUpdate.find(DA); - if (EdgesToUpdateItr != EdgesToUpdate.end()) { - auto &ExternalReplacement = G.addExternalAtom(DA->getName()); - for (auto *EdgeToUpdate : EdgesToUpdateItr->second) - EdgeToUpdate->setTarget(ExternalReplacement); - LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); - } else - LLVM_DEBUG(dbgs() << "deleted\n"); + // Collect all the symbols to remove, then remove them. + { + LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n"); + std::vector SymbolsToRemove; + for (auto *Sym : G.defined_symbols()) + if (!Sym->isLive()) + SymbolsToRemove.push_back(Sym); + for (auto *Sym : SymbolsToRemove) { + LLVM_DEBUG(dbgs() << " " << *Sym << "...\n"); + G.removeDefinedSymbol(*Sym); + } } - // Finally, discard any absolute symbols that were marked should-discard. + // Delete any unused blocks. { - std::vector AbsoluteAtomsToRemove; - for (auto *A : G.absolute_atoms()) - if (A->shouldDiscard() || A->isLive()) - AbsoluteAtomsToRemove.push_back(A); - for (auto *A : AbsoluteAtomsToRemove) - G.removeAbsoluteAtom(*A); + LLVM_DEBUG(dbgs() << "Dead-stripping blocks:\n"); + std::vector BlocksToRemove; + for (auto *B : G.blocks()) + if (!VisitedBlocks.count(B)) + BlocksToRemove.push_back(B); + for (auto *B : BlocksToRemove) { + LLVM_DEBUG(dbgs() << " " << *B << "...\n"); + G.removeBlock(*B); + } } } diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index eeb2527bd1b..07dee6cee20 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -41,39 +41,32 @@ public: protected: struct SegmentLayout { - using SectionAtomsList = std::vector; - struct SectionLayout { - SectionLayout(Section &S) : S(&S) {} + using BlocksList = std::vector; - Section *S; - SectionAtomsList Atoms; - }; - - using SectionLayoutList = std::vector; - - SectionLayoutList ContentSections; - SectionLayoutList ZeroFillSections; + BlocksList ContentBlocks; + BlocksList ZeroFillBlocks; }; using SegmentLayoutMap = DenseMap; // Phase 1: - // 1.1: Build atom graph + // 1.1: Build link graph // 1.2: Run pre-prune passes // 1.2: Prune graph // 1.3: Run post-prune passes - // 1.4: Sort atoms into segments + // 1.4: Sort blocks into segments // 1.5: Allocate segment memory // 1.6: Identify externals and make an async call to resolve function void linkPhase1(std::unique_ptr Self); // Phase 2: // 2.1: Apply resolution results - // 2.2: Fix up atom contents + // 2.2: Fix up block contents // 2.3: Call OnResolved callback // 2.3: Make an async call to transfer and finalize memory. void linkPhase2(std::unique_ptr Self, - Expected LookupResult); + Expected LookupResult, + SegmentLayoutMap Layout); // Phase 3: // 3.1: Call OnFinalized callback, handing off allocation. @@ -81,24 +74,37 @@ protected: // Build a graph from the given object buffer. // To be implemented by the client. - virtual Expected> + virtual Expected> buildGraph(MemoryBufferRef ObjBuffer) = 0; - // For debug dumping of the atom graph. + // For debug dumping of the link graph. virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + // Alight a JITTargetAddress to conform with block alignment requirements. + static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { + uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); + return Addr + Delta; + } + + // Alight a pointer to conform with block alignment requirements. + static char *alignToBlock(char *P, Block &B) { + uint64_t PAddr = static_cast(reinterpret_cast(P)); + uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment(); + return P + Delta; + } + private: // Run all passes in the given pass list, bailing out immediately if any pass // returns an error. - Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + Error runPasses(LinkGraphPassList &Passes); - // Copy atom contents and apply relocations. + // Copy block contents and apply relocations. // Implemented in JITLinker. virtual Error - copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const = 0; + copyAndFixUpBlocks(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; - void layOutAtoms(); + SegmentLayoutMap layOutBlocks(); Error allocateSegments(const SegmentLayoutMap &Layout); DenseSet getExternalSymbolNames() const; void applyLookupResult(AsyncLookupResult LR); @@ -108,8 +114,7 @@ private: std::unique_ptr Ctx; PassConfiguration Passes; - std::unique_ptr G; - SegmentLayoutMap Layout; + std::unique_ptr G; std::unique_ptr Alloc; }; @@ -140,17 +145,17 @@ private: } Error - copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const override { - LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + copyAndFixUpBlocks(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n"); for (auto &KV : Layout) { auto &Prot = KV.first; auto &SegLayout = KV.second; auto SegMem = Alloc.getWorkingMemory( static_cast(Prot)); - char *LastAtomEnd = SegMem.data(); - char *AtomDataPtr = LastAtomEnd; + char *LastBlockEnd = SegMem.data(); + char *BlockDataPtr = LastBlockEnd; LLVM_DEBUG({ dbgs() << " Processing segment " @@ -160,93 +165,79 @@ private: << " ]\n Processing content sections:\n"; }); - for (auto &SI : SegLayout.ContentSections) { - LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + for (auto *B : SegLayout.ContentBlocks) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); - AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment()); + // Pad to alignment/alignment-offset. + BlockDataPtr = alignToBlock(BlockDataPtr, *B); LLVM_DEBUG({ - dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr - << " to meet section alignment " - << " of " << SI.S->getAlignment() << "\n"; + dbgs() << " Bumped block pointer to " + << (const void *)BlockDataPtr << " to meet block alignment " + << B->getAlignment() << " and alignment offset " + << B->getAlignmentOffset() << "\n"; }); - for (auto *DA : SI.Atoms) { + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastBlockEnd != BlockDataPtr) + dbgs() << " Zero padding from " << (const void *)LastBlockEnd + << " to " << (const void *)BlockDataPtr << "\n"; + }); - // Align. - AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); - LLVM_DEBUG({ - dbgs() << " Bumped atom pointer to " - << (const void *)AtomDataPtr << " to meet alignment of " - << DA->getAlignment() << "\n"; - }); + while (LastBlockEnd != BlockDataPtr) + *LastBlockEnd++ = 0; - // Zero pad up to alignment. - LLVM_DEBUG({ - if (LastAtomEnd != AtomDataPtr) - dbgs() << " Zero padding from " << (const void *)LastAtomEnd - << " to " << (const void *)AtomDataPtr << "\n"; - }); - while (LastAtomEnd != AtomDataPtr) - *LastAtomEnd++ = 0; + // Copy initial block content. + LLVM_DEBUG({ + dbgs() << " Copying block " << *B << " content, " + << B->getContent().size() << " bytes, from " + << (const void *)B->getContent().data() << " to " + << (const void *)BlockDataPtr << "\n"; + }); + memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); - // Copy initial atom content. - LLVM_DEBUG({ - dbgs() << " Copying atom " << *DA << " content, " - << DA->getContent().size() << " bytes, from " - << (const void *)DA->getContent().data() << " to " - << (const void *)AtomDataPtr << "\n"; - }); - memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : B->edges()) { - // Copy atom data and apply fixups. - LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - for (auto &E : DA->edges()) { + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; - // Skip non-relocation edges. - if (!E.isRelocation()) - continue; - - // Dispatch to LinkerImpl for fixup. - if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) - return Err; - } - - // Point the atom's content to the fixed up buffer. - DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); - - // Update atom end pointer. - LastAtomEnd = AtomDataPtr + DA->getContent().size(); - AtomDataPtr = LastAtomEnd; + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*B, E, BlockDataPtr)) + return Err; } + + // Point the block's content to the fixed up buffer. + B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + + // Update block end pointer. + LastBlockEnd = BlockDataPtr + B->getContent().size(); + BlockDataPtr = LastBlockEnd; } // Zero pad the rest of the segment. LLVM_DEBUG({ dbgs() << " Zero padding end of segment from " - << (const void *)LastAtomEnd << " to " + << (const void *)LastBlockEnd << " to " << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; }); - while (LastAtomEnd != SegMem.data() + SegMem.size()) - *LastAtomEnd++ = 0; + while (LastBlockEnd != SegMem.data() + SegMem.size()) + *LastBlockEnd++ = 0; } return Error::success(); } }; -/// Dead strips and replaces discarded definitions with external atoms. +/// Removes dead symbols/blocks/addressables. /// -/// Finds the set of nodes reachable from any node initially marked live -/// (nodes marked should-discard are treated as not live, even if they are -/// reachable). All nodes not marked as live at the end of this process, -/// are deleted. Nodes that are live, but marked should-discard are replaced -/// with external atoms and all edges to them are re-written. -void prune(AtomGraph &G); - -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); +/// Finds the set of symbols and addressables reachable from any symbol +/// initially marked live. All symbols/addressables not marked live at the end +/// of this process are removed. +void prune(LinkGraph &G); } // end namespace jitlink } // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index 267307cfde0..ecc6793bbce 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -61,6 +61,10 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { AllocationMap SegBlocks; }; + if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate())) + return make_error("Page size is not a power of 2", + inconvertibleErrorCode()); + AllocationMap Blocks; const sys::Memory::ProtectionFlags ReadWrite = static_cast(sys::Memory::MF_READ | @@ -69,19 +73,12 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { for (auto &KV : Request) { auto &Seg = KV.second; - if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate()) + if (Seg.getAlignment() > sys::Process::getPageSizeEstimate()) return make_error("Cannot request higher than page " "alignment", inconvertibleErrorCode()); - if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0) - return make_error("Page size is not a multiple of " - "alignment", - inconvertibleErrorCode()); - - uint64_t ZeroFillStart = - alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); - uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + uint64_t SegmentSize = Seg.getContentSize() + Seg.getZeroFillSize(); std::error_code EC; auto SegMem = @@ -91,7 +88,7 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { return errorCodeToError(EC); // Zero out the zero-fill memory. - memset(static_cast(SegMem.base()) + ZeroFillStart, 0, + memset(static_cast(SegMem.base()) + Seg.getContentSize(), 0, Seg.getZeroFillSize()); // Record the block for this segment. diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp deleted file mode 100644 index c1040c942b2..00000000000 --- a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp +++ /dev/null @@ -1,412 +0,0 @@ -//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Generic MachO AtomGraph buliding code. -// -//===----------------------------------------------------------------------===// - -#include "MachOAtomGraphBuilder.h" - -#define DEBUG_TYPE "jitlink" - -namespace llvm { -namespace jitlink { - -MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} - -Expected> MachOAtomGraphBuilder::buildGraph() { - if (auto Err = parseSections()) - return std::move(Err); - - if (auto Err = addAtoms()) - return std::move(Err); - - if (auto Err = addRelocations()) - return std::move(Err); - - return std::move(G); -} - -MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) - : Obj(Obj), - G(std::make_unique(Obj.getFileName(), getPointerSize(Obj), - getEndianness(Obj))) {} - -void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, - CustomAtomizeFunction Atomizer) { - assert(!CustomAtomizeFunctions.count(SectionName) && - "Custom atomizer for this section already exists"); - CustomAtomizeFunctions[SectionName] = std::move(Atomizer); -} - -bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) { - // If these atoms are the same then they're trivially "locked". - if (&A == &B) - return true; - - // If A and B are different, check whether either is undefined. (in which - // case they are not locked). - if (!A.isDefined() || !B.isDefined()) - return false; - - // A and B are different, but they're both defined atoms. We need to check - // whether they're part of the same alt_entry chain. - auto &DA = static_cast(A); - auto &DB = static_cast(B); - - auto AStartItr = AltEntryStarts.find(&DA); - if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out. - return false; - - auto BStartItr = AltEntryStarts.find(&DB); - if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out. - return false; - - // A and B are layout locked if they're in the same chain. - return AStartItr->second == BStartItr->second; -} - -unsigned -MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { - return Obj.is64Bit() ? 8 : 4; -} - -support::endianness -MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { - return Obj.isLittleEndian() ? support::little : support::big; -} - -MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { - if (!CommonSymbolsSection) { - auto Prot = static_cast( - sys::Memory::MF_READ | sys::Memory::MF_WRITE); - auto &GenericSection = G->createSection("", 1, Prot, true); - CommonSymbolsSection = MachOSection(GenericSection); - } - return *CommonSymbolsSection; -} - -Error MachOAtomGraphBuilder::parseSections() { - for (auto &SecRef : Obj.sections()) { - assert((SecRef.getAlignment() <= std::numeric_limits::max()) && - "Section alignment does not fit in 32 bits"); - - Expected NameOrErr = SecRef.getName(); - if (!NameOrErr) - return NameOrErr.takeError(); - StringRef Name = *NameOrErr; - - unsigned SectionIndex = SecRef.getIndex() + 1; - - uint32_t Align = SecRef.getAlignment(); - if (!isPowerOf2_32(Align)) - return make_error("Section " + Name + - " has non-power-of-2 " - "alignment"); - - // FIXME: Get real section permissions - // How, exactly, on MachO? - sys::Memory::ProtectionFlags Prot; - if (SecRef.isText()) - Prot = static_cast(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - else - Prot = static_cast(sys::Memory::MF_READ | - sys::Memory::MF_WRITE); - - auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS()); - - LLVM_DEBUG({ - dbgs() << "Adding section " << Name << ": " - << format("0x%016" PRIx64, SecRef.getAddress()) - << ", align: " << SecRef.getAlignment() << "\n"; - }); - - assert(!Sections.count(SectionIndex) && "Section index already in use"); - - auto &MachOSec = - Sections - .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(), - SecRef.getAlignment()) - .first->second; - - if (!SecRef.isVirtual()) { - // If this section has content then record it. - Expected Content = SecRef.getContents(); - if (!Content) - return Content.takeError(); - if (Content->size() != SecRef.getSize()) - return make_error("Section content size does not match " - "declared size for " + - Name); - MachOSec.setContent(*Content); - } else { - // If this is a zero-fill section then just record the size. - MachOSec.setZeroFill(SecRef.getSize()); - } - - uint32_t SectionFlags = - Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags - : Obj.getSection(SecRef.getRawDataRefImpl()).flags; - - MachOSec.setNoDeadStrip(SectionFlags & MachO::S_ATTR_NO_DEAD_STRIP); - } - - return Error::success(); -} - -// Adds atoms with identified start addresses (but not lengths) for all named -// atoms. -// Also, for every section that contains named atoms, but does not have an -// atom at offset zero of that section, constructs an anonymous atom covering -// that range. -Error MachOAtomGraphBuilder::addNonCustomAtoms() { - using AddrToAtomMap = std::map; - DenseMap SecToAtoms; - - DenseMap FirstOrdinal; - std::vector AltEntryAtoms; - - DenseSet ProcessedSymbols; // Used to check for duplicate defs. - - for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; - ++SymI) { - object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); - - auto Name = Sym.getName(); - if (!Name) - return Name.takeError(); - - // Bail out on duplicate definitions: There should never be more than one - // definition for a symbol in a given object file. - if (ProcessedSymbols.count(*Name)) - return make_error("Duplicate definition within object: " + - *Name); - else - ProcessedSymbols.insert(*Name); - - auto Addr = Sym.getAddress(); - if (!Addr) - return Addr.takeError(); - - auto SymType = Sym.getType(); - if (!SymType) - return SymType.takeError(); - - auto Flags = Sym.getFlags(); - - if (Flags & object::SymbolRef::SF_Undefined) { - LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); - G->addExternalAtom(*Name); - continue; - } else if (Flags & object::SymbolRef::SF_Absolute) { - LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " - << format("0x%016" PRIx64, *Addr) << "\n"); - auto &A = G->addAbsoluteAtom(*Name, *Addr); - A.setGlobal(Flags & object::SymbolRef::SF_Global); - A.setExported(Flags & object::SymbolRef::SF_Exported); - A.setWeak(Flags & object::SymbolRef::SF_Weak); - continue; - } else if (Flags & object::SymbolRef::SF_Common) { - LLVM_DEBUG({ - dbgs() << "Adding common \"" << *Name - << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; - }); - auto &A = - G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, - std::max(Sym.getAlignment(), 1U), - Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); - A.setGlobal(Flags & object::SymbolRef::SF_Global); - A.setExported(Flags & object::SymbolRef::SF_Exported); - continue; - } - - LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); - - // This atom is neither undefined nor absolute, so it must be defined in - // this object. Get its section index. - auto SecItr = Sym.getSection(); - if (!SecItr) - return SecItr.takeError(); - - uint64_t SectionIndex = (*SecItr)->getIndex() + 1; - - LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); - - auto SecByIndexItr = Sections.find(SectionIndex); - if (SecByIndexItr == Sections.end()) - return make_error("Unrecognized section index in macho"); - - auto &Sec = SecByIndexItr->second; - - auto &DA = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, - std::max(Sym.getAlignment(), 1U)); - - DA.setGlobal(Flags & object::SymbolRef::SF_Global); - DA.setExported(Flags & object::SymbolRef::SF_Exported); - DA.setWeak(Flags & object::SymbolRef::SF_Weak); - - DA.setCallable(*SymType & object::SymbolRef::ST_Function); - - // Check NDesc flags. - { - uint16_t NDesc = 0; - if (Obj.is64Bit()) - NDesc = Obj.getSymbol64TableEntry(SymI->getRawDataRefImpl()).n_desc; - else - NDesc = Obj.getSymbolTableEntry(SymI->getRawDataRefImpl()).n_desc; - - // Record atom for alt-entry post-processing (where the layout-next - // constraints will be added). - if (NDesc & MachO::N_ALT_ENTRY) - AltEntryAtoms.push_back(&DA); - - // If this atom has a no-dead-strip attr attached then mark it live. - if (NDesc & MachO::N_NO_DEAD_STRIP) - DA.setLive(true); - } - - LLVM_DEBUG({ - dbgs() << " Added " << *Name - << " addr: " << format("0x%016" PRIx64, *Addr) - << ", align: " << DA.getAlignment() - << ", section: " << Sec.getGenericSection().getName() << "\n"; - }); - - auto &SecAtoms = SecToAtoms[&Sec]; - SecAtoms[DA.getAddress() - Sec.getAddress()] = &DA; - } - - // Add anonymous atoms. - for (auto &KV : Sections) { - auto &S = KV.second; - - // Skip empty sections. - if (S.empty()) - continue; - - // Skip sections with custom handling. - if (CustomAtomizeFunctions.count(S.getName())) - continue; - - auto SAI = SecToAtoms.find(&S); - - // If S is not in the SecToAtoms map then it contained no named atom. Add - // one anonymous atom to cover the whole section. - if (SAI == SecToAtoms.end()) { - SecToAtoms[&S][0] = &G->addAnonymousAtom( - S.getGenericSection(), S.getAddress(), S.getAlignment()); - continue; - } - - // Otherwise, check whether this section had an atom covering offset zero. - // If not, add one. - auto &SecAtoms = SAI->second; - if (!SecAtoms.count(0)) - SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), - S.getAlignment()); - } - - LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); - - // Set atom contents and any section-based flags. - for (auto &KV : SecToAtoms) { - auto &S = *KV.first; - auto &SecAtoms = KV.second; - - // Iterate the atoms in reverse order and set up their contents. - JITTargetAddress LastAtomAddr = S.getSize(); - for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { - auto Offset = I->first; - auto &A = *I->second; - LLVM_DEBUG({ - dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " - << S.getAddress() + LastAtomAddr << " ]\n"; - }); - - if (S.isZeroFill()) - A.setZeroFill(LastAtomAddr - Offset); - else - A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); - - // If the section has no-dead-strip set then mark the atom as live. - if (S.isNoDeadStrip()) - A.setLive(true); - - LastAtomAddr = Offset; - } - } - - LLVM_DEBUG(dbgs() << "Adding alt-entry starts\n"); - - // Sort alt-entry atoms by address in ascending order. - llvm::sort(AltEntryAtoms.begin(), AltEntryAtoms.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); - - // Process alt-entry atoms in address order to build the table of alt-entry - // atoms to alt-entry chain starts. - for (auto *DA : AltEntryAtoms) { - assert(!AltEntryStarts.count(DA) && "Duplicate entry in AltEntryStarts"); - - // DA is an alt-entry atom. Look for the predecessor atom that it is locked - // to, bailing out if we do not find one. - auto AltEntryPred = G->findAtomByAddress(DA->getAddress() - 1); - if (!AltEntryPred) - return AltEntryPred.takeError(); - - // Add a LayoutNext edge from the predecessor to this atom. - AltEntryPred->setLayoutNext(*DA); - - // Check to see whether the predecessor itself is an alt-entry atom. - auto AltEntryStartItr = AltEntryStarts.find(&*AltEntryPred); - if (AltEntryStartItr != AltEntryStarts.end()) { - // If the predecessor was an alt-entry atom then re-use its value. - LLVM_DEBUG({ - dbgs() << " " << *DA << " -> " << *AltEntryStartItr->second - << " (based on existing entry for " << *AltEntryPred << ")\n"; - }); - AltEntryStarts[DA] = AltEntryStartItr->second; - } else { - // If the predecessor does not have an entry then add an entry for this - // atom (i.e. the alt_entry atom) and a self-reference entry for the - /// predecessory atom that is the start of this chain. - LLVM_DEBUG({ - dbgs() << " " << *AltEntryPred << " -> " << *AltEntryPred << "\n" - << " " << *DA << " -> " << *AltEntryPred << "\n"; - }); - AltEntryStarts[&*AltEntryPred] = &*AltEntryPred; - AltEntryStarts[DA] = &*AltEntryPred; - } - } - - return Error::success(); -} - -Error MachOAtomGraphBuilder::addAtoms() { - // Add all named atoms. - if (auto Err = addNonCustomAtoms()) - return Err; - - // Process special sections. - for (auto &KV : Sections) { - auto &S = KV.second; - auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); - if (HI != CustomAtomizeFunctions.end()) { - auto &Atomize = HI->second; - if (auto Err = Atomize(S)) - return Err; - } - } - - return Error::success(); -} - -} // end namespace jitlink -} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h deleted file mode 100644 index 72d441b24d0..00000000000 --- a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h +++ /dev/null @@ -1,138 +0,0 @@ -//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Generic MachO AtomGraph building code. -// -//===----------------------------------------------------------------------===// - -#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H -#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H - -#include "llvm/ExecutionEngine/JITLink/JITLink.h" - -#include "JITLinkGeneric.h" - -#include "llvm/Object/MachO.h" - -namespace llvm { -namespace jitlink { - -class MachOAtomGraphBuilder { -public: - virtual ~MachOAtomGraphBuilder(); - Expected> buildGraph(); - -protected: - using OffsetToAtomMap = std::map; - - class MachOSection { - public: - MachOSection() = default; - - /// Create a MachO section with the given address and alignment. - MachOSection(Section &GenericSection, JITTargetAddress Address, - unsigned Alignment) - : Address(Address), GenericSection(&GenericSection), - Alignment(Alignment) {} - - /// Create a section without address, content or size (used for common - /// symbol sections). - MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} - - Section &getGenericSection() const { - assert(GenericSection && "Section is null"); - return *GenericSection; - } - - StringRef getName() const { - assert(GenericSection && "No generic section attached"); - return GenericSection->getName(); - } - - MachOSection &setContent(StringRef Content) { - assert(!ContentPtr && !Size && "Content/zeroFill already set"); - ContentPtr = Content.data(); - Size = Content.size(); - return *this; - } - - MachOSection &setZeroFill(uint64_t Size) { - assert(!ContentPtr && !this->Size && "Content/zeroFill already set"); - this->Size = Size; - return *this; - } - - bool isZeroFill() const { return !ContentPtr; } - - bool empty() const { return getSize() == 0; } - - size_t getSize() const { return Size; } - - StringRef getContent() const { - assert(ContentPtr && "getContent() called on zero-fill section"); - return {ContentPtr, static_cast(Size)}; - } - - JITTargetAddress getAddress() const { return Address; } - - unsigned getAlignment() const { return Alignment; } - - MachOSection &setNoDeadStrip(bool NoDeadStrip) { - this->NoDeadStrip = NoDeadStrip; - return *this; - } - - bool isNoDeadStrip() const { return NoDeadStrip; } - - private: - JITTargetAddress Address = 0; - Section *GenericSection = nullptr; - const char *ContentPtr = nullptr; - uint64_t Size = 0; - unsigned Alignment = 0; - bool NoDeadStrip = false; - }; - - using CustomAtomizeFunction = std::function; - - MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); - - AtomGraph &getGraph() const { return *G; } - - const object::MachOObjectFile &getObject() const { return Obj; } - - void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); - - virtual Error addRelocations() = 0; - - /// Returns true if Atom A and Atom B are at a fixed offset from one another - /// (i.e. if they're part of the same alt-entry chain). - bool areLayoutLocked(const Atom &A, const Atom &B); - -private: - static unsigned getPointerSize(const object::MachOObjectFile &Obj); - static support::endianness getEndianness(const object::MachOObjectFile &Obj); - - MachOSection &getCommonSection(); - - Error parseSections(); - Error addNonCustomAtoms(); - Error addAtoms(); - - const object::MachOObjectFile &Obj; - std::unique_ptr G; - DenseMap AltEntryStarts; - DenseMap Sections; - StringMap CustomAtomizeFunctions; - Optional CommonSymbolsSection; -}; - -} // end namespace jitlink -} // end namespace llvm - -#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H diff --git a/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp new file mode 100644 index 00000000000..57fbc699757 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -0,0 +1,535 @@ +//=--------- MachOLinkGraphBuilder.cpp - MachO LinkGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +static const char *CommonSectionName = "__common"; + +namespace llvm { +namespace jitlink { + +MachOLinkGraphBuilder::~MachOLinkGraphBuilder() {} + +Expected> MachOLinkGraphBuilder::buildGraph() { + + // Sanity check: we only operate on relocatable objects. + if (!Obj.isRelocatableObject()) + return make_error("Object is not a relocatable MachO"); + + if (auto Err = createNormalizedSections()) + return std::move(Err); + + if (auto Err = createNormalizedSymbols()) + return std::move(Err); + + if (auto Err = graphifyRegularSymbols()) + return std::move(Err); + + if (auto Err = graphifySectionsWithCustomParsers()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(std::make_unique(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOLinkGraphBuilder::addCustomSectionParser( + StringRef SectionName, SectionParserFunction Parser) { + assert(!CustomSectionParserFunctions.count(SectionName) && + "Custom parser for this section already exists"); + CustomSectionParserFunctions[SectionName] = std::move(Parser); +} + +Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) { + if ((Desc & MachO::N_WEAK_DEF) || (Desc & MachO::N_WEAK_REF)) + return Linkage::Weak; + return Linkage::Strong; +} + +Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) { + if (Name.startswith("l")) + return Scope::Local; + if (Type & MachO::N_PEXT) + return Scope::Hidden; + if (Type & MachO::N_EXT) + return Scope::Default; + return Scope::Local; +} + +bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) { + return NSym.Desc & MachO::N_ALT_ENTRY; +} + +unsigned +MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +Section &MachOLinkGraphBuilder::getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; +} + +Error MachOLinkGraphBuilder::createNormalizedSections() { + // Build normalized sections. Verifies that section data is in-range (for + // sections with content) and that address ranges are non-overlapping. + + LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); + + for (auto &SecRef : Obj.sections()) { + NormalizedSection NSec; + uint32_t DataOffset = 0; + + auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl()); + + auto Name = SecRef.getName(); + if (!Name) + return Name.takeError(); + + if (Obj.is64Bit()) { + const MachO::section_64 &Sec64 = + Obj.getSection64(SecRef.getRawDataRefImpl()); + + NSec.Address = Sec64.addr; + NSec.Size = Sec64.size; + NSec.Alignment = 1ULL << Sec64.align; + NSec.Flags = Sec64.flags; + DataOffset = Sec64.offset; + } else { + const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl()); + NSec.Address = Sec32.addr; + NSec.Size = Sec32.size; + NSec.Alignment = 1ULL << Sec32.align; + NSec.Flags = Sec32.flags; + DataOffset = Sec32.offset; + } + + LLVM_DEBUG({ + dbgs() << " " << *Name << ": " << formatv("{0:x16}", NSec.Address) + << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size) + << ", align: " << NSec.Alignment << ", index: " << SecIndex + << "\n"; + }); + + // Get the section data if any. + { + unsigned SectionType = NSec.Flags & MachO::SECTION_TYPE; + if (SectionType != MachO::S_ZEROFILL && + SectionType != MachO::S_GB_ZEROFILL) { + + if (DataOffset + NSec.Size > Obj.getData().size()) + return make_error( + "Section data extends past end of file"); + + NSec.Data = Obj.getData().data() + DataOffset; + } + } + + // Get prot flags. + // FIXME: Make sure this test is correct (it's probably missing cases + // as-is). + sys::Memory::ProtectionFlags Prot; + if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS) + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + NSec.GraphSection = &G->createSection(*Name, Prot); + IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); + } + + std::vector Sections; + Sections.reserve(IndexToSection.size()); + for (auto &KV : IndexToSection) + Sections.push_back(&KV.second); + + // If we didn't end up creating any sections then bail out. The code below + // assumes that we have at least one section. + if (Sections.empty()) + return Error::success(); + + llvm::sort(Sections, + [](const NormalizedSection *LHS, const NormalizedSection *RHS) { + assert(LHS && RHS && "Null section?"); + return LHS->Address < RHS->Address; + }); + + for (unsigned I = 0, E = Sections.size() - 1; I != E; ++I) { + auto &Cur = *Sections[I]; + auto &Next = *Sections[I + 1]; + if (Next.Address < Cur.Address + Cur.Size) + return make_error( + "Address range for section " + Cur.GraphSection->getName() + + formatv(" [ {0:x16} -- {1:x16} ] ", Cur.Address, + Cur.Address + Cur.Size) + + "overlaps " + + formatv(" [ {0:x16} -- {1:x16} ] ", Next.Address, + Next.Address + Next.Size)); + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::createNormalizedSymbols() { + LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); + + for (auto &SymRef : Obj.symbols()) { + + unsigned SymbolIndex = Obj.getSymbolIndex(SymRef.getRawDataRefImpl()); + uint64_t Value; + uint32_t NStrX; + uint8_t Type; + uint8_t Sect; + uint16_t Desc; + + if (Obj.is64Bit()) { + const MachO::nlist_64 &NL64 = + Obj.getSymbol64TableEntry(SymRef.getRawDataRefImpl()); + Value = NL64.n_value; + NStrX = NL64.n_strx; + Type = NL64.n_type; + Sect = NL64.n_sect; + Desc = NL64.n_desc; + } else { + const MachO::nlist &NL32 = + Obj.getSymbolTableEntry(SymRef.getRawDataRefImpl()); + Value = NL32.n_value; + NStrX = NL32.n_strx; + Type = NL32.n_type; + Sect = NL32.n_sect; + Desc = NL32.n_desc; + } + + // Skip stabs. + // FIXME: Are there other symbols we should be skipping? + if (Type & MachO::N_STAB) + continue; + + Optional Name; + if (NStrX) { + if (auto NameOrErr = SymRef.getName()) + Name = *NameOrErr; + else + return NameOrErr.takeError(); + } + + LLVM_DEBUG({ + dbgs() << " "; + if (!Name) + dbgs() << ""; + else + dbgs() << *Name; + dbgs() << ": value = " << formatv("{0:x16}", Value) + << ", type = " << formatv("{0:x2}", Type) + << ", desc = " << formatv("{0:x4}", Desc) << ", sect = "; + if (Sect) + dbgs() << static_cast(Sect - 1); + else + dbgs() << "none"; + dbgs() << "\n"; + }); + + // If this symbol has a section, sanity check that the addresses line up. + NormalizedSection *NSec = nullptr; + if (Sect != 0) { + if (auto NSecOrErr = findSectionByIndex(Sect - 1)) + NSec = &*NSecOrErr; + else + return NSecOrErr.takeError(); + + if (Value < NSec->Address || Value > NSec->Address + NSec->Size) + return make_error("Symbol address does not fall within " + "section"); + } + + IndexToSymbol[SymbolIndex] = + &createNormalizedSymbol(*Name, Value, Type, Sect, Desc, + getLinkage(Type), getScope(*Name, Type)); + } + + return Error::success(); +} + +void MachOLinkGraphBuilder::addSectionStartSymAndBlock( + Section &GraphSec, uint64_t Address, const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive) { + Block &B = + Data ? G->createContentBlock(GraphSec, StringRef(Data, Size), Address, + Alignment, 0) + : G->createZeroFillBlock(GraphSec, Size, Address, Alignment, 0); + auto &Sym = G->addAnonymousSymbol(B, 0, Size, false, IsLive); + assert(!AddrToCanonicalSymbol.count(Sym.getAddress()) && + "Anonymous block start symbol clashes with existing symbol address"); + AddrToCanonicalSymbol[Sym.getAddress()] = &Sym; +} + +Error MachOLinkGraphBuilder::graphifyRegularSymbols() { + + LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); + + /// We only have 256 section indexes: Use a vector rather than a map. + std::vector> SecIndexToSymbols; + SecIndexToSymbols.resize(256); + + // Create commons, externs, and absolutes, and partition all other symbols by + // section. + for (auto &KV : IndexToSymbol) { + auto &NSym = *KV.second; + + switch (NSym.Type & MachO::N_TYPE) { + case MachO::N_UNDF: + if (NSym.Value) { + if (!NSym.Name) + return make_error("Anonymous common symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addCommonSymbol( + *NSym.Name, NSym.S, getCommonSection(), NSym.Value, 0, + 1U << MachO::GET_COMM_ALIGN(NSym.Desc), + NSym.Desc & MachO::N_NO_DEAD_STRIP); + } else { + if (!NSym.Name) + return make_error("Anonymous external symbol at " + "index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addExternalSymbol(*NSym.Name, 0); + } + break; + case MachO::N_ABS: + if (!NSym.Name) + return make_error("Anonymous absolute symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addAbsoluteSymbol( + *NSym.Name, NSym.Value, 0, Linkage::Strong, Scope::Default, + NSym.Desc & MachO::N_NO_DEAD_STRIP); + break; + case MachO::N_SECT: + SecIndexToSymbols[NSym.Sect - 1].push_back(&NSym); + break; + case MachO::N_PBUD: + return make_error( + "Unupported N_PBUD symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("")) + + " at index " + Twine(KV.first)); + case MachO::N_INDR: + return make_error( + "Unupported N_INDR symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("")) + + " at index " + Twine(KV.first)); + default: + return make_error( + "Unrecognized symbol type " + Twine(NSym.Type & MachO::N_TYPE) + + " for symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("")) + + " at index " + Twine(KV.first)); + } + } + + // Loop over sections performing regular graphification for those that + // don't have custom parsers. + for (auto &KV : IndexToSection) { + auto SecIndex = KV.first; + auto &NSec = KV.second; + + // Skip sections with custom parsers. + if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) { + LLVM_DEBUG({ + dbgs() << " Skipping section " << NSec.GraphSection->getName() + << " as it has a custom parser.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " Processing section " << NSec.GraphSection->getName() + << "...\n"; + }); + + bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; + bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS; + + auto &SecNSymStack = SecIndexToSymbols[SecIndex]; + + // If this section is non-empty but there are no symbols covering it then + // create one block and anonymous symbol to cover the entire section. + if (SecNSymStack.empty()) { + if (NSec.Size > 0) { + LLVM_DEBUG({ + dbgs() << " Section non-empty, but contains no symbols. " + "Creating anonymous block to cover " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + NSec.Size) << "\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + NSec.Size, NSec.Alignment, + SectionIsNoDeadStrip); + } else + LLVM_DEBUG({ + dbgs() << " Section empty and contains no symbols. Skipping.\n"; + }); + continue; + } + + // Sort the symbol stack in by address, alt-entry status, scope, and name. + // We sort in reverse order so that symbols will be visited in the right + // order when we pop off the stack below. + llvm::sort(SecNSymStack, [](const NormalizedSymbol *LHS, + const NormalizedSymbol *RHS) { + if (LHS->Value != RHS->Value) + return LHS->Value > RHS->Value; + if (isAltEntry(*LHS) != isAltEntry(*RHS)) + return isAltEntry(*RHS); + if (LHS->S != RHS->S) + return static_cast(LHS->S) < static_cast(RHS->S); + return LHS->Name < RHS->Name; + }); + + // The first symbol in a section can not be an alt-entry symbol. + if (!SecNSymStack.empty() && isAltEntry(*SecNSymStack.back())) + return make_error( + "First symbol in " + NSec.GraphSection->getName() + " is alt-entry"); + + // If the section is non-empty but there is no symbol covering the start + // address then add an anonymous one. + if (SecNSymStack.back()->Value != NSec.Address) { + auto AnonBlockSize = SecNSymStack.back()->Value - NSec.Address; + LLVM_DEBUG({ + dbgs() << " Section start not covered by symbol. " + << "Creating anonymous block to cover [ " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + AnonBlockSize) << " ]\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + AnonBlockSize, NSec.Alignment, + SectionIsNoDeadStrip); + } + + // Visit section symbols in order by popping off the reverse-sorted stack, + // building blocks for each alt-entry chain and creating symbols as we go. + while (!SecNSymStack.empty()) { + SmallVector BlockSyms; + + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + while (!SecNSymStack.empty() && + (isAltEntry(*SecNSymStack.back()) || + SecNSymStack.back()->Value == BlockSyms.back()->Value)) { + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + } + + // BlockNSyms now contains the block symbols in reverse canonical order. + JITTargetAddress BlockStart = BlockSyms.front()->Value; + JITTargetAddress BlockEnd = SecNSymStack.empty() + ? NSec.Address + NSec.Size + : SecNSymStack.back()->Value; + JITTargetAddress BlockOffset = BlockStart - NSec.Address; + JITTargetAddress BlockSize = BlockEnd - BlockStart; + + LLVM_DEBUG({ + dbgs() << " Creating block for " << formatv("{0:x16}", BlockStart) + << " -- " << formatv("{0:x16}", BlockEnd) << ": " + << NSec.GraphSection->getName() << " + " + << formatv("{0:x16}", BlockOffset) << " with " + << BlockSyms.size() << " symbol(s)...\n"; + }); + + Block &B = + NSec.Data + ? G->createContentBlock( + *NSec.GraphSection, + StringRef(NSec.Data + BlockOffset, BlockSize), BlockStart, + NSec.Alignment, BlockStart % NSec.Alignment) + : G->createZeroFillBlock(*NSec.GraphSection, BlockSize, + BlockStart, NSec.Alignment, + BlockStart % NSec.Alignment); + + Optional LastCanonicalAddr; + JITTargetAddress SymEnd = BlockEnd; + while (!BlockSyms.empty()) { + auto &NSym = *BlockSyms.back(); + BlockSyms.pop_back(); + + bool SymLive = + (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; + + LLVM_DEBUG({ + dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- " + << formatv("{0:x16}", SymEnd) << ": "; + if (!NSym.Name) + dbgs() << ""; + else + dbgs() << NSym.Name; + if (SymLive) + dbgs() << " [no-dead-strip]"; + if (LastCanonicalAddr == NSym.Value) + dbgs() << " [non-canonical]"; + dbgs() << "\n"; + }); + + auto &Sym = + NSym.Name + ? G->addDefinedSymbol(B, NSym.Value - BlockStart, *NSym.Name, + SymEnd - NSym.Value, NSym.L, NSym.S, + SectionIsText, SymLive) + : G->addAnonymousSymbol(B, NSym.Value - BlockStart, + SymEnd - NSym.Value, SectionIsText, + SymLive); + NSym.GraphSymbol = &Sym; + if (LastCanonicalAddr != Sym.getAddress()) { + if (LastCanonicalAddr) + SymEnd = *LastCanonicalAddr; + LastCanonicalAddr = Sym.getAddress(); + setCanonicalSymbol(Sym); + } + } + } + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { + // Graphify special sections. + for (auto &KV : IndexToSection) { + auto &NSec = KV.second; + + auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName()); + if (HI != CustomSectionParserFunctions.end()) { + auto &Parse = HI->second; + if (auto Err = Parse(NSec)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h new file mode 100644 index 00000000000..cba26e9494b --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -0,0 +1,269 @@ +//===----- MachOLinkGraphBuilder.h - MachO LinkGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "EHFrameSupportImpl.h" +#include "JITLinkGeneric.h" +#include "llvm/Object/MachO.h" + +#include + +namespace llvm { +namespace jitlink { + +class MachOLinkGraphBuilder { +public: + virtual ~MachOLinkGraphBuilder(); + Expected> buildGraph(); + +protected: + class MachOEHFrameBinaryParser : public EHFrameBinaryParser { + public: + MachOEHFrameBinaryParser(MachOLinkGraphBuilder &Builder, + JITTargetAddress EHFrameAddress, + StringRef EHFrameContent, Section &EHFrameSection, + uint64_t CIEAlignment, uint64_t FDEAlignment, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : EHFrameBinaryParser(EHFrameAddress, EHFrameContent, + Builder.getGraph().getPointerSize(), + Builder.getGraph().getEndianness()), + Builder(Builder), EHFrameSection(EHFrameSection), + CIEAlignment(CIEAlignment), FDEAlignment(FDEAlignment), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + + Symbol *getSymbolAtAddress(JITTargetAddress Address) override { + if (auto *Sym = Builder.getSymbolByAddress(Address)) + if (Sym->getAddress() == Address) + return Sym; + return nullptr; + } + + Symbol &createCIERecord(JITTargetAddress RecordAddr, + StringRef RecordContent) override { + auto &G = Builder.getGraph(); + auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, + CIEAlignment, 0); + auto &CIESymbol = + G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); + Builder.setCanonicalSymbol(CIESymbol); + return CIESymbol; + } + + Expected createFDERecord(JITTargetAddress RecordAddr, + StringRef RecordContent, Symbol &CIE, + size_t CIEOffset, Symbol &Func, + size_t FuncOffset, Symbol *LSDA, + size_t LSDAOffset) override { + auto &G = Builder.getGraph(); + auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, + FDEAlignment, 0); + + // Add edges to CIE, Func, and (conditionally) LSDA. + B.addEdge(FDEToCIERelocKind, CIEOffset, CIE, 0); + B.addEdge(FDEToTargetRelocKind, FuncOffset, Func, 0); + + if (LSDA) + B.addEdge(FDEToTargetRelocKind, LSDAOffset, *LSDA, 0); + + auto &FDESymbol = + G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); + + // Add a keep-alive relocation from the function to the FDE to ensure it + // is not dead stripped. + Func.getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + + return FDESymbol; + } + + private: + MachOLinkGraphBuilder &Builder; + Section &EHFrameSection; + uint64_t CIEAlignment; + uint64_t FDEAlignment; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; + }; + + struct NormalizedSymbol { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSymbol(Optional Name, uint64_t Value, uint8_t Type, + uint8_t Sect, uint16_t Desc, Linkage L, Scope S) + : Name(Name), Value(Value), Type(Type), Sect(Sect), Desc(Desc), L(L), + S(S) { + assert(!Name || !Name->empty() && "Name must be none or non-empty"); + } + + public: + NormalizedSymbol(const NormalizedSymbol &) = delete; + NormalizedSymbol &operator=(const NormalizedSymbol &) = delete; + NormalizedSymbol(NormalizedSymbol &&) = delete; + NormalizedSymbol &operator=(NormalizedSymbol &&) = delete; + + Optional Name; + uint64_t Value = 0; + uint8_t Type = 0; + uint8_t Sect = 0; + uint16_t Desc = 0; + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + Symbol *GraphSymbol = nullptr; + }; + + class NormalizedSection { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSection() = default; + + public: + Section *GraphSection = nullptr; + uint64_t Address = 0; + uint64_t Size = 0; + uint64_t Alignment = 0; + uint32_t Flags = 0; + const char *Data = nullptr; + }; + + using SectionParserFunction = std::function; + + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj); + + LinkGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomSectionParser(StringRef SectionName, + SectionParserFunction Parse); + + virtual Error addRelocations() = 0; + + /// Create a symbol. + template + NormalizedSymbol &createNormalizedSymbol(ArgTs &&... Args) { + NormalizedSymbol *Sym = reinterpret_cast( + Allocator.Allocate()); + new (Sym) NormalizedSymbol(std::forward(Args)...); + return *Sym; + } + + /// Index is zero-based (MachO section indexes are usually one-based) and + /// assumed to be in-range. Client is responsible for checking. + NormalizedSection &getSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + assert(I != IndexToSection.end() && "No section recorded at index"); + return I->second; + } + + /// Try to get the section at the given index. Will return an error if the + /// given index is out of range, or if no section has been added for the given + /// index. + Expected findSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + if (I == IndexToSection.end()) + return make_error("No section recorded for index " + + formatv("{0:u}", Index)); + return I->second; + } + + /// Try to get the symbol at the given index. Will return an error if the + /// given index is out of range, or if no symbol has been added for the given + /// index. + Expected findSymbolByIndex(uint64_t Index) { + if (Index >= IndexToSymbol.size()) + return make_error("Symbol index out of range"); + auto *Sym = IndexToSymbol[Index]; + if (!Sym) + return make_error("No symbol at index " + + formatv("{0:u}", Index)); + return *Sym; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or null if no such symbol exists. + Symbol *getSymbolByAddress(JITTargetAddress Address) { + auto I = AddrToCanonicalSymbol.upper_bound(Address); + if (I == AddrToCanonicalSymbol.begin()) + return nullptr; + return std::prev(I)->second; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or an error if no such symbol exists. + Expected findSymbolByAddress(JITTargetAddress Address) { + auto *Sym = getSymbolByAddress(Address); + if (Sym) + if (Address < Sym->getAddress() + Sym->getSize()) + return *Sym; + return make_error("No symbol covering address " + + formatv("{0:x16}", Address)); + } + + static Linkage getLinkage(uint16_t Desc); + static Scope getScope(StringRef Name, uint8_t Type); + static bool isAltEntry(const NormalizedSymbol &NSym); + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + void setCanonicalSymbol(Symbol &Sym) { + auto *&CanonicalSymEntry = AddrToCanonicalSymbol[Sym.getAddress()]; + // There should be no symbol at this address, or, if there is, + // it should be a zero-sized symbol from an empty section (which + // we can safely override). + assert((!CanonicalSymEntry || CanonicalSymEntry->getSize() == 0) && + "Duplicate canonical symbol at address"); + CanonicalSymEntry = &Sym; + } + + Section &getCommonSection(); + void addSectionStartSymAndBlock(Section &GraphSec, uint64_t Address, + const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive); + + Error createNormalizedSections(); + Error createNormalizedSymbols(); + + /// Create graph blocks and symbols for externals, absolutes, commons and + /// all defined symbols in sections without custom parsers. + Error graphifyRegularSymbols(); + + /// Create graph blocks and symbols for all sections. + Error graphifySectionsWithCustomParsers(); + + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; + + const object::MachOObjectFile &Obj; + std::unique_ptr G; + + DenseMap IndexToSection; + Section *CommonSection = nullptr; + + DenseMap IndexToSymbol; + std::map AddrToCanonicalSymbol; + StringMap CustomSectionParserFunctions; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H diff --git a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 52481f8436e..d83787ffd59 100644 --- a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -13,7 +13,7 @@ #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" #include "BasicGOTAndStubsBuilder.h" -#include "MachOAtomGraphBuilder.h" +#include "MachOLinkGraphBuilder.h" #define DEBUG_TYPE "jitlink" @@ -23,16 +23,21 @@ using namespace llvm::jitlink::MachO_x86_64_Edges; namespace { -class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: - MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOAtomGraphBuilder(Obj), - NumSymbols(Obj.getSymtabLoadCommand().nsyms) { - addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { - return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), - EHFrameSection.getContent(), - EHFrameSection.getAddress(), NegDelta32, Delta64); - }); + MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOLinkGraphBuilder(Obj) { + addCustomSectionParser( + "__eh_frame", [this](NormalizedSection &EHFrameSection) { + if (!EHFrameSection.Data) + return make_error( + "__eh_frame section is marked zero-fill"); + return MachOEHFrameBinaryParser( + *this, EHFrameSection.Address, + StringRef(EHFrameSection.Data, EHFrameSection.Size), + *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) + .addToGraph(); + }); } private: @@ -102,17 +107,6 @@ private: ", length=" + formatv("{0:d}", RI.r_length)); } - Expected findAtomBySymbolIndex(const MachO::relocation_info &RI) { - auto &Obj = getObject(); - if (RI.r_symbolnum >= NumSymbols) - return make_error("Symbol index out of range"); - auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); - auto Name = SymI->getName(); - if (!Name) - return Name.takeError(); - return getGraph().getAtomByName(*Name); - } - MachO::relocation_info getRelocationInfo(const object::relocation_iterator RelItr) { MachO::any_relocation_info ARI = @@ -122,12 +116,12 @@ private: return RI; } - using PairRelocInfo = std::tuple; + using PairRelocInfo = std::tuple; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. Expected - parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, const MachO::relocation_info &SubRI, JITTargetAddress FixupAddress, const char *FixupContent, object::relocation_iterator &UnsignedRelItr, @@ -154,9 +148,11 @@ private: return make_error("length of x86_64 SUBTRACTOR and paired " "UNSIGNED reloc must match"); - auto FromAtom = findAtomBySymbolIndex(SubRI); - if (!FromAtom) - return FromAtom.takeError(); + Symbol *FromSymbol; + if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum)) + FromSymbol = FromSymbolOrErr->GraphSymbol; + else + return FromSymbolOrErr.takeError(); // Read the current fixup value. uint64_t FixupValue = 0; @@ -165,54 +161,60 @@ private: else FixupValue = *(const little32_t *)FixupContent; - // Find 'ToAtom' using symbol number or address, depending on whether the + // Find 'ToSymbol' using symbol number or address, depending on whether the // paired UNSIGNED relocation is extern. - Atom *ToAtom = nullptr; + Symbol *ToSymbol = nullptr; if (UnsignedRI.r_extern) { - // Find target atom by symbol index. - if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) - ToAtom = &*ToAtomOrErr; + // Find target symbol by symbol index. + if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum)) + ToSymbol = ToSymbolOrErr->GraphSymbol; else - return ToAtomOrErr.takeError(); + return ToSymbolOrErr.takeError(); } else { - if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) - ToAtom = &*ToAtomOrErr; + if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue)) + ToSymbol = &*ToSymbolOrErr; else - return ToAtomOrErr.takeError(); - FixupValue -= ToAtom->getAddress(); + return ToSymbolOrErr.takeError(); + FixupValue -= ToSymbol->getAddress(); } MachOX86RelocationKind DeltaKind; - Atom *TargetAtom; + Symbol *TargetSymbol; uint64_t Addend; - if (areLayoutLocked(AtomToFix, *FromAtom)) { - TargetAtom = ToAtom; + if (&BlockToFix == &FromSymbol->getAddressable()) { + TargetSymbol = ToSymbol; DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; - Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. - } else if (areLayoutLocked(AtomToFix, *ToAtom)) { - TargetAtom = &*FromAtom; + } else if (&BlockToFix == &ToSymbol->getAddressable()) { + TargetSymbol = FromSymbol; DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; - Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { - // AtomToFix was neither FromAtom nor ToAtom. + // BlockToFix was neither FromSymbol nor ToSymbol. return make_error("SUBTRACTOR relocation must fix up " - "either 'A' or 'B' (or an atom in one " - "of their alt-entry groups)"); + "either 'A' or 'B' (or a symbol in one " + "of their alt-entry chains)"); } - return PairRelocInfo(DeltaKind, TargetAtom, Addend); + return PairRelocInfo(DeltaKind, TargetSymbol, Addend); } Error addRelocations() override { using namespace support; - auto &G = getGraph(); auto &Obj = getObject(); for (auto &S : Obj.sections()) { JITTargetAddress SectionAddress = S.getAddress(); + if (S.isVirtual()) { + if (S.relocation_begin() != S.relocation_end()) + return make_error("Virtual section contains " + "relocations"); + continue; + } + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); RelItr != RelEnd; ++RelItr) { @@ -231,26 +233,26 @@ private: << format("0x%016" PRIx64, FixupAddress) << "\n"; }); - // Find the atom that the fixup points to. - DefinedAtom *AtomToFix = nullptr; + // Find the block that the fixup points to. + Block *BlockToFix = nullptr; { - auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); - if (!AtomToFixOrErr) - return AtomToFixOrErr.takeError(); - AtomToFix = &*AtomToFixOrErr; + auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress); + if (!SymbolToFixOrErr) + return SymbolToFixOrErr.takeError(); + BlockToFix = &SymbolToFixOrErr->getBlock(); } if (FixupAddress + static_cast(1ULL << RI.r_length) > - AtomToFix->getAddress() + AtomToFix->getContent().size()) + BlockToFix->getAddress() + BlockToFix->getContent().size()) return make_error( - "Relocation content extends past end of fixup atom"); + "Relocation extends past end of fixup block"); // Get a pointer to the fixup content. - const char *FixupContent = AtomToFix->getContent().data() + - (FixupAddress - AtomToFix->getAddress()); + const char *FixupContent = BlockToFix->getContent().data() + + (FixupAddress - BlockToFix->getAddress()); - // The target atom and addend will be populated by the switch below. - Atom *TargetAtom = nullptr; + // The target symbol and addend will be populated by the switch below. + Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; switch (*Kind) { @@ -258,53 +260,53 @@ private: case PCRel32: case PCRel32GOTLoad: case PCRel32GOT: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; break; case Pointer32: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; break; case Pointer64: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; break; case Pointer64Anon: { JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case PCRel32Minus1: case PCRel32Minus2: case PCRel32Minus4: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent + (1 << (*Kind - PCRel32Minus1)); break; case PCRel32Anon: { JITTargetAddress TargetAddress = FixupAddress + 4 + *(const ulittle32_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case PCRel32Minus1Anon: @@ -314,11 +316,11 @@ private: static_cast(1ULL << (*Kind - PCRel32Minus1Anon)); JITTargetAddress TargetAddress = FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case Delta32: @@ -329,12 +331,12 @@ private: // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetAtom, Addend) = *PairInfo; - assert(TargetAtom && "No target atom from parsePairRelocation?"); + std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } default: @@ -343,41 +345,38 @@ private: } LLVM_DEBUG({ - Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *AtomToFix, GE, + printEdge(dbgs(), *BlockToFix, GE, getMachOX86RelocationKindName(*Kind)); dbgs() << "\n"; }); - AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), - *TargetAtom, Addend); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); } } return Error::success(); } - - unsigned NumSymbols = 0; }; class MachO_x86_64_GOTAndStubsBuilder : public BasicGOTAndStubsBuilder { public: - MachO_x86_64_GOTAndStubsBuilder(AtomGraph &G) + MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder(G) {} bool isGOTEdge(Edge &E) const { return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; } - DefinedAtom &createGOTEntry(Atom &Target) { - auto &GOTEntryAtom = G.addAnonymousAtom(getGOTSection(), 0x0, 8); - GOTEntryAtom.setContent( - StringRef(reinterpret_cast(NullGOTEntryContent), 8)); - GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); - return GOTEntryAtom; + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } - void fixGOTEdge(Edge &E, Atom &GOTEntry) { + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && "Not a GOT edge?"); E.setKind(PCRel32); @@ -389,19 +388,16 @@ public: return E.getKind() == Branch32 && !E.getTarget().isDefined(); } - DefinedAtom &createStub(Atom &Target) { - auto &StubAtom = G.addAnonymousAtom(getStubsSection(), 0x0, 2); - StubAtom.setContent( - StringRef(reinterpret_cast(StubContent), 6)); - + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntryAtom = getGOTEntryAtom(Target); - StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); - - return StubAtom; + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } - void fixExternalBranchEdge(Edge &E, Atom &Stub) { + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); E.setTarget(Stub); @@ -410,7 +406,7 @@ public: private: Section &getGOTSection() { if (!GOTSection) - GOTSection = &G.createSection("$__GOT", 8, sys::Memory::MF_READ, false); + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); return *GOTSection; } @@ -418,11 +414,21 @@ private: if (!StubsSection) { auto StubsProt = static_cast( sys::Memory::MF_READ | sys::Memory::MF_EXEC); - StubsSection = &G.createSection("$__STUBS", 8, StubsProt, false); + StubsSection = &G.createSection("$__STUBS", StubsProt); } return *StubsSection; } + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast(StubContent), + sizeof(StubContent)); + } + static const uint8_t NullGOTEntryContent[8]; static const uint8_t StubContent[6]; Section *GOTSection = nullptr; @@ -451,30 +457,31 @@ private: return getMachOX86RelocationKindName(R); } - Expected> + Expected> buildGraph(MemoryBufferRef ObjBuffer) override { auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); if (!MachOObj) return MachOObj.takeError(); - return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); } - static Error targetOutOfRangeError(const Atom &A, const Edge &E) { + static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; { raw_string_ostream ErrStream(ErrMsg); ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, A, E, getMachOX86RelocationKindName(E.getKind())); + printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind())); ErrStream << "\n"; } return make_error(std::move(ErrMsg)); } - Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace support; - char *FixupPtr = AtomWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case Branch32: @@ -484,7 +491,7 @@ private: E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); if (Value < std::numeric_limits::min() || Value > std::numeric_limits::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -502,7 +509,7 @@ private: E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); if (Value < std::numeric_limits::min() || Value > std::numeric_limits::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -514,7 +521,7 @@ private: E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); if (Value < std::numeric_limits::min() || Value > std::numeric_limits::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -531,7 +538,7 @@ private: if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { if (Value < std::numeric_limits::min() || Value > std::numeric_limits::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; } else *(little64_t *)FixupPtr = Value; @@ -540,7 +547,7 @@ private: case Pointer32: { uint64_t Value = E.getTarget().getAddress() + E.getAddend(); if (Value > std::numeric_limits::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(ulittle32_t *)FixupPtr = Value; break; } @@ -563,10 +570,10 @@ void jitLink_MachO_x86_64(std::unique_ptr Ctx) { if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else - Config.PrePrunePasses.push_back(markAllAtomsLive); + Config.PrePrunePasses.push_back(markAllSymbolsLive); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { MachO_x86_64_GOTAndStubsBuilder(G).run(); return Error::success(); }); diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index 436fd55cd8d..5c7d888c2d6 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -226,7 +226,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { for (auto &KV : Aliases) OS << " " << *KV.first << ": " << KV.second.Aliasee << " " << KV.second.AliasFlags; - OS << " }\n"; + OS << " }"; return OS; } @@ -378,15 +378,12 @@ Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { }); #ifndef NDEBUG for (auto &KV : Symbols) { + auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); - if (I->second.isWeak()) - assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && - "Resolving symbol with incorrect flags"); - else - assert(I->second == KV.second.getFlags() && - "Resolving symbol with incorrect flags"); + assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && + "Resolving symbol with incorrect flags"); } #endif @@ -949,11 +946,14 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { if (SymI->second.getFlags().hasError()) SymbolsInErrorState.insert(KV.first); else { - assert((KV.second.getFlags() & ~JITSymbolFlags::Weak) == - (SymI->second.getFlags() & ~JITSymbolFlags::Weak) && + auto Flags = KV.second.getFlags(); + Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); + assert(Flags == (SymI->second.getFlags() & + ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && "Resolved flags should match the declared flags"); - Worklist.push_back({SymI, KV.second}); + Worklist.push_back( + {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); } } @@ -970,7 +970,6 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { // Resolved symbols can not be weak: discard the weak flag. JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); - ResolvedFlags &= ~JITSymbolFlags::Weak; SymI->second.setAddress(ResolvedSym.getAddress()); SymI->second.setFlags(ResolvedFlags); SymI->second.setState(SymbolState::Resolved); diff --git a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index e1b8d52acb4..952ca6071ff 100644 --- a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -41,7 +41,7 @@ public: } void lookup(const DenseSet &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation) override { + std::unique_ptr LC) override { JITDylibSearchList SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( @@ -54,19 +54,16 @@ public: InternedSymbols.insert(ES.intern(S)); // OnResolve -- De-intern the symbols and pass the result to the linker. - // FIXME: Capture LookupContinuation by move once we have c++14. - auto SharedLookupContinuation = - std::make_shared( - std::move(LookupContinuation)); - auto OnResolve = [this, SharedLookupContinuation](Expected Result) { + auto OnResolve = [this, LookupContinuation = std::move(LC)]( + Expected Result) mutable { auto Main = Layer.getExecutionSession().intern("_main"); if (!Result) - (*SharedLookupContinuation)(Result.takeError()); + LookupContinuation->run(Result.takeError()); else { AsyncLookupResult LR; for (auto &KV : *Result) LR[*KV.first] = KV.second; - (*SharedLookupContinuation)(std::move(LR)); + LookupContinuation->run(std::move(LR)); } }; @@ -76,29 +73,25 @@ public: }); } - void notifyResolved(AtomGraph &G) override { + void notifyResolved(LinkGraph &G) override { auto &ES = Layer.getExecutionSession(); SymbolFlagsMap ExtraSymbolsToClaim; bool AutoClaim = Layer.AutoClaimObjectSymbols; SymbolMap InternedResult; - for (auto *DA : G.defined_atoms()) - if (DA->hasName() && DA->isGlobal()) { - auto InternedName = ES.intern(DA->getName()); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getScope() != Scope::Local) { + auto InternedName = ES.intern(Sym->getName()); JITSymbolFlags Flags; - if (DA->isExported()) - Flags |= JITSymbolFlags::Exported; - if (DA->isWeak()) - Flags |= JITSymbolFlags::Weak; - if (DA->isCallable()) + if (Sym->isCallable()) Flags |= JITSymbolFlags::Callable; - if (DA->isCommon()) - Flags |= JITSymbolFlags::Common; + if (Sym->getScope() == Scope::Default) + Flags |= JITSymbolFlags::Exported; InternedResult[InternedName] = - JITEvaluatedSymbol(DA->getAddress(), Flags); + JITEvaluatedSymbol(Sym->getAddress(), Flags); if (AutoClaim && !MR.getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -106,17 +99,17 @@ public: } } - for (auto *A : G.absolute_atoms()) - if (A->hasName()) { - auto InternedName = ES.intern(A->getName()); + for (auto *Sym : G.absolute_symbols()) + if (Sym->hasName()) { + auto InternedName = ES.intern(Sym->getName()); JITSymbolFlags Flags; Flags |= JITSymbolFlags::Absolute; - if (A->isWeak()) - Flags |= JITSymbolFlags::Weak; - if (A->isCallable()) + if (Sym->isCallable()) Flags |= JITSymbolFlags::Callable; + if (Sym->getLinkage() == Linkage::Weak) + Flags |= JITSymbolFlags::Weak; InternedResult[InternedName] = - JITEvaluatedSymbol(A->getAddress(), Flags); + JITEvaluatedSymbol(Sym->getAddress(), Flags); if (AutoClaim && !MR.getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -148,17 +141,17 @@ public: } } - AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { - return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; } Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { // Add passes to mark duplicate defs as should-discard, and to walk the - // atom graph to build the symbol dependence graph. + // link graph to build the symbol dependence graph. Config.PrePrunePasses.push_back( - [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); }); Config.PostPrunePasses.push_back( - [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); Layer.modifyPassConfig(MR, TT, Config); @@ -166,65 +159,59 @@ public: } private: - using AnonAtomNamedDependenciesMap = - DenseMap; + using AnonToNamedDependenciesMap = DenseMap; - Error markSymbolsToDiscard(AtomGraph &G) { + Error externalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); - for (auto *DA : G.defined_atoms()) - if (DA->isWeak() && DA->hasName()) { - auto S = ES.intern(DA->getName()); - auto I = MR.getSymbols().find(S); - if (I == MR.getSymbols().end()) - DA->setShouldDiscard(true); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { + if (!MR.getSymbols().count(ES.intern(Sym->getName()))) + G.makeExternal(*Sym); } - for (auto *A : G.absolute_atoms()) - if (A->isWeak() && A->hasName()) { - auto S = ES.intern(A->getName()); - auto I = MR.getSymbols().find(S); - if (I == MR.getSymbols().end()) - A->setShouldDiscard(true); + for (auto *Sym : G.absolute_symbols()) + if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { + if (!MR.getSymbols().count(ES.intern(Sym->getName()))) + G.makeExternal(*Sym); } return Error::success(); } - Error markResponsibilitySymbolsLive(AtomGraph &G) const { + Error markResponsibilitySymbolsLive(LinkGraph &G) const { auto &ES = Layer.getExecutionSession(); - for (auto *DA : G.defined_atoms()) - if (DA->hasName() && - MR.getSymbols().count(ES.intern(DA->getName()))) - DA->setLive(true); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && MR.getSymbols().count(ES.intern(Sym->getName()))) + Sym->setLive(true); return Error::success(); } - Error computeNamedSymbolDependencies(AtomGraph &G) { + Error computeNamedSymbolDependencies(LinkGraph &G) { auto &ES = MR.getTargetJITDylib().getExecutionSession(); auto AnonDeps = computeAnonDeps(G); - for (auto *DA : G.defined_atoms()) { + for (auto *Sym : G.defined_symbols()) { // Skip anonymous and non-global atoms: we do not need dependencies for // these. - if (!DA->hasName() || !DA->isGlobal()) + if (Sym->getScope() == Scope::Local) continue; - auto DAName = ES.intern(DA->getName()); - SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + auto SymName = ES.intern(Sym->getName()); + SymbolNameSet &SymDeps = NamedSymbolDeps[SymName]; - for (auto &E : DA->edges()) { - auto &TA = E.getTarget(); + for (auto &E : Sym->getBlock().edges()) { + auto &TargetSym = E.getTarget(); - if (TA.hasName()) - DADeps.insert(ES.intern(TA.getName())); + if (TargetSym.getScope() != Scope::Local) + SymDeps.insert(ES.intern(TargetSym.getName())); else { - assert(TA.isDefined() && "Anonymous atoms must be defined"); - auto &DTA = static_cast(TA); - auto I = AnonDeps.find(&DTA); + assert(TargetSym.isDefined() && + "Anonymous/local symbols must be defined"); + auto I = AnonDeps.find(&TargetSym); if (I != AnonDeps.end()) for (auto &S : I->second) - DADeps.insert(S); + SymDeps.insert(S); } } } @@ -232,58 +219,59 @@ private: return Error::success(); } - AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + AnonToNamedDependenciesMap computeAnonDeps(LinkGraph &G) { auto &ES = MR.getTargetJITDylib().getExecutionSession(); - AnonAtomNamedDependenciesMap DepMap; + AnonToNamedDependenciesMap DepMap; - // For all anonymous atoms: + // For all anonymous symbols: // (1) Add their named dependencies. // (2) Add them to the worklist for further iteration if they have any - // depend on any other anonymous atoms. + // depend on any other anonymous symbols. struct WorklistEntry { - WorklistEntry(DefinedAtom *DA, DenseSet DAAnonDeps) - : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + WorklistEntry(Symbol *Sym, DenseSet SymAnonDeps) + : Sym(Sym), SymAnonDeps(std::move(SymAnonDeps)) {} - DefinedAtom *DA = nullptr; - DenseSet DAAnonDeps; + Symbol *Sym = nullptr; + DenseSet SymAnonDeps; }; std::vector Worklist; - for (auto *DA : G.defined_atoms()) - if (!DA->hasName()) { - auto &DANamedDeps = DepMap[DA]; - DenseSet DAAnonDeps; + for (auto *Sym : G.defined_symbols()) + if (!Sym->hasName()) { + auto &SymNamedDeps = DepMap[Sym]; + DenseSet SymAnonDeps; - for (auto &E : DA->edges()) { - auto &TA = E.getTarget(); - if (TA.hasName()) - DANamedDeps.insert(ES.intern(TA.getName())); + for (auto &E : Sym->getBlock().edges()) { + auto &TargetSym = E.getTarget(); + if (TargetSym.hasName()) + SymNamedDeps.insert(ES.intern(TargetSym.getName())); else { - assert(TA.isDefined() && "Anonymous atoms must be defined"); - DAAnonDeps.insert(static_cast(&TA)); + assert(TargetSym.isDefined() && + "Anonymous symbols must be defined"); + SymAnonDeps.insert(&TargetSym); } } - if (!DAAnonDeps.empty()) - Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + if (!SymAnonDeps.empty()) + Worklist.push_back(WorklistEntry(Sym, std::move(SymAnonDeps))); } - // Loop over all anonymous atoms with anonymous dependencies, propagating + // Loop over all anonymous symbols with anonymous dependencies, propagating // their respective *named* dependencies. Iterate until we hit a stable // state. bool Changed; do { Changed = false; for (auto &WLEntry : Worklist) { - auto *DA = WLEntry.DA; - auto &DANamedDeps = DepMap[DA]; - auto &DAAnonDeps = WLEntry.DAAnonDeps; + auto *Sym = WLEntry.Sym; + auto &SymNamedDeps = DepMap[Sym]; + auto &SymAnonDeps = WLEntry.SymAnonDeps; - for (auto *TA : DAAnonDeps) { - auto I = DepMap.find(TA); + for (auto *TargetSym : SymAnonDeps) { + auto I = DepMap.find(TargetSym); if (I != DepMap.end()) for (const auto &S : I->second) - Changed |= DANamedDeps.insert(S).second; + Changed |= SymNamedDeps.insert(S).second; } } } while (Changed); @@ -414,7 +402,7 @@ Error ObjectLinkingLayer::removeAllModules() { } EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( - jitlink::EHFrameRegistrar &Registrar) + EHFrameRegistrar &Registrar) : Registrar(Registrar) {} void EHFrameRegistrationPlugin::modifyPassConfig( diff --git a/test/ExecutionEngine/JITLink/X86/MachO_zero_fill_alignment.s b/test/ExecutionEngine/JITLink/X86/MachO_zero_fill_alignment.s index 3b3a3853fdc..b65b0cb6f59 100644 --- a/test/ExecutionEngine/JITLink/X86/MachO_zero_fill_alignment.s +++ b/test/ExecutionEngine/JITLink/X86/MachO_zero_fill_alignment.s @@ -1,14 +1,14 @@ # RUN: rm -rf %t && mkdir -p %t # RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj -o %t/macho_zero_fill_align.o %s -# RUN: llvm-jitlink -noexec %t/macho_zero_fill_align.o -entry higher_zero_fill_align +# RUN: llvm-jitlink -noexec %t/macho_zero_fill_align.o -entry _higher_zero_fill_align .section __DATA,__data - .globl low_aligned_data + .globl _low_aligned_data .p2align 0 -low_aligned_data: +_low_aligned_data: .byte 42 - .globl higher_zero_fill_align -.zerofill __DATA,__zero_fill,higher_zero_fill_align,8,3 + .globl _higher_zero_fill_align +.zerofill __DATA,__zero_fill,_higher_zero_fill_align,8,3 .subsections_via_symbols diff --git a/tools/llvm-jitlink/llvm-jitlink-macho.cpp b/tools/llvm-jitlink/llvm-jitlink-macho.cpp index 067c38a56cd..9488dfe9b75 100644 --- a/tools/llvm-jitlink/llvm-jitlink-macho.cpp +++ b/tools/llvm-jitlink/llvm-jitlink-macho.cpp @@ -26,53 +26,55 @@ static bool isMachOStubsSection(Section &S) { return S.getName() == "$__STUBS"; } -static Expected getFirstRelocationEdge(AtomGraph &G, DefinedAtom &DA) { - auto EItr = std::find_if(DA.edges().begin(), DA.edges().end(), +static Expected getFirstRelocationEdge(LinkGraph &G, Block &B) { + auto EItr = std::find_if(B.edges().begin(), B.edges().end(), [](Edge &E) { return E.isRelocation(); }); - if (EItr == DA.edges().end()) + if (EItr == B.edges().end()) return make_error("GOT entry in " + G.getName() + ", \"" + - DA.getSection().getName() + + B.getSection().getName() + "\" has no relocations", inconvertibleErrorCode()); return *EItr; } -static Expected getMachOGOTTarget(AtomGraph &G, DefinedAtom &DA) { - auto E = getFirstRelocationEdge(G, DA); +static Expected getMachOGOTTarget(LinkGraph &G, Block &B) { + auto E = getFirstRelocationEdge(G, B); if (!E) return E.takeError(); - auto &TA = E->getTarget(); - if (!TA.hasName()) - return make_error("GOT entry in " + G.getName() + ", \"" + - DA.getSection().getName() + - "\" points to anonymous " - "atom", - inconvertibleErrorCode()); - if (TA.isDefined() || TA.isAbsolute()) + auto &TargetSym = E->getTarget(); + if (!TargetSym.hasName()) return make_error( - "GOT entry \"" + TA.getName() + "\" in " + G.getName() + ", \"" + - DA.getSection().getName() + "\" does not point to an external atom", + "GOT entry in " + G.getName() + ", \"" + + TargetSym.getBlock().getSection().getName() + + "\" points to anonymous " + "symbol", inconvertibleErrorCode()); - return TA; + if (TargetSym.isDefined() || TargetSym.isAbsolute()) + return make_error( + "GOT entry \"" + TargetSym.getName() + "\" in " + G.getName() + ", \"" + + TargetSym.getBlock().getSection().getName() + + "\" does not point to an external symbol", + inconvertibleErrorCode()); + return TargetSym; } -static Expected getMachOStubTarget(AtomGraph &G, DefinedAtom &DA) { - auto E = getFirstRelocationEdge(G, DA); +static Expected getMachOStubTarget(LinkGraph &G, Block &B) { + auto E = getFirstRelocationEdge(G, B); if (!E) return E.takeError(); - auto &GOTA = E->getTarget(); - if (!GOTA.isDefined() || - !isMachOGOTSection(static_cast(GOTA).getSection())) - return make_error("Stubs entry in " + G.getName() + ", \"" + - DA.getSection().getName() + - "\" does not point to GOT entry", - inconvertibleErrorCode()); - return getMachOGOTTarget(G, static_cast(GOTA)); + auto &GOTSym = E->getTarget(); + if (!GOTSym.isDefined() || !isMachOGOTSection(GOTSym.getBlock().getSection())) + return make_error( + "Stubs entry in " + G.getName() + ", \"" + + GOTSym.getBlock().getSection().getName() + + "\" does not point to GOT entry", + inconvertibleErrorCode()); + return getMachOGOTTarget(G, GOTSym.getBlock()); } namespace llvm { -Error registerMachOStubsAndGOT(Session &S, AtomGraph &G) { +Error registerMachOStubsAndGOT(Session &S, LinkGraph &G) { auto FileName = sys::path::filename(G.getName()); if (S.FileInfos.count(FileName)) { return make_error("When -check is passed, file names must be " @@ -88,12 +90,12 @@ Error registerMachOStubsAndGOT(Session &S, AtomGraph &G) { for (auto &Sec : G.sections()) { LLVM_DEBUG({ dbgs() << " Section \"" << Sec.getName() << "\": " - << (Sec.atoms_empty() ? "empty. skipping." : "processing...") + << (Sec.symbols_empty() ? "empty. skipping." : "processing...") << "\n"; }); // Skip empty sections. - if (Sec.atoms_empty()) + if (Sec.symbols_empty()) continue; if (FileInfo.SectionInfos.count(Sec.getName())) @@ -105,54 +107,65 @@ Error registerMachOStubsAndGOT(Session &S, AtomGraph &G) { bool isGOTSection = isMachOGOTSection(Sec); bool isStubsSection = isMachOStubsSection(Sec); - auto *FirstAtom = *Sec.atoms().begin(); - auto *LastAtom = FirstAtom; - for (auto *DA : Sec.atoms()) { - if (DA->getAddress() < FirstAtom->getAddress()) - FirstAtom = DA; - if (DA->getAddress() > LastAtom->getAddress()) - LastAtom = DA; + bool SectionContainsContent = false; + bool SectionContainsZeroFill = false; + + auto *FirstSym = *Sec.symbols().begin(); + auto *LastSym = FirstSym; + for (auto *Sym : Sec.symbols()) { + if (Sym->getAddress() < FirstSym->getAddress()) + FirstSym = Sym; + if (Sym->getAddress() > LastSym->getAddress()) + LastSym = Sym; if (isGOTSection) { - if (Sec.isZeroFill()) - return make_error("Content atom in zero-fill section", + if (Sym->isSymbolZeroFill()) + return make_error("zero-fill atom in GOT section", inconvertibleErrorCode()); - if (auto TA = getMachOGOTTarget(G, *DA)) { - FileInfo.GOTEntryInfos[TA->getName()] = {DA->getContent(), - DA->getAddress()}; - } else - return TA.takeError(); - } else if (isStubsSection) { - if (Sec.isZeroFill()) - return make_error("Content atom in zero-fill section", - inconvertibleErrorCode()); - - if (auto TA = getMachOStubTarget(G, *DA)) - FileInfo.StubInfos[TA->getName()] = {DA->getContent(), - DA->getAddress()}; + if (auto TS = getMachOGOTTarget(G, Sym->getBlock())) + FileInfo.GOTEntryInfos[TS->getName()] = {Sym->getSymbolContent(), + Sym->getAddress()}; else - return TA.takeError(); - } else if (DA->hasName() && DA->isGlobal()) { - if (DA->isZeroFill()) - S.SymbolInfos[DA->getName()] = {DA->getSize(), DA->getAddress()}; - else { - if (Sec.isZeroFill()) - return make_error("Content atom in zero-fill section", - inconvertibleErrorCode()); - S.SymbolInfos[DA->getName()] = {DA->getContent(), DA->getAddress()}; + return TS.takeError(); + SectionContainsContent = true; + } else if (isStubsSection) { + if (Sym->isSymbolZeroFill()) + return make_error("zero-fill atom in Stub section", + inconvertibleErrorCode()); + + if (auto TS = getMachOStubTarget(G, Sym->getBlock())) + FileInfo.StubInfos[TS->getName()] = {Sym->getSymbolContent(), + Sym->getAddress()}; + else + return TS.takeError(); + SectionContainsContent = true; + } else if (Sym->hasName()) { + if (Sym->isSymbolZeroFill()) { + S.SymbolInfos[Sym->getName()] = {Sym->getSize(), Sym->getAddress()}; + SectionContainsZeroFill = true; + } else { + S.SymbolInfos[Sym->getName()] = {Sym->getSymbolContent(), + Sym->getAddress()}; + SectionContainsContent = true; } } } - JITTargetAddress SecAddr = FirstAtom->getAddress(); - uint64_t SecSize = (LastAtom->getAddress() + LastAtom->getSize()) - - FirstAtom->getAddress(); + JITTargetAddress SecAddr = FirstSym->getAddress(); + uint64_t SecSize = + (LastSym->getBlock().getAddress() + LastSym->getBlock().getSize()) - + SecAddr; - if (Sec.isZeroFill()) + if (SectionContainsZeroFill && SectionContainsContent) + return make_error("Mixed zero-fill and content sections not " + "supported yet", + inconvertibleErrorCode()); + if (SectionContainsZeroFill) FileInfo.SectionInfos[Sec.getName()] = {SecSize, SecAddr}; else FileInfo.SectionInfos[Sec.getName()] = { - StringRef(FirstAtom->getContent().data(), SecSize), SecAddr}; + StringRef(FirstSym->getBlock().getContent().data(), SecSize), + SecAddr}; } return Error::success(); diff --git a/tools/llvm-jitlink/llvm-jitlink.cpp b/tools/llvm-jitlink/llvm-jitlink.cpp index dfee97241a9..7edbea23a04 100644 --- a/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/tools/llvm-jitlink/llvm-jitlink.cpp @@ -86,9 +86,9 @@ static cl::opt ShowAddrs( cl::desc("Print registered symbol, section, got and stub addresses"), cl::init(false)); -static cl::opt ShowAtomGraph( +static cl::opt ShowLinkGraph( "show-graph", - cl::desc("Print the atom graph after fixups have been applied"), + cl::desc("Print the link graph after fixups have been applied"), cl::init(false)); static cl::opt ShowSizes( @@ -151,17 +151,14 @@ operator<<(raw_ostream &OS, const Session::FileInfoMap &FIM) { return OS; } -static uint64_t computeTotalAtomSizes(AtomGraph &G) { +static uint64_t computeTotalBlockSizes(LinkGraph &G) { uint64_t TotalSize = 0; - for (auto *DA : G.defined_atoms()) - if (DA->isZeroFill()) - TotalSize += DA->getZeroFillSize(); - else - TotalSize += DA->getContent().size(); + for (auto *B : G.blocks()) + TotalSize += B->getSize(); return TotalSize; } -static void dumpSectionContents(raw_ostream &OS, AtomGraph &G) { +static void dumpSectionContents(raw_ostream &OS, LinkGraph &G) { constexpr JITTargetAddress DumpWidth = 16; static_assert(isPowerOf2_64(DumpWidth), "DumpWidth must be a power of two"); @@ -172,56 +169,55 @@ static void dumpSectionContents(raw_ostream &OS, AtomGraph &G) { std::sort(Sections.begin(), Sections.end(), [](const Section *LHS, const Section *RHS) { - if (LHS->atoms_empty() && RHS->atoms_empty()) + if (LHS->symbols_empty() && RHS->symbols_empty()) return false; - if (LHS->atoms_empty()) + if (LHS->symbols_empty()) return false; - if (RHS->atoms_empty()) + if (RHS->symbols_empty()) return true; - return (*LHS->atoms().begin())->getAddress() < - (*RHS->atoms().begin())->getAddress(); + SectionRange LHSRange(*LHS); + SectionRange RHSRange(*RHS); + return LHSRange.getStart() < RHSRange.getStart(); }); for (auto *S : Sections) { OS << S->getName() << " content:"; - if (S->atoms_empty()) { + if (S->symbols_empty()) { OS << "\n section empty\n"; continue; } - // Sort atoms into order, then render. - std::vector Atoms(S->atoms().begin(), S->atoms().end()); - std::sort(Atoms.begin(), Atoms.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); + // Sort symbols into order, then render. + std::vector Syms(S->symbols().begin(), S->symbols().end()); + llvm::sort(Syms, [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); - JITTargetAddress NextAddr = Atoms.front()->getAddress() & ~(DumpWidth - 1); - for (auto *DA : Atoms) { - bool IsZeroFill = DA->isZeroFill(); - JITTargetAddress AtomStart = DA->getAddress(); - JITTargetAddress AtomSize = - IsZeroFill ? DA->getZeroFillSize() : DA->getContent().size(); - JITTargetAddress AtomEnd = AtomStart + AtomSize; - const uint8_t *AtomData = - IsZeroFill ? nullptr : DA->getContent().bytes_begin(); + JITTargetAddress NextAddr = Syms.front()->getAddress() & ~(DumpWidth - 1); + for (auto *Sym : Syms) { + bool IsZeroFill = Sym->getBlock().isZeroFill(); + JITTargetAddress SymStart = Sym->getAddress(); + JITTargetAddress SymSize = Sym->getSize(); + JITTargetAddress SymEnd = SymStart + SymSize; + const uint8_t *SymData = + IsZeroFill ? nullptr : Sym->getSymbolContent().bytes_begin(); - // Pad any space before the atom starts. - while (NextAddr != AtomStart) { + // Pad any space before the symbol starts. + while (NextAddr != SymStart) { if (NextAddr % DumpWidth == 0) OS << formatv("\n{0:x16}:", NextAddr); OS << " "; ++NextAddr; } - // Render the atom content. - while (NextAddr != AtomEnd) { + // Render the symbol content. + while (NextAddr != SymEnd) { if (NextAddr % DumpWidth == 0) OS << formatv("\n{0:x16}:", NextAddr); if (IsZeroFill) OS << " 00"; else - OS << formatv(" {0:x-2}", AtomData[NextAddr - AtomStart]); + OS << formatv(" {0:x-2}", SymData[NextAddr - SymStart]); ++NextAddr; } } @@ -291,18 +287,17 @@ public: for (auto &KV : Request) { auto &Seg = KV.second; - if (Seg.getContentAlignment() > PageSize) + if (Seg.getAlignment() > PageSize) return make_error("Cannot request higher than page " "alignment", inconvertibleErrorCode()); - if (PageSize % Seg.getContentAlignment() != 0) + if (PageSize % Seg.getAlignment() != 0) return make_error("Page size is not a multiple of " "alignment", inconvertibleErrorCode()); - uint64_t ZeroFillStart = - alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); + uint64_t ZeroFillStart = Seg.getContentSize(); uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); // Round segment size up to page boundary. @@ -427,7 +422,7 @@ void Session::dumpSessionInfo(raw_ostream &OS) { void Session::modifyPassConfig(const Triple &FTT, PassConfiguration &PassConfig) { if (!CheckFiles.empty()) - PassConfig.PostFixupPasses.push_back([this](AtomGraph &G) { + PassConfig.PostFixupPasses.push_back([this](LinkGraph &G) { if (TT.getObjectFormat() == Triple::MachO) return registerMachOStubsAndGOT(*this, G); return make_error("Unsupported object format for GOT/stub " @@ -435,27 +430,26 @@ void Session::modifyPassConfig(const Triple &FTT, inconvertibleErrorCode()); }); - if (ShowAtomGraph) - PassConfig.PostFixupPasses.push_back([](AtomGraph &G) -> Error { - outs() << "Atom graph post-fixup:\n"; + if (ShowLinkGraph) + PassConfig.PostFixupPasses.push_back([](LinkGraph &G) -> Error { + outs() << "Link graph post-fixup:\n"; G.dump(outs()); return Error::success(); }); - if (ShowSizes) { - PassConfig.PrePrunePasses.push_back([this](AtomGraph &G) -> Error { - SizeBeforePruning += computeTotalAtomSizes(G); - return Error::success(); - }); - PassConfig.PostFixupPasses.push_back([this](AtomGraph &G) -> Error { - SizeAfterFixups += computeTotalAtomSizes(G); - return Error::success(); - }); + PassConfig.PrePrunePasses.push_back([this](LinkGraph &G) -> Error { + SizeBeforePruning += computeTotalBlockSizes(G); + return Error::success(); + }); + PassConfig.PostFixupPasses.push_back([this](LinkGraph &G) -> Error { + SizeAfterFixups += computeTotalBlockSizes(G); + return Error::success(); + }); } if (ShowRelocatedSectionContents) - PassConfig.PostFixupPasses.push_back([](AtomGraph &G) -> Error { + PassConfig.PostFixupPasses.push_back([](LinkGraph &G) -> Error { outs() << "Relocated section contents for " << G.getName() << ":\n"; dumpSectionContents(outs(), G); return Error::success(); @@ -757,8 +751,8 @@ Error runChecks(Session &S) { static void dumpSessionStats(Session &S) { if (ShowSizes) - outs() << "Total size of all atoms before pruning: " << S.SizeBeforePruning - << "\nTotal size of all atoms after fixups: " << S.SizeAfterFixups + outs() << "Total size of all blocks before pruning: " << S.SizeBeforePruning + << "\nTotal size of all blocks after fixups: " << S.SizeAfterFixups << "\n"; } diff --git a/tools/llvm-jitlink/llvm-jitlink.h b/tools/llvm-jitlink/llvm-jitlink.h index 269597a29a3..f94a50993c1 100644 --- a/tools/llvm-jitlink/llvm-jitlink.h +++ b/tools/llvm-jitlink/llvm-jitlink.h @@ -65,7 +65,7 @@ struct Session { uint64_t SizeAfterFixups = 0; }; -Error registerMachOStubsAndGOT(Session &S, jitlink::AtomGraph &G); +Error registerMachOStubsAndGOT(Session &S, jitlink::LinkGraph &G); } // end namespace llvm diff --git a/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp b/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp index 23f8a691c8f..c5d7dc2fdc9 100644 --- a/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp +++ b/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.cpp @@ -145,7 +145,7 @@ void JITLinkTestCommon::TestJITLinkContext::notifyFailed(Error Err) { void JITLinkTestCommon::TestJITLinkContext::lookup( const DenseSet &Symbols, - JITLinkAsyncLookupContinuation LookupContinuation) { + std::unique_ptr LC) { jitlink::AsyncLookupResult LookupResult; DenseSet MissingSymbols; for (const auto &Symbol : Symbols) { @@ -157,7 +157,7 @@ void JITLinkTestCommon::TestJITLinkContext::lookup( } if (MissingSymbols.empty()) - LookupContinuation(std::move(LookupResult)); + LC->run(std::move(LookupResult)); else { std::string ErrMsg; { @@ -167,12 +167,12 @@ void JITLinkTestCommon::TestJITLinkContext::lookup( ErrMsgStream << " " << Sym; ErrMsgStream << " ]\n"; } - LookupContinuation( + LC->run( make_error(std::move(ErrMsg), inconvertibleErrorCode())); } } -void JITLinkTestCommon::TestJITLinkContext::notifyResolved(AtomGraph &G) { +void JITLinkTestCommon::TestJITLinkContext::notifyResolved(LinkGraph &G) { if (NotifyResolved) NotifyResolved(G); } @@ -186,7 +186,7 @@ void JITLinkTestCommon::TestJITLinkContext::notifyFinalized( Error JITLinkTestCommon::TestJITLinkContext::modifyPassConfig( const Triple &TT, PassConfiguration &Config) { if (TestCase) - Config.PostFixupPasses.push_back([&](AtomGraph &G) -> Error { + Config.PostFixupPasses.push_back([&](LinkGraph &G) -> Error { TestCase(G); return Error::success(); }); @@ -196,11 +196,11 @@ Error JITLinkTestCommon::TestJITLinkContext::modifyPassConfig( JITLinkTestCommon::JITLinkTestCommon() { initializeLLVMTargets(); } Expected> -JITLinkTestCommon::disassemble(const MCDisassembler &Dis, - jitlink::DefinedAtom &Atom, size_t Offset) { +JITLinkTestCommon::disassemble(const MCDisassembler &Dis, jitlink::Block &B, + size_t Offset) { ArrayRef InstBuffer( - reinterpret_cast(Atom.getContent().data()) + Offset, - Atom.getContent().size() - Offset); + reinterpret_cast(B.getContent().data()) + Offset, + B.getContent().size() - Offset); MCInst Inst; uint64_t InstSize; @@ -214,11 +214,9 @@ JITLinkTestCommon::disassemble(const MCDisassembler &Dis, return std::make_pair(Inst, InstSize); } -Expected -JITLinkTestCommon::decodeImmediateOperand(const MCDisassembler &Dis, - jitlink::DefinedAtom &Atom, - size_t OpIdx, size_t Offset) { - auto InstAndSize = disassemble(Dis, Atom, Offset); +Expected JITLinkTestCommon::decodeImmediateOperand( + const MCDisassembler &Dis, jitlink::Block &B, size_t OpIdx, size_t Offset) { + auto InstAndSize = disassemble(Dis, B, Offset); if (!InstAndSize) return InstAndSize.takeError(); diff --git a/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h b/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h index 8e1273ed911..5c90532d897 100644 --- a/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h +++ b/unittests/ExecutionEngine/JITLink/JITLinkTestCommon.h @@ -77,9 +77,9 @@ public: class TestJITLinkContext : public jitlink::JITLinkContext { public: - using TestCaseFunction = std::function; + using TestCaseFunction = std::function; - using NotifyResolvedFunction = std::function; + using NotifyResolvedFunction = std::function; using NotifyFinalizedFunction = std::function)>; @@ -103,11 +103,11 @@ public: void notifyFailed(Error Err) override; - void - lookup(const DenseSet &Symbols, - jitlink::JITLinkAsyncLookupContinuation LookupContinuation) override; + void lookup( + const DenseSet &Symbols, + std::unique_ptr LC) override; - void notifyResolved(jitlink::AtomGraph &G) override; + void notifyResolved(jitlink::LinkGraph &G) override; void notifyFinalized( std::unique_ptr A) override; @@ -140,56 +140,60 @@ public: } template - static Expected readInt(jitlink::AtomGraph &G, jitlink::DefinedAtom &A, + static Expected readInt(jitlink::LinkGraph &G, jitlink::Block &B, size_t Offset = 0) { - if (Offset + sizeof(T) > A.getContent().size()) - return make_error("Reading past end of atom content", + if (Offset + sizeof(T) > B.getSize()) + return make_error("Reading past end of block content", inconvertibleErrorCode()); - return support::endian::read(A.getContent().data() + Offset, + return support::endian::read(B.getContent().data() + Offset, G.getEndianness()); } template - static Expected readInt(jitlink::AtomGraph &G, StringRef AtomName, + static Expected readInt(jitlink::LinkGraph &G, StringRef SymbolName, size_t Offset = 0) { - auto DA = G.findDefinedAtomByName(AtomName); - if (!DA) - return DA.takeError(); - return readInt(G, *DA); + for (auto *Sym : G.defined_symbols()) { + if (Sym->getName() == SymbolName) + return readInt(G, Sym->getBlock(), Sym->getOffset() + Offset); + } + return make_error("Symbol \"" + SymbolName + "\" not found", + inconvertibleErrorCode()); } static Expected> - disassemble(const MCDisassembler &Dis, jitlink::DefinedAtom &Atom, - size_t Offset = 0); + disassemble(const MCDisassembler &Dis, jitlink::Block &B, size_t Offset = 0); static Expected decodeImmediateOperand(const MCDisassembler &Dis, - jitlink::DefinedAtom &Atom, + jitlink::Block &B, size_t OpIdx, size_t Offset = 0); - static jitlink::Atom &atom(jitlink::AtomGraph &G, StringRef Name) { - return G.getAtomByName(Name); + static jitlink::Symbol &symbol(jitlink::LinkGraph &G, StringRef Name) { + for (auto *Sym : G.defined_symbols()) + if (Sym->getName() == Name) + return *Sym; + for (auto *Sym : G.external_symbols()) + if (Sym->getName() == Name) + return *Sym; + for (auto *Sym : G.absolute_symbols()) + if (Sym->getName() == Name) + return *Sym; + llvm_unreachable("Name must reference a symbol"); } - static jitlink::DefinedAtom &definedAtom(jitlink::AtomGraph &G, - StringRef Name) { - return G.getDefinedAtomByName(Name); - } - - static JITTargetAddress atomAddr(jitlink::AtomGraph &G, StringRef Name) { - return atom(G, Name).getAddress(); + static JITTargetAddress symbolAddr(jitlink::LinkGraph &G, StringRef Name) { + return symbol(G, Name).getAddress(); } template - static size_t countEdgesMatching(jitlink::DefinedAtom &DA, - const PredT &Pred) { - return std::count_if(DA.edges().begin(), DA.edges().end(), Pred); + static size_t countEdgesMatching(jitlink::Block &B, const PredT &Pred) { + return std::count_if(B.edges().begin(), B.edges().end(), Pred); } template - static size_t countEdgesMatching(jitlink::AtomGraph &G, StringRef Name, + static size_t countEdgesMatching(jitlink::LinkGraph &G, StringRef Name, const PredT &Pred) { - return countEdgesMatching(definedAtom(G, Name), Pred); + return countEdgesMatching(symbol(G, Name), Pred); } private: diff --git a/unittests/ExecutionEngine/JITLink/MachO_x86_64_Tests.cpp b/unittests/ExecutionEngine/JITLink/MachO_x86_64_Tests.cpp index e051ad551c7..9b76edae499 100644 --- a/unittests/ExecutionEngine/JITLink/MachO_x86_64_Tests.cpp +++ b/unittests/ExecutionEngine/JITLink/MachO_x86_64_Tests.cpp @@ -24,7 +24,7 @@ class JITLinkTest_MachO_x86_64 : public JITLinkTestCommon, public testing::Test { public: using BasicVerifyGraphFunction = - std::function; + std::function; void runBasicVerifyGraphTest(StringRef AsmSrc, StringRef Triple, StringMap Externals, @@ -40,7 +40,7 @@ public: } auto JTCtx = std::make_unique( - **TR, [&](AtomGraph &G) { RunGraphTest(G, (*TR)->getDisassembler()); }); + **TR, [&](LinkGraph &G) { RunGraphTest(G, (*TR)->getDisassembler()); }); JTCtx->externals() = std::move(Externals); @@ -48,78 +48,77 @@ public: } protected: - static void verifyIsPointerTo(AtomGraph &G, DefinedAtom &A, Atom &Target) { - EXPECT_EQ(A.edges_size(), 1U) << "Incorrect number of edges for pointer"; - if (A.edges_size() != 1U) + static void verifyIsPointerTo(LinkGraph &G, Block &B, Symbol &Target) { + EXPECT_EQ(B.edges_size(), 1U) << "Incorrect number of edges for pointer"; + if (B.edges_size() != 1U) return; - auto &E = *A.edges().begin(); + auto &E = *B.edges().begin(); + EXPECT_EQ(E.getOffset(), 0U) << "Expected edge offset of zero"; EXPECT_EQ(E.getKind(), Pointer64) << "Expected pointer to have a pointer64 relocation"; EXPECT_EQ(&E.getTarget(), &Target) << "Expected edge to point at target"; - EXPECT_THAT_EXPECTED(readInt(G, A), HasValue(Target.getAddress())) + EXPECT_THAT_EXPECTED(readInt(G, B), HasValue(Target.getAddress())) << "Pointer does not point to target"; } - static void verifyGOTLoad(AtomGraph &G, DefinedAtom &A, Edge &E, - Atom &Target) { + static void verifyGOTLoad(LinkGraph &G, Edge &E, Symbol &Target) { EXPECT_EQ(E.getAddend(), 0U) << "Expected GOT load to have a zero addend"; EXPECT_TRUE(E.getTarget().isDefined()) - << "GOT entry should be a defined atom"; + << "GOT entry should be a defined symbol"; if (!E.getTarget().isDefined()) return; - verifyIsPointerTo(G, static_cast(E.getTarget()), Target); + verifyIsPointerTo(G, E.getTarget().getBlock(), Target); } - static void verifyCall(const MCDisassembler &Dis, AtomGraph &G, - DefinedAtom &Caller, Edge &E, Atom &Callee) { + static void verifyCall(const MCDisassembler &Dis, LinkGraph &G, + Block &CallerBlock, Edge &E, Symbol &Callee) { EXPECT_EQ(E.getKind(), Branch32) << "Edge is not a Branch32"; EXPECT_EQ(E.getAddend(), 0U) << "Expected no addend on stub call"; EXPECT_EQ(&E.getTarget(), &Callee) << "Edge does not point at expected callee"; - JITTargetAddress FixupAddress = Caller.getAddress() + E.getOffset(); + JITTargetAddress FixupAddress = CallerBlock.getAddress() + E.getOffset(); uint64_t PCRelDelta = Callee.getAddress() - (FixupAddress + 4); EXPECT_THAT_EXPECTED( - decodeImmediateOperand(Dis, Caller, 0, E.getOffset() - 1), + decodeImmediateOperand(Dis, CallerBlock, 0, E.getOffset() - 1), HasValue(PCRelDelta)); } - static void verifyIndirectCall(const MCDisassembler &Dis, AtomGraph &G, - DefinedAtom &Caller, Edge &E, Atom &Callee) { + static void verifyIndirectCall(const MCDisassembler &Dis, LinkGraph &G, + Block &CallerBlock, Edge &E, Symbol &Callee) { EXPECT_EQ(E.getKind(), PCRel32) << "Edge is not a PCRel32"; EXPECT_EQ(E.getAddend(), 0) << "Expected no addend on stub cal"; - EXPECT_TRUE(E.getTarget().isDefined()) << "Target is not a defined atom"; + EXPECT_TRUE(E.getTarget().isDefined()) << "Target is not a defined symbol"; if (!E.getTarget().isDefined()) return; - verifyIsPointerTo(G, static_cast(E.getTarget()), Callee); + verifyIsPointerTo(G, E.getTarget().getBlock(), Callee); - JITTargetAddress FixupAddress = Caller.getAddress() + E.getOffset(); + JITTargetAddress FixupAddress = CallerBlock.getAddress() + E.getOffset(); uint64_t PCRelDelta = E.getTarget().getAddress() - (FixupAddress + 4); EXPECT_THAT_EXPECTED( - decodeImmediateOperand(Dis, Caller, 3, E.getOffset() - 2), + decodeImmediateOperand(Dis, CallerBlock, 3, E.getOffset() - 2), HasValue(PCRelDelta)); } - static void verifyCallViaStub(const MCDisassembler &Dis, AtomGraph &G, - DefinedAtom &Caller, Edge &E, Atom &Callee) { - verifyCall(Dis, G, Caller, E, E.getTarget()); + static void verifyCallViaStub(const MCDisassembler &Dis, LinkGraph &G, + Block &CallerBlock, Edge &E, Symbol &Callee) { + verifyCall(Dis, G, CallerBlock, E, E.getTarget()); if (!E.getTarget().isDefined()) { ADD_FAILURE() << "Edge target is not a stub"; return; } - auto &StubAtom = static_cast(E.getTarget()); - EXPECT_EQ(StubAtom.edges_size(), 1U) + auto &StubBlock = E.getTarget().getBlock(); + EXPECT_EQ(StubBlock.edges_size(), 1U) << "Expected one edge from stub to target"; - auto &StubEdge = *StubAtom.edges().begin(); + auto &StubEdge = *StubBlock.edges().begin(); - verifyIndirectCall(Dis, G, static_cast(StubAtom), StubEdge, - Callee); + verifyIndirectCall(Dis, G, StubBlock, StubEdge, Callee); } }; @@ -161,24 +160,24 @@ TEST_F(JITLinkTest_MachO_x86_64, BasicRelocations) { {{"_y", JITEvaluatedSymbol(0xdeadbeef, JITSymbolFlags::Exported)}, {"_baz", JITEvaluatedSymbol(0xcafef00d, JITSymbolFlags::Exported)}}, true, false, MCTargetOptions(), - [](AtomGraph &G, const MCDisassembler &Dis) { - // Name the atoms in the asm above. - auto &Baz = atom(G, "_baz"); - auto &Y = atom(G, "_y"); - - auto &Bar = definedAtom(G, "_bar"); - auto &Foo = definedAtom(G, "_foo"); - auto &Foo_1 = definedAtom(G, "_foo.1"); - auto &Foo_2 = definedAtom(G, "_foo.2"); - auto &X = definedAtom(G, "_x"); - auto &P = definedAtom(G, "_p"); + [](LinkGraph &G, const MCDisassembler &Dis) { + // Name the symbols in the asm above. + auto &Baz = symbol(G, "_baz"); + auto &Y = symbol(G, "_y"); + auto &Bar = symbol(G, "_bar"); + auto &Foo = symbol(G, "_foo"); + auto &Foo_1 = symbol(G, "_foo.1"); + auto &Foo_2 = symbol(G, "_foo.2"); + auto &X = symbol(G, "_x"); + auto &P = symbol(G, "_p"); // Check unsigned reloc for _p { - EXPECT_EQ(P.edges_size(), 1U) << "Unexpected number of relocations"; - EXPECT_EQ(P.edges().begin()->getKind(), Pointer64) + EXPECT_EQ(P.getBlock().edges_size(), 1U) + << "Unexpected number of relocations"; + EXPECT_EQ(P.getBlock().edges().begin()->getKind(), Pointer64) << "Unexpected edge kind for _p"; - EXPECT_THAT_EXPECTED(readInt(G, P), + EXPECT_THAT_EXPECTED(readInt(G, P.getBlock()), HasValue(X.getAddress())) << "Unsigned relocation did not apply correctly"; } @@ -188,41 +187,45 @@ TEST_F(JITLinkTest_MachO_x86_64, BasicRelocations) { // indirect call, and that the pointer for the indirect call points to // baz. { - EXPECT_EQ(Bar.edges_size(), 1U) + EXPECT_EQ(Bar.getBlock().edges_size(), 1U) << "Incorrect number of edges for bar"; - EXPECT_EQ(Bar.edges().begin()->getKind(), Branch32) + EXPECT_EQ(Bar.getBlock().edges().begin()->getKind(), Branch32) << "Unexpected edge kind for _bar"; - verifyCallViaStub(Dis, G, Bar, *Bar.edges().begin(), Baz); + verifyCallViaStub(Dis, G, Bar.getBlock(), + *Bar.getBlock().edges().begin(), Baz); } // Check that _foo is a direct call to _bar. { - EXPECT_EQ(Foo.edges_size(), 1U) + EXPECT_EQ(Foo.getBlock().edges_size(), 1U) << "Incorrect number of edges for foo"; - EXPECT_EQ(Foo.edges().begin()->getKind(), Branch32); - verifyCall(Dis, G, Foo, *Foo.edges().begin(), Bar); + EXPECT_EQ(Foo.getBlock().edges().begin()->getKind(), Branch32); + verifyCall(Dis, G, Foo.getBlock(), *Foo.getBlock().edges().begin(), + Bar); } // Check .got load in _foo.1 { - EXPECT_EQ(Foo_1.edges_size(), 1U) + EXPECT_EQ(Foo_1.getBlock().edges_size(), 1U) << "Incorrect number of edges for foo_1"; - EXPECT_EQ(Foo_1.edges().begin()->getKind(), PCRel32); - verifyGOTLoad(G, Foo_1, *Foo_1.edges().begin(), Y); + EXPECT_EQ(Foo_1.getBlock().edges().begin()->getKind(), PCRel32); + verifyGOTLoad(G, *Foo_1.getBlock().edges().begin(), Y); } // Check PCRel ref to _p in _foo.2 { - EXPECT_EQ(Foo_2.edges_size(), 1U) + EXPECT_EQ(Foo_2.getBlock().edges_size(), 1U) << "Incorrect number of edges for foo_2"; - EXPECT_EQ(Foo_2.edges().begin()->getKind(), PCRel32); + EXPECT_EQ(Foo_2.getBlock().edges().begin()->getKind(), PCRel32); JITTargetAddress FixupAddress = - Foo_2.getAddress() + Foo_2.edges().begin()->getOffset(); + Foo_2.getBlock().getAddress() + + Foo_2.getBlock().edges().begin()->getOffset(); uint64_t PCRelDelta = P.getAddress() - (FixupAddress + 4); - EXPECT_THAT_EXPECTED(decodeImmediateOperand(Dis, Foo_2, 4, 0), - HasValue(PCRelDelta)) + EXPECT_THAT_EXPECTED( + decodeImmediateOperand(Dis, Foo_2.getBlock(), 4, 0), + HasValue(PCRelDelta)) << "PCRel load does not reference expected target"; } });