From 7247401c119cd34a0f1bf2467be78f60e664a27b Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Sun, 23 Feb 2020 22:53:48 -0800 Subject: [PATCH] [JITLink] Add a MachO x86-64 GOT and Stub bypass optimization. This optimization bypasses GOT loads and calls/branches through stubs when the ultimate target of the access/branch is found to be within range of the reference. Extra debugging output is also added to the generic JITLink algorithm and basic GOT and Stubs builder utility to aid debugging. --- .../llvm/ExecutionEngine/JITLink/JITLink.h | 17 ++- .../ExecutionEngine/JITLink/MachO_x86_64.h | 1 + .../JITLink/BasicGOTAndStubsBuilder.h | 30 ++++- .../JITLink/JITLinkGeneric.cpp | 114 +++++++++++++++++- lib/ExecutionEngine/JITLink/JITLinkGeneric.h | 97 +++------------ lib/ExecutionEngine/JITLink/MachO_x86_64.cpp | 106 +++++++++++++++- .../MachO_GOTAndStubsOptimizationHelper.s | 8 ++ .../X86/MachO_GOTAndStubsOptimization.s | 31 +++++ .../JITLink/X86/MachO_x86-64_relocations.s | 12 +- 9 files changed, 320 insertions(+), 96 deletions(-) create mode 100644 test/ExecutionEngine/JITLink/X86/Inputs/MachO_GOTAndStubsOptimizationHelper.s create mode 100644 test/ExecutionEngine/JITLink/X86/MachO_GOTAndStubsOptimization.s diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index d673a89386d..967397f10ad 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -1176,7 +1176,7 @@ struct PassConfiguration { /// Pre-prune passes. /// /// These passes are called on the graph after it is built, and before any - /// symbols have been pruned. + /// symbols have been pruned. Graph nodes still have their original vmaddrs. /// /// Notable use cases: Marking symbols live or should-discard. LinkGraphPassList PrePrunePasses; @@ -1184,15 +1184,26 @@ struct PassConfiguration { /// Post-prune passes. /// /// These passes are called on the graph after dead stripping, but before - /// fixups are applied. + /// memory is allocated or nodes assigned their final addresses. /// /// Notable use cases: Building GOT, stub, and TLV symbols. LinkGraphPassList PostPrunePasses; + /// Pre-fixup passes. + /// + /// These passes are called on the graph after memory has been allocated, + /// content copied into working memory, and nodes have been assigned their + /// final addresses. + /// + /// Notable use cases: Late link-time optimizations like GOT and stub + /// elimination. + LinkGraphPassList PostAllocationPasses; + /// Post-fixup passes. /// /// These passes are called on the graph after block contents has been copied - /// to working memory, and fixups applied. + /// to working memory, and fixups applied. Graph nodes have been updated to + /// their final target vmaddrs. /// /// Notable use cases: Testing and validation. LinkGraphPassList PostFixupPasses; diff --git a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h index 00a7feb86e8..27fcdf4fa99 100644 --- a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h +++ b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -22,6 +22,7 @@ namespace MachO_x86_64_Edges { enum MachOX86RelocationKind : Edge::Kind { Branch32 = Edge::FirstRelocation, + Branch32ToStub, Pointer32, Pointer64, Pointer64Anon, diff --git a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h index b47a798c760..82258a35a67 100644 --- a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -15,6 +15,8 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#define DEBUG_TYPE "jitlink" + namespace llvm { namespace jitlink { @@ -27,12 +29,25 @@ public: // the newly added ones, so just copy the existing blocks out. std::vector Blocks(G.blocks().begin(), G.blocks().end()); + LLVM_DEBUG(dbgs() << "Creating GOT entries and stubs:\n"); + for (auto *B : Blocks) for (auto &E : B->edges()) - if (impl().isGOTEdge(E)) + if (impl().isGOTEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Updating GOT edge "; + printEdge(dbgs(), *B, E, ""); + dbgs() << "\n"; + }); impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); - else if (impl().isExternalBranchEdge(E)) + } else if (impl().isExternalBranchEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Updating external branch edge "; + printEdge(dbgs(), *B, E, ""); + dbgs() << "\n"; + }); impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); + } } protected: @@ -44,11 +59,17 @@ protected: // Build the entry if it doesn't exist. if (GOTEntryI == GOTEntries.end()) { auto &GOTEntry = impl().createGOTEntry(Target); + LLVM_DEBUG({ + dbgs() << " Created GOT entry for " << Target.getName() << ": " + << GOTEntry << "\n"; + }); GOTEntryI = GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; } assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); + LLVM_DEBUG( + { dbgs() << " Using GOT entry " << *GOTEntryI->second << "\n"; }); return *GOTEntryI->second; } @@ -59,10 +80,15 @@ protected: if (StubI == Stubs.end()) { auto &StubSymbol = impl().createStub(Target); + LLVM_DEBUG({ + dbgs() << " Created stub for " << Target.getName() << ": " + << StubSymbol << "\n"; + }); StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; } assert(StubI != Stubs.end() && "Count not get stub symbol"); + LLVM_DEBUG({ dbgs() << " Using stub " << *StubI->second << "\n"; }); return *StubI->second; } diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 7b594fd2c0e..ec6681c823f 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -24,6 +24,8 @@ JITLinkerBase::~JITLinkerBase() {} void JITLinkerBase::linkPhase1(std::unique_ptr Self) { + LLVM_DEBUG({ dbgs() << "Building jitlink graph for new input...\n"; }); + // Build the link graph. if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) G = std::move(*GraphOrErr); @@ -31,6 +33,10 @@ void JITLinkerBase::linkPhase1(std::unique_ptr Self) { return Ctx->notifyFailed(GraphOrErr.takeError()); assert(G && "Graph should have been created by buildGraph above"); + LLVM_DEBUG({ + dbgs() << "Starting link phase 1 for graph " << G->getName() << "\n"; + }); + // Prune and optimize the graph. if (auto Err = runPasses(Passes.PrePrunePasses)) return Ctx->notifyFailed(std::move(Err)); @@ -59,10 +65,17 @@ void JITLinkerBase::linkPhase1(std::unique_ptr Self) { return Ctx->notifyFailed(std::move(Err)); // Notify client that the defined symbols have been assigned addresses. + LLVM_DEBUG( + { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; }); Ctx->notifyResolved(*G); auto ExternalSymbols = getExternalSymbolNames(); + LLVM_DEBUG({ + dbgs() << "Issuing lookup for external symbols for " << G->getName() + << " (may trigger materialization/linking of other graphs)...\n"; + }); + // We're about to hand off ownership of ourself to the continuation. Grab a // pointer to the context so that we can call it to initiate the lookup. // @@ -87,6 +100,11 @@ void JITLinkerBase::linkPhase1(std::unique_ptr Self) { void JITLinkerBase::linkPhase2(std::unique_ptr Self, Expected LR, SegmentLayoutMap Layout) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 2 for graph " << G->getName() << "\n"; + }); + // If the lookup failed, bail out. if (!LR) return deallocateAndBailOut(LR.takeError()); @@ -94,13 +112,25 @@ void JITLinkerBase::linkPhase2(std::unique_ptr Self, // Assign addresses to external addressables. applyLookupResult(*LR); + // Copy block content to working memory. + copyBlockContentToWorkingMemory(Layout, *Alloc); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() + << "\" before post-allocation passes:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostAllocationPasses)) + return deallocateAndBailOut(std::move(Err)); + LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; dumpGraph(dbgs()); }); - // Copy block content to working memory and fix up. - if (auto Err = copyAndFixUpBlocks(Layout, *Alloc)) + // Fix up block content. + if (auto Err = fixUpBlocks(*G)) return deallocateAndBailOut(std::move(Err)); LLVM_DEBUG({ @@ -122,9 +152,16 @@ void JITLinkerBase::linkPhase2(std::unique_ptr Self, } void JITLinkerBase::linkPhase3(std::unique_ptr Self, Error Err) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 3 for graph " << G->getName() << "\n"; + }); + if (Err) return deallocateAndBailOut(std::move(Err)); Ctx->notifyFinalized(std::move(Alloc)); + + LLVM_DEBUG({ dbgs() << "Link of graph " << G->getName() << " complete\n"; }); } Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) { @@ -165,7 +202,7 @@ JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { } LLVM_DEBUG({ - dbgs() << "Segment ordering:\n"; + dbgs() << "Computed segment ordering:\n"; for (auto &KV : Layout) { dbgs() << " Segment " << static_cast(KV.first) << ":\n"; @@ -302,6 +339,77 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { "All strong external symbols should have been resolved by now"); } +void JITLinkerBase::copyBlockContentToWorkingMemory( + const SegmentLayoutMap &Layout, JITLinkMemoryManager::Allocation &Alloc) { + + LLVM_DEBUG(dbgs() << "Copying block content:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = + Alloc.getWorkingMemory(static_cast(Prot)); + char *LastBlockEnd = SegMem.data(); + char *BlockDataPtr = LastBlockEnd; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto *B : SegLayout.ContentBlocks) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Pad to alignment/alignment-offset. + BlockDataPtr = alignToBlock(BlockDataPtr, *B); + + LLVM_DEBUG({ + dbgs() << " Bumped block pointer to " << (const void *)BlockDataPtr + << " to meet block alignment " << B->getAlignment() + << " and alignment offset " << B->getAlignmentOffset() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastBlockEnd != BlockDataPtr) + dbgs() << " Zero padding from " << (const void *)LastBlockEnd + << " to " << (const void *)BlockDataPtr << "\n"; + }); + + while (LastBlockEnd != BlockDataPtr) + *LastBlockEnd++ = 0; + + // Copy initial block content. + LLVM_DEBUG({ + dbgs() << " Copying block " << *B << " content, " + << B->getContent().size() << " bytes, from " + << (const void *)B->getContent().data() << " to " + << (const void *)BlockDataPtr << "\n"; + }); + memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); + + // Point the block's content to the fixed up buffer. + B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + + // Update block end pointer. + LastBlockEnd = BlockDataPtr + B->getContent().size(); + BlockDataPtr = LastBlockEnd; + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastBlockEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastBlockEnd != SegMem.data() + SegMem.size()) + *LastBlockEnd++ = 0; + } +} + void JITLinkerBase::deallocateAndBailOut(Error Err) { assert(Err && "Should not be bailing out on success value"); assert(Alloc && "can not call deallocateAndBailOut before allocation"); diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index d5687b7afc9..53459094649 100644 --- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -100,14 +100,14 @@ private: // Copy block contents and apply relocations. // Implemented in JITLinker. - virtual Error - copyAndFixUpBlocks(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const = 0; + virtual Error fixUpBlocks(LinkGraph &G) const = 0; SegmentLayoutMap layOutBlocks(); Error allocateSegments(const SegmentLayoutMap &Layout); JITLinkContext::LookupMap getExternalSymbolNames() const; void applyLookupResult(AsyncLookupResult LR); + void copyBlockContentToWorkingMemory(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc); void deallocateAndBailOut(Error Err); void dumpGraph(raw_ostream &OS); @@ -144,88 +144,25 @@ private: return static_cast(*this); } - Error - copyAndFixUpBlocks(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const override { - LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n"); - for (auto &KV : Layout) { - auto &Prot = KV.first; - auto &SegLayout = KV.second; + Error fixUpBlocks(LinkGraph &G) const override { + LLVM_DEBUG(dbgs() << "Fixing up blocks:\n"); - auto SegMem = Alloc.getWorkingMemory( - static_cast(Prot)); - char *LastBlockEnd = SegMem.data(); - char *BlockDataPtr = LastBlockEnd; + for (auto *B : G.blocks()) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); - LLVM_DEBUG({ - dbgs() << " Processing segment " - << static_cast(Prot) << " [ " - << (const void *)SegMem.data() << " .. " - << (const void *)((char *)SegMem.data() + SegMem.size()) - << " ]\n Processing content sections:\n"; - }); + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : B->edges()) { - for (auto *B : SegLayout.ContentBlocks) { - LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; - // Pad to alignment/alignment-offset. - BlockDataPtr = alignToBlock(BlockDataPtr, *B); - - LLVM_DEBUG({ - dbgs() << " Bumped block pointer to " - << (const void *)BlockDataPtr << " to meet block alignment " - << B->getAlignment() << " and alignment offset " - << B->getAlignmentOffset() << "\n"; - }); - - // Zero pad up to alignment. - LLVM_DEBUG({ - if (LastBlockEnd != BlockDataPtr) - dbgs() << " Zero padding from " << (const void *)LastBlockEnd - << " to " << (const void *)BlockDataPtr << "\n"; - }); - - while (LastBlockEnd != BlockDataPtr) - *LastBlockEnd++ = 0; - - // Copy initial block content. - LLVM_DEBUG({ - dbgs() << " Copying block " << *B << " content, " - << B->getContent().size() << " bytes, from " - << (const void *)B->getContent().data() << " to " - << (const void *)BlockDataPtr << "\n"; - }); - memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); - - // Copy Block data and apply fixups. - LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - for (auto &E : B->edges()) { - - // Skip non-relocation edges. - if (!E.isRelocation()) - continue; - - // Dispatch to LinkerImpl for fixup. - if (auto Err = impl().applyFixup(*B, E, BlockDataPtr)) - return Err; - } - - // Point the block's content to the fixed up buffer. - B->setContent(StringRef(BlockDataPtr, B->getContent().size())); - - // Update block end pointer. - LastBlockEnd = BlockDataPtr + B->getContent().size(); - BlockDataPtr = LastBlockEnd; + // Dispatch to LinkerImpl for fixup. + auto *BlockData = const_cast(B->getContent().data()); + if (auto Err = impl().applyFixup(*B, E, BlockData)) + return Err; } - - // Zero pad the rest of the segment. - LLVM_DEBUG({ - dbgs() << " Zero padding end of segment from " - << (const void *)LastBlockEnd << " to " - << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; - }); - while (LastBlockEnd != SegMem.data() + SegMem.size()) - *LastBlockEnd++ = 0; } return Error::success(); diff --git a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 69ec72aae29..86d10025d33 100644 --- a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -350,6 +350,9 @@ private: class MachO_x86_64_GOTAndStubsBuilder : public BasicGOTAndStubsBuilder { public: + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder(G) {} @@ -367,7 +370,13 @@ public: void fixGOTEdge(Edge &E, Symbol &GOTEntry) { assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && "Not a GOT edge?"); - E.setKind(PCRel32); + // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is + // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to + // check for GOT optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + if (E.getKind() == PCRel32GOT) + E.setKind(PCRel32); + E.setTarget(GOTEntry); // Leave the edge addend as-is. } @@ -388,6 +397,11 @@ public: void fixExternalBranchEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + + // Set the edge kind to Branch32ToStub. We will use this to check for stub + // optimization opportunities in the optimizeMachO_x86_64_GOTAndStubs pass + // below. + E.setKind(Branch32ToStub); E.setTarget(Stub); } @@ -417,8 +431,6 @@ private: sizeof(StubContent)); } - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[6]; Section *GOTSection = nullptr; Section *StubsSection = nullptr; }; @@ -429,6 +441,89 @@ const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = { 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; } // namespace +Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == PCRel32GOTLoad) { + assert(E.getOffset() >= 3 && "GOT edge occurs too early in block"); + + // Switch the edge kind to PCRel32: Whether we change the edge target + // or not this will be the desired kind. + E.setKind(PCRel32); + + // Optimize GOT references. + auto &GOTBlock = E.getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT entry block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + // Check that this is a recognized MOV instruction. + // FIXME: Can we assume this? + constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; + if (strncmp(B->getContent().data() + E.getOffset() - 3, + reinterpret_cast(MOVQRIPRel), 2) != 0) + continue; + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits::min() && + Displacement <= std::numeric_limits::max()) { + E.setTarget(GOTTarget); + auto *BlockData = reinterpret_cast( + const_cast(B->getContent().data())); + BlockData[E.getOffset() - 2] = 0x8d; + LLVM_DEBUG({ + dbgs() << " Replaced GOT load wih LEA:\n "; + printEdge(dbgs(), *B, E, + getMachOX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } else if (E.getKind() == Branch32ToStub) { + + // Switch the edge kind to PCRel32: Whether we change the edge target + // or not this will be the desired kind. + E.setKind(Branch32); + + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == + sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits::min() && + Displacement <= std::numeric_limits::max()) { + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, + getMachOX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + + return Error::success(); +} + namespace llvm { namespace jitlink { @@ -570,6 +665,9 @@ void jitLink_MachO_x86_64(std::unique_ptr Ctx) { MachO_x86_64_GOTAndStubsBuilder(G).run(); return Error::success(); }); + + // Add GOT/Stubs optimizer pass. + Config.PostAllocationPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); } if (auto Err = Ctx->modifyPassConfig(TT, Config)) @@ -583,6 +681,8 @@ StringRef getMachOX86RelocationKindName(Edge::Kind R) { switch (R) { case Branch32: return "Branch32"; + case Branch32ToStub: + return "Branch32ToStub"; case Pointer32: return "Pointer32"; case Pointer64: diff --git a/test/ExecutionEngine/JITLink/X86/Inputs/MachO_GOTAndStubsOptimizationHelper.s b/test/ExecutionEngine/JITLink/X86/Inputs/MachO_GOTAndStubsOptimizationHelper.s new file mode 100644 index 00000000000..9cfe9a2c5f4 --- /dev/null +++ b/test/ExecutionEngine/JITLink/X86/Inputs/MachO_GOTAndStubsOptimizationHelper.s @@ -0,0 +1,8 @@ + .section __TEXT,__text,regular,pure_instructions + .macosx_version_min 10, 14 + .globl bypass_got + .p2align 4, 0x90 +bypass_got: + movq _x@GOTPCREL(%rip), %rax + +.subsections_via_symbols diff --git a/test/ExecutionEngine/JITLink/X86/MachO_GOTAndStubsOptimization.s b/test/ExecutionEngine/JITLink/X86/MachO_GOTAndStubsOptimization.s new file mode 100644 index 00000000000..98df053c9e5 --- /dev/null +++ b/test/ExecutionEngine/JITLink/X86/MachO_GOTAndStubsOptimization.s @@ -0,0 +1,31 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=x86_64-apple-macos10.9 -filetype=obj \ +# RUN: -o %t/helper.o %S/Inputs/MachO_GOTAndStubsOptimizationHelper.s +# RUN: llvm-mc -triple=x86_64-apple-macos10.9 -filetype=obj \ +# RUN: -o %t/testcase.o %s +# RUN: llvm-jitlink -noexec -slab-allocate 64Kb -entry=bypass_stub -check %s \ +# RUN: %t/testcase.o %t/helper.o +# +# Test that references to in-range GOT and stub targets can be bypassed. +# The helper file contains a function that uses the GOT for _x, and this file +# contains an external call to that function. By slab allocating the JIT memory +# we can ensure that the references and targets will be in-range of one another, +# which should cause both the GOT load and stub to be bypassed. + + .section __TEXT,__text,regular,pure_instructions + .macosx_version_min 10, 14 + .globl bypass_stub + .p2align 4, 0x90 + +# jitlink-check: decode_operand(bypass_got, 4) = _x - next_pc(bypass_got) +# jitlink-check: decode_operand(bypass_stub, 0) = bypass_got - next_pc(bypass_stub) +bypass_stub: + callq bypass_got + + .section __DATA,__data + .globl _x + .p2align 2 +_x: + .long 42 + +.subsections_via_symbols diff --git a/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s b/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s index 87ed036556e..b128d6fd9be 100644 --- a/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s +++ b/test/ExecutionEngine/JITLink/X86/MachO_x86-64_relocations.s @@ -1,6 +1,8 @@ # RUN: rm -rf %t && mkdir -p %t # RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj -o %t/macho_reloc.o %s -# RUN: llvm-jitlink -noexec -define-abs external_data=0xdeadbeef -define-abs external_func=0xcafef00d -check=%s %t/macho_reloc.o +# RUN: llvm-jitlink -noexec -define-abs external_data=0xffffffffdeadbeef \ +# RUN: -define-abs external_func=0xffffffffcafef00d \ +# RUN: -define-abs lowaddr_symbol=0x1000 -check=%s %t/macho_reloc.o .section __TEXT,__text,regular,pure_instructions @@ -170,11 +172,11 @@ named_func_addr_quad: # Check X86_64_RELOC_UNSIGNED / long / extern handling by putting the address of # an external function (defined to reside in the low 4Gb) into a long symbol. # -# jitlink-check: *{4}named_func_addr_long = external_func - .globl named_func_addr_long +# jitlink-check: *{4}named_lowaddr_symbol_long = lowaddr_symbol + .globl named_lowaddr_symbol_long .p2align 2 -named_func_addr_long: - .long external_func +named_lowaddr_symbol_long: + .long lowaddr_symbol # Check X86_64_RELOC_UNSIGNED / quad / non-extern handling by putting the # address of a local anonymous function into a quad symbol.