[JITLink][ELF/x86-64] Add support for R_X86_64_GOTPC64 and R_X86_64_GOT64.

Start adding support for ELF x86-64 large code model, PIC relocations.

include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h

@@ -26,6 +26,7 @@ enum ELFX86RelocationKind : Edge::Kind {
   Pointer64,
   Pointer64Anon,
   PCRel32,
+  PCRel64,
   PCRel32Minus1,
   PCRel32Minus2,
   PCRel32Minus4,
@@ -35,6 +36,8 @@ enum ELFX86RelocationKind : Edge::Kind {
   PCRel32Minus4Anon,
   PCRel32GOTLoad,
   PCRel32GOT,
+  PCRel64GOT,
+  GOT64,
   PCRel32TLV,
   Delta32,
   Delta64,

include/llvm/ExecutionEngine/JITLink/JITLink.h

@@ -674,7 +674,7 @@ public:
     assert((First || !Last) && "Last can not be null if start is non-null");
     return Last;
   }
-  bool isEmpty() const {
+  bool empty() const {
     assert((First || !Last) && "Last can not be null if start is non-null");
     return !First;
   }

lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h

@@ -40,18 +40,24 @@ public:
       : F(std::move(F)) {}
 
   Error operator()(LinkGraph &G) {
-    for (auto *Sym : G.external_symbols()) {
+
+    // This pass will affect the external symbols set, so copy them out into a
+    // vector and iterate over that.
+    std::vector<Symbol *> Externals(G.external_symbols().begin(),
+                                    G.external_symbols().end());
+
+    for (auto *Sym : Externals) {
       SectionRangeSymbolDesc D = F(G, *Sym);
       if (D.Sec) {
         auto &SR = getSectionRange(*D.Sec);
         if (D.IsStart) {
-          if (SR.isEmpty())
+          if (SR.empty())
             G.makeAbsolute(*Sym, 0);
           else
             G.makeDefined(*Sym, *SR.getFirstBlock(), 0, 0, Linkage::Strong,
                           Scope::Local, false);
         } else {
-          if (SR.isEmpty())
+          if (SR.empty())
             G.makeAbsolute(*Sym, 0);
           else
             G.makeDefined(*Sym, *SR.getLastBlock(),

lib/ExecutionEngine/JITLink/ELF_x86_64.cpp

@@ -40,7 +40,8 @@ public:
       PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder;
 
   bool isGOTEdgeToFix(Edge &E) const {
-    return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad;
+    return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad ||
+           E.getKind() == PCRel64GOT || E.getKind() == GOT64;
   }
 
   Symbol &createGOTEntry(Symbol &Target) {
@@ -51,14 +52,25 @@ public:
   }
 
   void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
-    assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) &&
+    // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary
-           "Not a GOT edge?");
+    // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now:
-    // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is
+    // We will use the kind to check for GOT optimization opportunities in the
-    // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to
-    // check for GOT optimization opportunities in the
     // optimizeMachO_x86_64_GOTAndStubs pass below.
-    if (E.getKind() == PCRel32GOT)
+    // If it's a GOT64 leave it as is.
+    switch (E.getKind()) {
+    case PCRel32GOT:
       E.setKind(PCRel32);
+      break;
+    case PCRel64GOT:
+      E.setKind(PCRel64);
+      break;
+    case GOT64:
+      break;
+    case PCRel32GOTLoad:
+      break;
+    default:
+      llvm_unreachable("Unexpected GOT edge kind");
+    }
 
     E.setTarget(GOTEntry);
     // Leave the edge addend as-is.
@@ -117,7 +129,8 @@ private:
   Section *StubsSection = nullptr;
 };
 
-StringRef ELFGOTSectionName = "$__GOT";
+constexpr StringRef ELFGOTSectionName = "$__GOT";
+constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
 
 const char *const DwarfSectionNames[] = {
 #define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION)        \
@@ -245,6 +258,7 @@ private:
     case ELF::R_X86_64_PC32:
       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
     case ELF::R_X86_64_PC64:
+    case ELF::R_X86_64_GOTPC64:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64;
     case ELF::R_X86_64_64:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64;
@@ -252,6 +266,10 @@ private:
     case ELF::R_X86_64_GOTPCRELX:
     case ELF::R_X86_64_REX_GOTPCRELX:
       return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad;
+    case ELF::R_X86_64_GOTPCREL64:
+      return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT;
+    case ELF::R_X86_64_GOT64:
+      return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64;
     case ELF::R_X86_64_PLT32:
       return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32;
     }
@@ -699,9 +717,57 @@ public:
   ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
                       std::unique_ptr<LinkGraph> G,
                       PassConfiguration PassConfig)
-      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
+      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
+    getPassConfig().PostAllocationPasses.push_back(
+        [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
+  }
 
 private:
+  Symbol *GOTSymbol = nullptr;
+
+  Error getOrCreateGOTSymbol(LinkGraph &G) {
+    Section *GOTSection = nullptr;
+
+    auto DefineExternalGOTSymbolIfPresent =
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
+              if (Sym.getName() == ELFGOTSymbolName)
+                if ((GOTSection = G.findSectionByName(ELFGOTSectionName))) {
+                  GOTSymbol = &Sym;
+                  return {*GOTSection, true};
+                }
+              return {};
+            });
+
+    // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
+    // external.
+    if (auto Err = DefineExternalGOTSymbolIfPresent(G))
+      return Err;
+
+    // If there's a GOT section but we didn't find an external GOT symbol...
+    if (GOTSection && !GOTSymbol) {
+
+      // Check for an existing defined symbol.
+      for (auto *Sym : GOTSection->symbols())
+        if (Sym->getName() == ELFGOTSymbolName) {
+          GOTSymbol = Sym;
+          return Error::success();
+        }
+
+      // If there's no defined symbol then create one.
+      SectionRange SR(*GOTSection);
+      if (SR.empty())
+        GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0,
+                                         Linkage::Strong, Scope::Local, true);
+      else
+        GOTSymbol =
+            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
+                                Linkage::Strong, Scope::Local, false, true);
+    }
+
+    return Error::success();
+  }
+
   Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
                    char *BlockWorkingMem) const {
     using namespace ELF_x86_64_Edges;
@@ -720,6 +786,11 @@ private:
         return makeTargetOutOfRangeError(G, B, E);
       break;
     }
+    case ELFX86RelocationKind::PCRel64: {
+      int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
+      *(little64_t *)FixupPtr = Value;
+      break;
+    }
     case ELFX86RelocationKind::Pointer64: {
       int64_t Value = E.getTarget().getAddress() + E.getAddend();
       *(ulittle64_t *)FixupPtr = Value;
@@ -751,6 +822,13 @@ private:
       *(little64_t *)FixupPtr = Value;
       break;
     }
+    case ELFX86RelocationKind::GOT64: {
+      assert(GOTSymbol && "No GOT section symbol");
+      int64_t Value =
+          E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend();
+      *(little64_t *)FixupPtr = Value;
+      break;
+    }
     default:
       LLVM_DEBUG({
         dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind())
@@ -793,9 +871,6 @@ identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
     if (auto *Sec =
             G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size())))
       return {*Sec, false};
-  } else if (SymName == "_GLOBAL_OFFSET_TABLE_") {
-    if (auto *GOTSec = G.findSectionByName(ELFGOTSectionName))
-      return {*GOTSec, true};
   }
   return {};
 }
@@ -822,13 +897,13 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
     Config.PostPrunePasses.push_back(
         PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass);
 
-    // Add GOT/Stubs optimizer pass.
-    Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs);
-
     // Resolve any external section start / end symbols.
-    Config.PreFixupPasses.push_back(
+    Config.PostAllocationPasses.push_back(
         createDefineExternalSectionStartAndEndSymbolsPass(
             identifyELFSectionStartAndEndSymbols));
+
+    // Add GOT/Stubs optimizer pass.
+    Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs);
   }
 
   if (auto Err = Ctx->modifyPassConfig(*G, Config))

lib/ExecutionEngine/JITLink/JITLinkGeneric.h

@@ -51,6 +51,12 @@ protected:
 
   using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>;
 
+  // Returns the PassConfiguration for this instance. This can be used by
+  // JITLinkerBase implementations to add late passes that reference their
+  // own data structures (e.g. for ELF implementations to locate / construct
+  // a GOT start symbol prior to fixup).
+  PassConfiguration &getPassConfig() { return Passes; }
+
   // Phase 1:
   //   1.1: Run pre-prune passes
   //   1.2: Prune graph

test/ExecutionEngine/JITLink/X86/ELF_x86-64_large_pic_relocations.s

@@ -0,0 +1,58 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj \
+# RUN:         -large-code-model -o %t/elf_lg_pic_reloc.o %s
+# RUN: llvm-jitlink -noexec -slab-allocate 100Kb -slab-address 0xfff00000 \
+# RUN:              -check %s %t/elf_lg_pic_reloc.o
+#
+# Test ELF large/PIC relocations.
+
+        .text
+        .file   "testcase.c"
+
+        # Empty main entry point.
+        .globl  main
+        .p2align        4, 0x90
+        .type   main,@function
+main:
+        retq
+
+        .size   main, .-main
+
+# Test R_X86_64_GOTPC64 handling. We want to check that the offset of the
+# operand is the 64-bit delta to the start of the GOT.
+# jitlink-check: decode_operand(test_gotpc64, 1) = \
+# jitlink-check:   _GLOBAL_OFFSET_TABLE_ - test_lg_pic_GOT
+# jitlink-check: decode_operand(test_got64, 1) = \
+# jitlink-check:   got_addr(elf_lg_pic_reloc.o, named_data) - \
+# jitlink-check:     _GLOBAL_OFFSET_TABLE_
+        .globl test_lg_pic_GOT
+        .p2align    4, 0x90
+        .type   test_lg_pic_GOT,@function
+test_lg_pic_GOT:
+.L0$pb:
+        leaq    .L0$pb(%rip), %rax
+
+        .globl test_gotpc64
+test_gotpc64:
+        movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx
+        .size   test_gotpc64, .-test_gotpc64
+
+        addq    %rax, %rcx
+        .globl test_got64
+test_got64:
+        movabsq $named_data@GOT, %rax
+        .size   test_got64, .-test_got64
+
+        .size   test_lg_pic_GOT, .-test_lg_pic_GOT
+
+        .data
+
+        .type   named_data,@object
+        .p2align        3
+named_data:
+        .quad   42
+        .size   named_data, 8
+
+        .ident  "clang version 10.0.0-4ubuntu1 "
+        .section        ".note.GNU-stack","",@progbits
+        .addrsig

test/ExecutionEngine/JITLink/X86/ELF_x86-64_small_pic_relocations.s

@@ -1,12 +1,13 @@
 # RUN: rm -rf %t && mkdir -p %t
-# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj -o %t/elf_reloc.o %s
+# RUN: llvm-mc -triple=x86_64-unknown-linux -position-independent -filetype=obj \
+# RUN:         -o %t/elf_sm_pic_reloc.o %s
 # RUN: llvm-jitlink -noexec -slab-allocate 100Kb -slab-address 0xfff00000 \
 # RUN:              -define-abs external_data=0x1 \
 # RUN:              -define-abs extern_in_range32=0xffe00000 \
 # RUN:              -define-abs extern_out_of_range32=0x7fff00000000 \
-# RUN:              -check %s %t/elf_reloc.o
+# RUN:              -check %s %t/elf_sm_pic_reloc.o
 #
-# Test standard ELF relocations.
+# Test ELF small/PIC relocations.
 
         .text
         .file   "testcase.c"
@@ -55,7 +56,8 @@ test_call_local:
 # resolution, the target turns out to be in-range from the callsite and so the
 # edge is relaxed in post-allocation optimization.
 #
-# jitlink-check: decode_operand(test_call_extern, 0) = extern_in_range32 - next_pc(test_call_extern)
+# jitlink-check: decode_operand(test_call_extern, 0) = \
+# jitlink-check:     extern_in_range32 - next_pc(test_call_extern)
         .globl  test_call_extern
         .p2align       4, 0x90
         .type   test_call_extern,@function
@@ -70,8 +72,10 @@ test_call_extern:
 # entry.
 #
 # jitlink-check: decode_operand(test_call_extern_plt, 0) = \
-# jitlink-check:     stub_addr(elf_reloc.o, extern_out_of_range32) - next_pc(test_call_extern_plt)
+# jitlink-check:     stub_addr(elf_sm_pic_reloc.o, extern_out_of_range32) - \
-# jitlink-check: *{8}(got_addr(elf_reloc.o, extern_out_of_range32)) = extern_out_of_range32
+# jitlink-check:        next_pc(test_call_extern_plt)
+# jitlink-check: *{8}(got_addr(elf_sm_pic_reloc.o, extern_out_of_range32)) = \
+# jitlink-check:     extern_out_of_range32
         .globl  test_call_extern_plt
         .p2align       4, 0x90
         .type   test_call_extern_plt,@function
@@ -82,8 +86,9 @@ test_call_extern_plt:
 
 # Test GOTPCREL handling. We want to check both the offset to the GOT entry and its
 # contents.
-# jitlink-check: decode_operand(test_gotpcrel, 4) = got_addr(elf_reloc.o, named_data) - next_pc(test_gotpcrel)
+# jitlink-check: decode_operand(test_gotpcrel, 4) = \
-# jitlink-check: *{8}(got_addr(elf_reloc.o, named_data)) = named_data
+# jitlink-check:     got_addr(elf_sm_pic_reloc.o, named_data) - next_pc(test_gotpcrel)
+# jitlink-check: *{8}(got_addr(elf_sm_pic_reloc.o, named_data)) = named_data
 
         .globl test_gotpcrel
         .p2align      4, 0x90
@@ -96,7 +101,7 @@ test_gotpcrel:
 # Test REX_GOTPCRELX handling. We want to check both the offset to the GOT entry and its
 # contents.
 # jitlink-check: decode_operand(test_rex_gotpcrelx, 4) = \
-# jitlink-check:   got_addr(elf_reloc.o, external_data) - next_pc(test_rex_gotpcrelx)
+# jitlink-check:   got_addr(elf_sm_pic_reloc.o, external_data) - next_pc(test_rex_gotpcrelx)
 
         .globl test_rex_gotpcrelx
         .p2align      4, 0x90

tools/llvm-jitlink/llvm-jitlink-elf.cpp

@@ -133,8 +133,9 @@ Error registerELFGraphInfo(Session &S, LinkGraph &G) {
         else
           return TS.takeError();
         SectionContainsContent = true;
-      } else if (Sym->hasName()) {
+      }
-        dbgs() << "Symbol: " << Sym->getName() << "\n";
+
+      if (Sym->hasName()) {
         if (Sym->isSymbolZeroFill()) {
           S.SymbolInfos[Sym->getName()] = {Sym->getSize(), Sym->getAddress()};
           SectionContainsZeroFill = true;