Re-re-re-apply "[ORC][ORC-RT] Add initial native-TLV support to MachOPlatform."

The ccache builders have recevied a config update that should eliminate the
build issues seen previously.

include/llvm/ExecutionEngine/Orc/MachOPlatform.h

@@ -29,6 +29,7 @@ namespace orc {
 
 struct MachOPerObjectSectionsToRegister {
   ExecutorAddressRange EHFrameSection;
+  ExecutorAddressRange ThreadDataSection;
 };
 
 struct MachOJITDylibInitializers {
@@ -158,14 +159,16 @@ private:
     void addMachOHeaderSupportPasses(MaterializationResponsibility &MR,
                                      jitlink::PassConfiguration &Config);
 
-    void addEHSupportPasses(MaterializationResponsibility &MR,
+    void addEHAndTLVSupportPasses(MaterializationResponsibility &MR,
-                            jitlink::PassConfiguration &Config);
+                                  jitlink::PassConfiguration &Config);
 
     Error preserveInitSections(jitlink::LinkGraph &G,
                                MaterializationResponsibility &MR);
 
     Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD);
 
+    Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD);
+
     std::mutex PluginMutex;
     MachOPlatform ∓
     InitSymbolDepMap InitSymbolDeps;
@@ -213,6 +216,8 @@ private:
 
   Error registerPerObjectSections(const MachOPerObjectSectionsToRegister &POSR);
 
+  Expected<uint64_t> createPThreadKey();
+
   ExecutionSession &ES;
   ObjectLinkingLayer &ObjLinkingLayer;
   ExecutorProcessControl &EPC;
@@ -223,6 +228,7 @@ private:
   ExecutorAddress orc_rt_macho_platform_bootstrap;
   ExecutorAddress orc_rt_macho_platform_shutdown;
   ExecutorAddress orc_rt_macho_register_object_sections;
+  ExecutorAddress orc_rt_macho_create_pthread_key;
 
   DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols;
 
@@ -233,11 +239,13 @@ private:
   std::vector<MachOPerObjectSectionsToRegister> BootstrapPOSRs;
 
   DenseMap<JITTargetAddress, JITDylib *> HeaderAddrToJITDylib;
+  DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey;
 };
 
 namespace shared {
 
-using SPSMachOPerObjectSectionsToRegister = SPSTuple<SPSExecutorAddressRange>;
+using SPSMachOPerObjectSectionsToRegister =
+    SPSTuple<SPSExecutorAddressRange, SPSExecutorAddressRange>;
 
 template <>
 class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister,
@@ -246,19 +254,19 @@ class SPSSerializationTraits<SPSMachOPerObjectSectionsToRegister,
 public:
   static size_t size(const MachOPerObjectSectionsToRegister &MOPOSR) {
     return SPSMachOPerObjectSectionsToRegister::AsArgList::size(
-        MOPOSR.EHFrameSection);
+        MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
   }
 
   static bool serialize(SPSOutputBuffer &OB,
                         const MachOPerObjectSectionsToRegister &MOPOSR) {
     return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize(
-        OB, MOPOSR.EHFrameSection);
+        OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
   }
 
   static bool deserialize(SPSInputBuffer &IB,
                           MachOPerObjectSectionsToRegister &MOPOSR) {
     return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize(
-        IB, MOPOSR.EHFrameSection);
+        IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection);
   }
 };
 

lib/ExecutionEngine/JITLink/MachO_x86_64.cpp

@@ -313,6 +313,14 @@ private:
           Addend = *(const little32_t *)FixupContent - 4;
           Kind = x86_64::RequestGOTAndTransformToDelta32;
           break;
+        case MachOPCRel32TLV:
+          if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+            TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+          else
+            return TargetSymbolOrErr.takeError();
+          Addend = *(const little32_t *)FixupContent;
+          Kind = x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable;
+          break;
         case MachOPointer32:
           if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
             TargetSymbol = TargetSymbolOrErr->GraphSymbol;
@@ -392,9 +400,6 @@ private:
           assert(TargetSymbol && "No target symbol from parsePairRelocation?");
           break;
         }
-        case MachOPCRel32TLV:
-          return make_error<JITLinkError>(
-              "MachO TLV relocations not yet supported");
         }
 
         LLVM_DEBUG({

lib/ExecutionEngine/Orc/MachOPlatform.cpp

@@ -128,6 +128,9 @@ constexpr MachOHeaderMaterializationUnit::HeaderSymbol
 
 StringRef EHFrameSectionName = "__TEXT,__eh_frame";
 StringRef ModInitFuncSectionName = "__DATA,__mod_init_func";
+StringRef ThreadBSSSectionName = "__DATA,__thread_bss";
+StringRef ThreadDataSectionName = "__DATA,__thread_data";
+StringRef ThreadVarsSectionName = "__DATA,__thread_vars";
 
 StringRef InitSectionNames[] = {ModInitFuncSectionName};
 
@@ -478,7 +481,8 @@ Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) {
       {"___orc_rt_macho_platform_bootstrap", &orc_rt_macho_platform_bootstrap},
       {"___orc_rt_macho_platform_shutdown", &orc_rt_macho_platform_shutdown},
       {"___orc_rt_macho_register_object_sections",
-       &orc_rt_macho_register_object_sections}};
+       &orc_rt_macho_register_object_sections},
+      {"___orc_rt_macho_create_pthread_key", &orc_rt_macho_create_pthread_key}};
 
   SymbolLookupSet RuntimeSymbols;
   std::vector<std::pair<SymbolStringPtr, ExecutorAddress *>> AddrsToRecord;
@@ -573,6 +577,20 @@ Error MachOPlatform::registerPerObjectSections(
   return ErrResult;
 }
 
+Expected<uint64_t> MachOPlatform::createPThreadKey() {
+  if (!orc_rt_macho_create_pthread_key)
+    return make_error<StringError>(
+        "Attempting to create pthread key in target, but runtime support has "
+        "not been loaded yet",
+        inconvertibleErrorCode());
+
+  Expected<uint64_t> Result(0);
+  if (auto Err = EPC.runSPSWrapper<SPSExpected<uint64_t>(void)>(
+          orc_rt_macho_create_pthread_key.getValue(), Result))
+    return std::move(Err);
+  return Result;
+}
+
 void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
     jitlink::PassConfiguration &Config) {
@@ -590,8 +608,8 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
   if (MR.getInitializerSymbol())
     addInitializerSupportPasses(MR, Config);
 
-  // Add passes for eh-frame support.
+  // Add passes for eh-frame and TLV support.
-  addEHSupportPasses(MR, Config);
+  addEHAndTLVSupportPasses(MR, Config);
 }
 
 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
@@ -645,10 +663,18 @@ void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses(
   });
 }
 
-void MachOPlatform::MachOPlatformPlugin::addEHSupportPasses(
+void MachOPlatform::MachOPlatformPlugin::addEHAndTLVSupportPasses(
     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
 
-  // Add a pass to register the final addresses of the eh-frame sections
+  // Insert TLV lowering at the start of the PostPrunePasses, since we want
+  // it to run before GOT/PLT lowering.
+  Config.PostPrunePasses.insert(
+      Config.PostPrunePasses.begin(),
+      [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
+        return fixTLVSectionsAndEdges(G, JD);
+      });
+
+  // Add a pass to register the final addresses of the eh-frame and TLV sections
   // with the runtime.
   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
     MachOPerObjectSectionsToRegister POSR;
@@ -660,7 +686,33 @@ void MachOPlatform::MachOPlatformPlugin::addEHSupportPasses(
                                ExecutorAddress(R.getEnd())};
     }
 
-    if (POSR.EHFrameSection.StartAddress) {
+    // Get a pointer to the thread data section if there is one. It will be used
+    // below.
+    jitlink::Section *ThreadDataSection =
+        G.findSectionByName(ThreadDataSectionName);
+
+    // Handle thread BSS section if there is one.
+    if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
+      // If there's already a thread data section in this graph then merge the
+      // thread BSS section content into it, otherwise just treat the thread
+      // BSS section as the thread data section.
+      if (ThreadDataSection)
+        G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
+      else
+        ThreadDataSection = ThreadBSSSection;
+    }
+
+    // Having merged thread BSS (if present) and thread data (if present),
+    // record the resulting section range.
+    if (ThreadDataSection) {
+      jitlink::SectionRange R(*ThreadDataSection);
+      if (!R.empty())
+        POSR.ThreadDataSection = {ExecutorAddress(R.getStart()),
+                                  ExecutorAddress(R.getEnd())};
+    }
+
+    if (POSR.EHFrameSection.StartAddress ||
+        POSR.ThreadDataSection.StartAddress) {
 
       // If we're still bootstrapping the runtime then just record this
       // frame for now.
@@ -738,5 +790,61 @@ Error MachOPlatform::MachOPlatformPlugin::registerInitSections(
   return MP.registerInitInfo(JD, InitSections);
 }
 
+Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
+    jitlink::LinkGraph &G, JITDylib &JD) {
+
+  // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr.
+  for (auto *Sym : G.external_symbols())
+    if (Sym->getName() == "__tlv_bootstrap") {
+      Sym->setName("___orc_rt_macho_tlv_get_addr");
+      break;
+    }
+
+  // Store key in __thread_vars struct fields.
+  if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) {
+    Optional<uint64_t> Key;
+    {
+      std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
+      auto I = MP.JITDylibToPThreadKey.find(&JD);
+      if (I != MP.JITDylibToPThreadKey.end())
+        Key = I->second;
+    }
+
+    if (!Key) {
+      if (auto KeyOrErr = MP.createPThreadKey())
+        Key = *KeyOrErr;
+      else
+        return KeyOrErr.takeError();
+    }
+
+    uint64_t PlatformKeyBits =
+        support::endian::byte_swap(*Key, G.getEndianness());
+
+    for (auto *B : ThreadDataSec->blocks()) {
+      if (B->getSize() != 3 * G.getPointerSize())
+        return make_error<StringError>("__thread_vars block at " +
+                                           formatv("{0:x}", B->getAddress()) +
+                                           " has unexpected size",
+                                       inconvertibleErrorCode());
+
+      auto NewBlockContent = G.allocateBuffer(B->getSize());
+      llvm::copy(B->getContent(), NewBlockContent.data());
+      memcpy(NewBlockContent.data() + G.getPointerSize(), &PlatformKeyBits,
+             G.getPointerSize());
+      B->setContent(NewBlockContent);
+    }
+  }
+
+  // Transform any TLV edges into GOT edges.
+  for (auto *B : G.blocks())
+    for (auto &E : B->edges())
+      if (E.getKind() ==
+          jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable)
+        E.setKind(
+            jitlink::x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable);
+
+  return Error::success();
+}
+
 } // End namespace orc.
 } // End namespace llvm.