From a020f7f14c798f2d629f181922c73e7bb870de97 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Wed, 14 Jul 2021 21:09:36 +1000 Subject: [PATCH] [ORC][ORC-RT] Introduce ORC-runtime based MachO-Platform. Adds support for MachO static initializers/deinitializers and eh-frame registration via the ORC runtime. This commit introduces cooperative support code into the ORC runtime and ORC LLVM libraries (especially the MachOPlatform class) to support macho runtime features for JIT'd code. This commit introduces support for static initializers, static destructors (via cxa_atexit interposition), and eh-frame registration. Near-future commits will add support for MachO native thread-local variables, and language runtime registration (e.g. for Objective-C and Swift). The llvm-jitlink tool is updated to use the ORC runtime where available, and regression tests for the new MachOPlatform support are added to compiler-rt. Notable changes on the ORC runtime side: 1. The new macho_platform.h / macho_platform.cpp files contain the bulk of the runtime-side support. This includes eh-frame registration; jit versions of dlopen, dlsym, and dlclose; a cxa_atexit interpose to record static destructors, and an '__orc_rt_macho_run_program' function that defines running a JIT'd MachO program in terms of the jit- dlopen/dlsym/dlclose functions. 2. Replaces JITTargetAddress (and casting operations) with ExecutorAddress (copied from LLVM) to improve type-safety of address management. 3. Adds serialization support for ExecutorAddress and unordered_map types to the runtime-side Simple Packed Serialization code. 4. Adds orc-runtime regression tests to ensure that static initializers and cxa-atexit interposes work as expected. Notable changes on the LLVM side: 1. The MachOPlatform class is updated to: 1.1. Load the ORC runtime into the ExecutionSession. 1.2. Set up standard aliases for macho-specific runtime functions. E.g. ___cxa_atexit -> ___orc_rt_macho_cxa_atexit. 1.3. Install the MachOPlatformPlugin to scrape LinkGraphs for information needed to support MachO features (e.g. eh-frames, mod-inits), and communicate this information to the runtime. 1.4. Provide entry-points that the runtime can call to request initializers, perform symbol lookup, and request deinitialiers (the latter is implemented as an empty placeholder as macho object deinits are rarely used). 1.5. Create a MachO header object for each JITDylib (defining the __mh_header and __dso_handle symbols). 2. The llvm-jitlink tool (and llvm-jitlink-executor) are updated to use the runtime when available. 3. A `lookupInitSymbolsAsync` method is added to the Platform base class. This can be used to issue an async lookup for initializer symbols. The existing `lookupInitSymbols` method is retained (the GenericIRPlatform code is still using it), but is deprecated and will be removed soon. 4. JIT-dispatch support code is added to ExecutorProcessControl. The JIT-dispatch system allows handlers in the JIT process to be associated with 'tag' symbols in the executor, and allows the executor to make remote procedure calls back to the JIT process (via __orc_rt_jit_dispatch) using those tags. The primary use case is ORC runtime code that needs to call bakc to handlers in orc::Platform subclasses. E.g. __orc_rt_macho_jit_dlopen calling back to MachOPlatform::rt_getInitializers using __orc_rt_macho_get_initializers_tag. (The system is generic however, and could be used by non-runtime code). The new ExecutorProcessControl::JITDispatchInfo struct provides the address (in the executor) of the jit-dispatch function and a jit-dispatch context object, and implementations of the dispatch function are added to SelfExecutorProcessControl and OrcRPCExecutorProcessControl. 5. OrcRPCTPCServer is updated to support JIT-dispatch calls over ORC-RPC. 6. Serialization support for StringMap is added to the LLVM-side Simple Packed Serialization code. 7. A JITLink::allocateBuffer operation is introduced to allocate writable memory attached to the graph. This is used by the MachO header synthesis code, and will be generically useful for other clients who want to create new graph content from scratch. --- .../llvm/ExecutionEngine/JITLink/JITLink.h | 6 + include/llvm/ExecutionEngine/Orc/Core.h | 10 + .../Orc/ExecutorProcessControl.h | 36 +- .../ExecutionEngine/Orc/LLVMSPSSerializers.h | 69 ++ .../llvm/ExecutionEngine/Orc/MachOPlatform.h | 295 ++++-- .../Orc/OrcRPCExecutorProcessControl.h | 40 +- .../Orc/Shared/SimplePackedSerialization.h | 1 + .../Orc/Shared/WrapperFunctionUtils.h | 6 +- .../Orc/TargetProcess/OrcRPCTPCServer.h | 126 ++- lib/ExecutionEngine/Orc/Core.cpp | 43 + .../Orc/ExecutorProcessControl.cpp | 29 + lib/ExecutionEngine/Orc/MachOPlatform.cpp | 857 +++++++++++++----- .../llvm-jitlink-executor.cpp | 2 +- tools/llvm-jitlink/llvm-jitlink.cpp | 231 +++-- tools/llvm-jitlink/llvm-jitlink.h | 2 +- .../Orc/SimplePackedSerializationTest.cpp | 7 + 16 files changed, 1337 insertions(+), 423 deletions(-) create mode 100644 include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index 15b5286b869..81d8dbee480 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -916,6 +916,12 @@ public: const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); } + /// Allocate a mutable buffer of the given size using the LinkGraph's + /// allocator. + MutableArrayRef allocateBuffer(size_t Size) { + return {Allocator.Allocate(Size), Size}; + } + /// Allocate a copy of the given string using the LinkGraph's allocator. /// This can be useful when renaming symbols or adding new content to the /// graph. diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 9219e288015..159553b5559 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -1224,9 +1224,19 @@ public: /// A utility function for looking up initializer symbols. Performs a blocking /// lookup for the given symbols in each of the given JITDylibs. + /// + /// Note: This function is deprecated and will be removed in the near future. static Expected> lookupInitSymbols(ExecutionSession &ES, const DenseMap &InitSyms); + + /// Performs an async lookup for the the given symbols in each of the given + /// JITDylibs, calling the given handler with the compound result map once + /// all lookups have completed. + static void + lookupInitSymbolsAsync(unique_function OnComplete, + ExecutionSession &ES, + const DenseMap &InitSyms); }; /// Represents an abstract task for ORC to run. diff --git a/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h index 566637e1044..f8d6192706a 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h @@ -18,6 +18,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" #include "llvm/Support/DynamicLibrary.h" @@ -113,6 +114,13 @@ public: const SymbolLookupSet &Symbols; }; + /// Contains the address of the dispatch function and context that the ORC + /// runtime can use to call functions in the JIT. + struct JITDispatchInfo { + ExecutorAddress JITDispatchFunctionAddress; + ExecutorAddress JITDispatchContextAddress; + }; + virtual ~ExecutorProcessControl(); /// Intern a symbol name in the SymbolStringPool. @@ -127,6 +135,9 @@ public: /// Get the page size for the target process. unsigned getPageSize() const { return PageSize; } + /// Get the JIT dispatch function and context address for the executor. + const JITDispatchInfo &getJITDispatchInfo() const { return JDI; } + /// Return a MemoryAccess object for the target process. MemoryAccess &getMemoryAccess() const { return *MemAccess; } @@ -198,14 +209,17 @@ public: /// Run a wrapper function using SPS to serialize the arguments and /// deserialize the results. - template - Error runSPSWrapper(JITTargetAddress WrapperFnAddr, RetT &RetVal, - const ArgTs &...Args) { + /// + /// If SPSSignature is a non-void function signature then the second argument + /// (the first in the Args list) should be a reference to a return value. + template + Error runSPSWrapper(JITTargetAddress WrapperFnAddr, + WrapperCallArgTs &&...WrapperCallArgs) { return shared::WrapperFunction::call( [this, WrapperFnAddr](const char *ArgData, size_t ArgSize) { return runWrapper(WrapperFnAddr, ArrayRef(ArgData, ArgSize)); }, - RetVal, Args...); + std::forward(WrapperCallArgs)...); } /// Wrap a handler that takes concrete argument types (and a sender for a @@ -223,6 +237,15 @@ public: }; } + template + static AsyncWrapperFunction + wrapAsyncWithSPS(ClassT *Instance, void (ClassT::*Method)(MethodArgTs...)) { + return wrapAsyncWithSPS( + [Instance, Method](MethodArgTs &&...MethodArgs) { + (Instance->*Method)(std::forward(MethodArgs)...); + }); + } + /// For each symbol name, associate the AsyncWrapperFunction implementation /// value with the address of that symbol. /// @@ -250,6 +273,7 @@ protected: std::shared_ptr SSP; Triple TargetTriple; unsigned PageSize = 0; + JITDispatchInfo JDI; MemoryAccess *MemAccess = nullptr; jitlink::JITLinkMemoryManager *MemMgr = nullptr; @@ -318,6 +342,10 @@ private: void writeBuffers(ArrayRef Ws, WriteResultFn OnWriteComplete) override; + static shared::detail::CWrapperFunctionResult + jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag, + const char *Data, size_t Size); + std::unique_ptr OwnedMemMgr; char GlobalManglingPrefix = 0; std::vector> DynamicLibraries; diff --git a/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h b/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h new file mode 100644 index 00000000000..f3d616deae8 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h @@ -0,0 +1,69 @@ +//===-- LLVMSPSSerializers.h - SPS serialization for LLVM types -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// SPS Serialization for common LLVM types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H +#define LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" + +namespace llvm { +namespace orc { +namespace shared { + +template +class SPSSerializationTraits>, + StringMap> { +public: + static size_t size(const StringMap &M) { + size_t Sz = SPSArgList::size(static_cast(M.size())); + for (auto &E : M) + Sz += SPSArgList::size(E.first(), E.second); + return Sz; + } + + static bool serialize(SPSOutputBuffer &OB, const StringMap &M) { + if (!SPSArgList::serialize(OB, static_cast(M.size()))) + return false; + + for (auto &E : M) + if (!SPSArgList::serialize(OB, E.first(), E.second)) + return false; + + return true; + } + + static bool deserialize(SPSInputBuffer &IB, StringMap &M) { + uint64_t Size; + assert(M.empty() && "M already contains elements"); + + if (!SPSArgList::deserialize(IB, Size)) + return false; + + while (Size--) { + StringRef S; + ValueT V; + if (!SPSArgList::deserialize(IB, S, V)) + return false; + if (!M.insert(std::make_pair(S, V)).second) + return false; + } + + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H diff --git a/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 10d0f90510c..50f26b399a6 100644 --- a/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -15,7 +15,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" @@ -26,83 +27,106 @@ namespace llvm { namespace orc { +struct MachOPerObjectSectionsToRegister { + ExecutorAddressRange EHFrameSection; +}; -class MachOJITDylibInitializers { -public: - using RawPointerSectionList = std::vector; +struct MachOJITDylibInitializers { + using SectionList = std::vector; - void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) { - this->ObjCImageInfoAddr = ObjCImageInfoAddr; - } + MachOJITDylibInitializers(std::string Name, + ExecutorAddress MachOHeaderAddress) + : Name(std::move(Name)), + MachOHeaderAddress(std::move(MachOHeaderAddress)) {} - void addModInitsSection(ExecutorAddressRange ModInit) { - ModInitSections.push_back(std::move(ModInit)); - } + std::string Name; + ExecutorAddress MachOHeaderAddress; - const RawPointerSectionList &getModInitsSections() const { - return ModInitSections; - } - - void addObjCSelRefsSection(ExecutorAddressRange ObjCSelRefs) { - ObjCSelRefsSections.push_back(std::move(ObjCSelRefs)); - } - - const RawPointerSectionList &getObjCSelRefsSections() const { - return ObjCSelRefsSections; - } - - void addObjCClassListSection(ExecutorAddressRange ObjCClassList) { - ObjCClassListSections.push_back(std::move(ObjCClassList)); - } - - const RawPointerSectionList &getObjCClassListSections() const { - return ObjCClassListSections; - } - - void runModInits() const; - void registerObjCSelectors() const; - Error registerObjCClasses() const; - -private: - - JITTargetAddress ObjCImageInfoAddr; - RawPointerSectionList ModInitSections; - RawPointerSectionList ObjCSelRefsSections; - RawPointerSectionList ObjCClassListSections; + StringMap InitSections; }; class MachOJITDylibDeinitializers {}; +using MachOJITDylibInitializerSequence = std::vector; + +using MachOJITDylibDeinitializerSequence = + std::vector; + /// Mediates between MachO initialization and ExecutionSession state. class MachOPlatform : public Platform { public: - using InitializerSequence = - std::vector>; - - using DeinitializerSequence = - std::vector>; - - MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr StandardSymbolsObject); + /// Try to create a MachOPlatform instance, adding the ORC runtime to the + /// given JITDylib. + /// + /// The ORC runtime requires access to a number of symbols in libc++, and + /// requires access to symbols in libobjc, and libswiftCore to support + /// Objective-C and Swift code. It is up to the caller to ensure that the + /// requried symbols can be referenced by code added to PlatformJD. The + /// standard way to achieve this is to first attach dynamic library search + /// generators for either the given process, or for the specific required + /// libraries, to PlatformJD, then to create the platform instance: + /// + /// \code{.cpp} + /// auto &PlatformJD = ES.createBareJITDylib("stdlib"); + /// PlatformJD.addGenerator( + /// ExitOnErr(EPCDynamicLibrarySearchGenerator + /// ::GetForTargetProcess(EPC))); + /// ES.setPlatform( + /// ExitOnErr(MachOPlatform::Create(ES, ObjLayer, EPC, PlatformJD, + /// "/path/to/orc/runtime"))); + /// \endcode + /// + /// Alternatively, these symbols could be added to another JITDylib that + /// PlatformJD links against. + /// + /// Clients are also responsible for ensuring that any JIT'd code that + /// depends on runtime functions (including any code using TLV or static + /// destructors) can reference the runtime symbols. This is usually achieved + /// by linking any JITDylibs containing regular code against + /// PlatformJD. + /// + /// By default, MachOPlatform will add the set of aliases returned by the + /// standardPlatformAliases function. This includes both required aliases + /// (e.g. __cxa_atexit -> __orc_rt_macho_cxa_atexit for static destructor + /// support), and optional aliases that provide JIT versions of common + /// functions (e.g. dlopen -> __orc_rt_macho_jit_dlopen). Clients can + /// override these defaults by passing a non-None value for the + /// RuntimeAliases function, in which case the client is responsible for + /// setting up all aliases (including the required ones). + static Expected> + Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + const char *OrcRuntimePath, + Optional RuntimeAliases = None); ExecutionSession &getExecutionSession() const { return ES; } + ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + ExecutorProcessControl &getExecutorProcessControl() const { return EPC; } Error setupJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) override; Error notifyRemoving(ResourceTracker &RT) override; - Expected getInitializerSequence(JITDylib &JD); + /// Returns an AliasMap containing the default aliases for the MachOPlatform. + /// This can be modified by clients when constructing the platform to add + /// or remove aliases. + static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); - Expected getDeinitializerSequence(JITDylib &JD); + /// Returns the array of required CXX aliases. + static ArrayRef> requiredCXXAliases(); + + /// Returns the array of standard runtime utility aliases for MachO. + static ArrayRef> + standardRuntimeUtilityAliases(); private: - // This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func, - // __objc_classlist and __sel_ref sections and records their extents so that - // they can be run in the target process. - class InitScraperPlugin : public ObjectLinkingLayer::Plugin { + // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO + // platform features including initializers, exceptions, TLV, and language + // runtime registration. + class MachOPlatformPlugin : public ObjectLinkingLayer::Plugin { public: - InitScraperPlugin(MachOPlatform &MP) : MP(MP) {} + MachOPlatformPlugin(MachOPlatform &MP) : MP(MP) {} void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, @@ -128,36 +152,173 @@ private: using InitSymbolDepMap = DenseMap; - void preserveInitSectionIfPresent(JITLinkSymbolSet &Symbols, - jitlink::LinkGraph &G, - StringRef SectionName); + void addInitializerSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); - Error processObjCImageInfo(jitlink::LinkGraph &G, + void addMachOHeaderSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + void addEHSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + Error preserveInitSections(jitlink::LinkGraph &G, MaterializationResponsibility &MR); - std::mutex InitScraperMutex; + Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); + + std::mutex PluginMutex; MachOPlatform &MP; - DenseMap> ObjCImageInfos; InitSymbolDepMap InitSymbolDeps; }; - void registerInitInfo(JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, - ExecutorAddressRange ModInits, - ExecutorAddressRange ObjCSelRefs, - ExecutorAddressRange ObjCClassList); + using SendInitializerSequenceFn = + unique_function)>; + + using SendDeinitializerSequenceFn = + unique_function)>; + + using SendSymbolAddressFn = unique_function)>; + + static bool supportedTarget(const Triple &TT); + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, + Error &Err); + + // Associate MachOPlatform JIT-side runtime support functions with handlers. + Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); + + void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, + JITDylib &JD, + std::vector DFSLinkOrder); + + void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, + JITDylib &JD); + + void rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName); + + void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle); + + void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddress Handle, + StringRef SymbolName); + + // Records the addresses of runtime symbols used by the platform. + Error bootstrapMachORuntime(JITDylib &PlatformJD); + + Error registerInitInfo(JITDylib &JD, + ArrayRef InitSections); + + Error registerPerObjectSections(const MachOPerObjectSectionsToRegister &POSR); ExecutionSession &ES; ObjectLinkingLayer &ObjLinkingLayer; - std::unique_ptr StandardSymbolsObject; + ExecutorProcessControl &EPC; + + SymbolStringPtr MachOHeaderStartSymbol; + std::atomic RuntimeBootstrapped{false}; + + ExecutorAddress orc_rt_macho_platform_bootstrap; + ExecutorAddress orc_rt_macho_platform_shutdown; + ExecutorAddress orc_rt_macho_register_object_sections; DenseMap RegisteredInitSymbols; // InitSeqs gets its own mutex to avoid locking the whole session when // aggregating data from the jitlink. - std::mutex InitSeqsMutex; + std::mutex PlatformMutex; DenseMap InitSeqs; + std::vector BootstrapPOSRs; + + DenseMap HeaderAddrToJITDylib; }; +namespace shared { + +using SPSMachOPerObjectSectionsToRegister = SPSTuple; + +template <> +class SPSSerializationTraits { + +public: + static size_t size(const MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection); + } +}; + +using SPSNamedExecutorAddressRangeSequenceMap = + SPSSequence>; + +using SPSMachOJITDylibInitializers = + SPSTuple; + +using SPSMachOJITDylibInitializerSequence = + SPSSequence; + +/// Serialization traits for MachOJITDylibInitializers. +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } +}; + +using SPSMachOJITDylibDeinitializers = SPSEmpty; + +using SPSMachOJITDylibDeinitializerSequence = + SPSSequence; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibDeinitializers &MOJDDs) { + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibDeinitializers &MOJDDs) { + MOJDDs = MachOJITDylibDeinitializers(); + return true; + } +}; + +} // end namespace shared } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h b/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h index 69e37f9af9e..fedd79c9314 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h @@ -283,7 +283,11 @@ public: OrcRPCExecutorProcessControlBase(std::shared_ptr SSP, RPCEndpointT &EP, ErrorReporter ReportError) : ExecutorProcessControl(std::move(SSP)), - ReportError(std::move(ReportError)), EP(EP) {} + ReportError(std::move(ReportError)), EP(EP) { + using ThisT = OrcRPCExecutorProcessControlBase; + EP.template addAsyncHandler(*this, + &ThisT::runWrapperInJIT); + } void reportError(Error Err) { ReportError(std::move(Err)); } @@ -396,20 +400,32 @@ protected: /// Subclasses must call this during construction to initialize the /// TargetTriple and PageSize members. Error initializeORCRPCEPCBase() { - if (auto TripleOrErr = EP.template callB()) - TargetTriple = Triple(*TripleOrErr); - else - return TripleOrErr.takeError(); - - if (auto PageSizeOrErr = EP.template callB()) - PageSize = *PageSizeOrErr; - else - return PageSizeOrErr.takeError(); - - return Error::success(); + if (auto EPI = EP.template callB()) { + this->TargetTriple = Triple(EPI->Triple); + this->PageSize = PageSize; + this->JDI = {ExecutorAddress(EPI->DispatchFuncAddr), + ExecutorAddress(EPI->DispatchCtxAddr)}; + return Error::success(); + } else + return EPI.takeError(); } private: + Error runWrapperInJIT( + std::function)> SendResult, + JITTargetAddress FunctionTag, std::vector ArgBuffer) { + + runJITSideWrapperFunction( + [this, SendResult = std::move(SendResult)]( + Expected R) { + if (auto Err = SendResult(std::move(R))) + ReportError(std::move(Err)); + }, + FunctionTag, + {reinterpret_cast(ArgBuffer.data()), ArgBuffer.size()}); + return Error::success(); + } + ErrorReporter ReportError; RPCEndpointT &EP; }; diff --git a/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h index a2ad84647d2..854f1098d5a 100644 --- a/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h +++ b/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -85,6 +85,7 @@ public: bool skip(size_t Size) { if (Size > Remaining) return false; + Buffer += Size; Remaining -= Size; return true; } diff --git a/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h b/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h index ceaea1d2b20..2f14a1c7633 100644 --- a/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h @@ -298,7 +298,7 @@ public: SendWFR(ResultSerializer::serialize(std::move(Result))); }; - callAsync(std::forward(H), std::move(SendResult), Args, + callAsync(std::forward(H), std::move(SendResult), std::move(Args), ArgIndices{}); } @@ -314,9 +314,9 @@ private: typename ArgTupleT, std::size_t... I> static void callAsync(HandlerT &&H, SerializeAndSendResultT &&SerializeAndSendResult, - ArgTupleT &Args, std::index_sequence) { + ArgTupleT Args, std::index_sequence) { return std::forward(H)(std::move(SerializeAndSendResult), - std::get(Args)...); + std::move(std::get(Args))...); } }; diff --git a/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h b/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h index 458947cc4d4..96e4341fce6 100644 --- a/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h +++ b/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h @@ -41,6 +41,13 @@ enum WireProtectionFlags : uint8_t { LLVM_MARK_AS_BITMASK_ENUM(WPF_Exec) }; +struct ExecutorProcessInfo { + std::string Triple; + unsigned PageSize; + JITTargetAddress DispatchFuncAddr; + JITTargetAddress DispatchCtxAddr; +}; + /// Convert from sys::Memory::ProtectionFlags inline WireProtectionFlags toWireProtectionFlags(sys::Memory::ProtectionFlags PF) { @@ -95,6 +102,41 @@ using ReleaseOrFinalizeMemRequest = namespace shared { +template <> class SerializationTypeName { +public: + static const char *getName() { return "WrapperFunctionResult"; } +}; + +template +class SerializationTraits< + ChannelT, WrapperFunctionResult, WrapperFunctionResult, + std::enable_if_t::value>> { +public: + static Error serialize(ChannelT &C, const WrapperFunctionResult &E) { + if (auto Err = serializeSeq(C, static_cast(E.size()))) + return Err; + if (E.size() == 0) + return Error::success(); + return C.appendBytes(E.data(), E.size()); + } + + static Error deserialize(ChannelT &C, WrapperFunctionResult &E) { + uint64_t Size; + if (auto Err = deserializeSeq(C, Size)) + return Err; + + WrapperFunctionResult Tmp; + char *Data = WrapperFunctionResult::allocate(Tmp, Size); + + if (auto Err = C.readBytes(Data, Size)) + return Err; + + E = std::move(Tmp); + + return Error::success(); + } +}; + template <> class SerializationTypeName { public: static const char *getName() { return "UInt8Write"; } @@ -136,9 +178,9 @@ public: static const char *getName() { return "ReleaseOrFinalizeMemRequestElement"; } }; -template <> class SerializationTypeName { +template <> class SerializationTypeName { public: - static const char *getName() { return "WrapperFunctionResult"; } + static const char *getName() { return "ExecutorProcessInfo"; } }; template @@ -234,26 +276,17 @@ public: }; template -class SerializationTraits< - ChannelT, shared::WrapperFunctionResult, shared::WrapperFunctionResult, - std::enable_if_t::value>> { +class SerializationTraits { public: - static Error serialize(ChannelT &C, const shared::WrapperFunctionResult &E) { - if (auto Err = serializeSeq(C, static_cast(E.size()))) - return Err; - if (E.size() == 0) - return Error::success(); - return C.appendBytes(E.data(), E.size()); + static Error serialize(ChannelT &C, + const orcrpctpc::ExecutorProcessInfo &EPI) { + return serializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr, + EPI.DispatchCtxAddr); } - static Error deserialize(ChannelT &C, shared::WrapperFunctionResult &E) { - - uint64_t Size; - if (auto Err = deserializeSeq(C, Size)) - return Err; - - char *DataPtr = shared::WrapperFunctionResult::allocate(E, Size); - return C.readBytes(DataPtr, E.size()); + static Error deserialize(ChannelT &C, orcrpctpc::ExecutorProcessInfo &EPI) { + return deserializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr, + EPI.DispatchCtxAddr); } }; @@ -265,15 +298,11 @@ using RemoteSymbolLookupSet = std::vector>; using RemoteLookupRequest = std::pair; -class GetTargetTriple - : public shared::RPCFunction { +class GetExecutorProcessInfo + : public shared::RPCFunction { public: - static const char *getName() { return "GetTargetTriple"; } -}; - -class GetPageSize : public shared::RPCFunction { -public: - static const char *getName() { return "GetPageSize"; } + static const char *getName() { return "GetJITDispatchInfo"; } }; class ReserveMem @@ -349,7 +378,7 @@ public: class RunMain : public shared::RPCFunction Args)> { public: static const char *getName() { return "RunMain"; } @@ -372,18 +401,18 @@ public: /// TargetProcessControl for a process connected via an ORC RPC Endpoint. template class OrcRPCTPCServer { +private: + using ThisT = OrcRPCTPCServer; + public: /// Create an OrcRPCTPCServer from the given endpoint. OrcRPCTPCServer(RPCEndpointT &EP) : EP(EP) { - using ThisT = OrcRPCTPCServer; TripleStr = sys::getProcessTriple(); PageSize = sys::Process::getPageSizeEstimate(); - EP.template addHandler(*this, - &ThisT::getTargetTriple); - EP.template addHandler(*this, &ThisT::getPageSize); - + EP.template addHandler( + *this, &ThisT::getExecutorProcessInfo); EP.template addHandler(*this, &ThisT::reserveMemory); EP.template addHandler(*this, &ThisT::finalizeMemory); @@ -428,9 +457,34 @@ public: return Error::success(); } + Expected + runWrapperInJIT(JITTargetAddress FunctionId, ArrayRef ArgBuffer) { + return EP.template callB( + FunctionId, + ArrayRef(reinterpret_cast(ArgBuffer.data()), + ArgBuffer.size())); + } + private: - std::string getTargetTriple() { return TripleStr; } - uint64_t getPageSize() { return PageSize; } + static shared::detail::CWrapperFunctionResult + jitDispatchViaOrcRPCTPCServer(void *Ctx, const void *FnTag, const char *Data, + size_t Size) { + assert(Ctx && "Attempt to dispatch with null context ptr"); + auto R = static_cast(Ctx)->runWrapperInJIT( + pointerToJITTargetAddress(FnTag), {Data, Size}); + if (!R) { + auto ErrMsg = toString(R.takeError()); + return shared::WrapperFunctionResult::createOutOfBandError(ErrMsg.data()) + .release(); + } + return R->release(); + } + + orcrpctpc::ExecutorProcessInfo getExecutorProcessInfo() { + return {TripleStr, static_cast(PageSize), + pointerToJITTargetAddress(jitDispatchViaOrcRPCTPCServer), + pointerToJITTargetAddress(this)}; + } template static void handleWriteUInt(const std::vector &Ws) { @@ -569,7 +623,7 @@ private: return Result; } - int32_t runMain(JITTargetAddress MainFnAddr, + int64_t runMain(JITTargetAddress MainFnAddr, const std::vector &Args) { Optional ProgramNameOverride; if (ProgramName) diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index f6ebfc318bf..c613c57c15f 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -1750,6 +1750,49 @@ Expected> Platform::lookupInitSymbols( return std::move(CompoundResult); } +void Platform::lookupInitSymbolsAsync( + unique_function OnComplete, ExecutionSession &ES, + const DenseMap &InitSyms) { + + class TriggerOnComplete { + public: + using OnCompleteFn = unique_function; + TriggerOnComplete(OnCompleteFn OnComplete) + : OnComplete(std::move(OnComplete)) {} + ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); } + void reportResult(Error Err) { + std::lock_guard Lock(ResultMutex); + LookupResult = joinErrors(std::move(LookupResult), std::move(Err)); + } + + private: + std::mutex ResultMutex; + Error LookupResult{Error::success()}; + OnCompleteFn OnComplete; + }; + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + auto TOC = std::make_shared(std::move(OnComplete)); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [TOC](Expected Result) { + TOC->reportResult(Result.takeError()); + }, + NoDependenciesToRegister); + } +} + void Task::anchor() {} void MaterializationTask::printDescription(raw_ostream &OS) { diff --git a/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 12fa42ccdef..9442eab2997 100644 --- a/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -14,6 +14,8 @@ #include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#define DEBUG_TYPE "orc" + namespace llvm { namespace orc { @@ -47,6 +49,10 @@ Error ExecutorProcessControl::associateJITSideWrapperFunctions( "AsyncWrapperFunction implementation missing"); TagToFunc[KV.second.getAddress()] = std::make_shared(std::move(I->second)); + LLVM_DEBUG({ + dbgs() << "Associated function tag \"" << *KV.first << "\" (" + << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n"; + }); } return Error::success(); } @@ -84,6 +90,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( this->PageSize = PageSize; this->MemMgr = OwnedMemMgr.get(); this->MemAccess = this; + this->JDI = {ExecutorAddress::fromPtr(jitDispatchViaWrapperFunctionManager), + ExecutorAddress::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; } @@ -198,5 +206,26 @@ void SelfExecutorProcessControl::writeBuffers( OnWriteComplete(Error::success()); } +shared::detail::CWrapperFunctionResult +SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( + void *Ctx, const void *FnTag, const char *Data, size_t Size) { + + LLVM_DEBUG({ + dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size + << " byte payload.\n"; + }); + + std::promise ResultP; + auto ResultF = ResultP.get_future(); + static_cast(Ctx)->runJITSideWrapperFunction( + [ResultP = + std::move(ResultP)](shared::WrapperFunctionResult Result) mutable { + ResultP.set_value(std::move(Result)); + }, + pointerToJITTargetAddress(FnTag), {Data, Size}); + + return ResultF.get().release(); +} + } // end namespace orc } // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 0fe4f00a1b6..686c3e8c165 100644 --- a/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -9,27 +9,182 @@ #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "orc" +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace { + +class MachOHeaderMaterializationUnit : public MaterializationUnit { +public: + MachOHeaderMaterializationUnit(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) + : MaterializationUnit(createHeaderSymbols(MOP, HeaderStartSymbol), + HeaderStartSymbol), + MOP(MOP) {} + + StringRef getName() const override { return "MachOHeaderMU"; } + + void materialize(std::unique_ptr R) override { + unsigned PointerSize; + support::endianness Endianness; + + switch (MOP.getExecutorProcessControl().getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto G = std::make_unique( + "", MOP.getExecutorProcessControl().getTargetTriple(), + PointerSize, Endianness, jitlink::getGenericEdgeKindName); + auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ); + auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + + // Init symbol is header-start symbol. + G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(), + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + for (auto &HS : AdditionalHeaderSymbols) + G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { + {"___mh_executable_header", 0}}; + + static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + MachO::mach_header_64 Hdr; + Hdr.magic = MachO::MH_MAGIC_64; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + Hdr.filetype = MachO::MH_DYLIB; // Custom file type? + Hdr.ncmds = 0; + Hdr.sizeofcmds = 0; + Hdr.flags = 0; + Hdr.reserved = 0; + + if (G.getEndianness() != support::endian::system_endianness()) + MachO::swapStruct(Hdr); + + auto HeaderContent = G.allocateString( + StringRef(reinterpret_cast(&Hdr), sizeof(Hdr))); + + return G.createContentBlock(HeaderSection, HeaderContent, 0, 8, 0); + } + + static SymbolFlagsMap + createHeaderSymbols(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; + + return HeaderSymbolFlags; + } + + MachOPlatform &MOP; +}; + +constexpr MachOHeaderMaterializationUnit::HeaderSymbol + MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; + +StringRef EHFrameSectionName = "__TEXT,__eh_frame"; +StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; + +StringRef InitSectionNames[] = {ModInitFuncSectionName}; + +} // end anonymous namespace + namespace llvm { namespace orc { -MachOPlatform::MachOPlatform( - ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr StandardSymbolsObject) - : ES(ES), ObjLinkingLayer(ObjLinkingLayer), - StandardSymbolsObject(std::move(StandardSymbolsObject)) { - ObjLinkingLayer.addPlugin(std::make_unique(*this)); +Expected> +MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + const char *OrcRuntimePath, + Optional RuntimeAliases) { + + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error("Unsupported MachOPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); + + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + // Add JIT-dispatch function support symbols. + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern("___orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("___orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( + ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr( + new MachOPlatform(ES, ObjLinkingLayer, EPC, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), Err)); + if (Err) + return std::move(Err); + return std::move(P); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - auto ObjBuffer = MemoryBuffer::getMemBuffer( - StandardSymbolsObject->getMemBufferRef(), false); - return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); + return JD.define(std::make_unique( + *this, MachOHeaderStartSymbol)); } Error MachOPlatform::notifyAdding(ResourceTracker &RT, @@ -52,58 +207,117 @@ Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { llvm_unreachable("Not supported yet"); } -Expected -MachOPlatform::getInitializerSequence(JITDylib &JD) { +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Building initializer sequence for " - << JD.getName() << "\n"; - }); +SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, requiredCXXAliases()); + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} - std::vector DFSLinkOrder; +ArrayRef> +MachOPlatform::requiredCXXAliases() { + static const std::pair RequiredCXXAliases[] = { + {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}}; - while (true) { + return RequiredCXXAliases; +} - DenseMap NewInitSymbols; +ArrayRef> +MachOPlatform::standardRuntimeUtilityAliases() { + static const std::pair + StandardRuntimeUtilityAliases[] = { + {"___orc_rt_run_program", "___orc_rt_macho_run_program"}, + {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}}; - ES.runSessionLocked([&]() { - DFSLinkOrder = JD.getDFSLinkOrder(); + return StandardRuntimeUtilityAliases; +} - for (auto &InitJD : DFSLinkOrder) { - auto RISItr = RegisteredInitSymbols.find(InitJD.get()); - if (RISItr != RegisteredInitSymbols.end()) { - NewInitSymbols[InitJD.get()] = std::move(RISItr->second); - RegisteredInitSymbols.erase(RISItr); - } - } - }); +bool MachOPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} - if (NewInitSymbols.empty()) - break; +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), EPC(EPC), + MachOHeaderStartSymbol(ES.intern("___dso_handle")) { + ErrorAsOutParameter _(&Err); - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Issuing lookups for new init symbols: " - "(lookup may require multiple rounds)\n"; - for (auto &KV : NewInitSymbols) - dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; - }); + ObjLinkingLayer.addPlugin(std::make_unique(*this)); - // Outside the lock, issue the lookup. - if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) - ; // Nothing to do in the success case. - else - return R.takeError(); + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); + + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; } - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Init symbol lookup complete, building init " - "sequence\n"; - }); + RegisteredInitSymbols[&PlatformJD].add( + MachOHeaderStartSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); - // Lock again to collect the initializers. - InitializerSequence FullInitSeq; + // Associate wrapper function tags with JIT-side function implementations. + if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { + Err = std::move(E2); + return; + } + + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto E2 = bootstrapMachORuntime(PlatformJD)) { + Err = std::move(E2); + return; + } +} + +Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutorProcessControl::WrapperFunctionAssociationMap WFs; + + using GetInitializersSPSSig = + SPSExpected(SPSString); + WFs[ES.intern("___orc_rt_macho_get_initializers_tag")] = + EPC.wrapAsyncWithSPS( + this, &MachOPlatform::rt_getInitializers); + + using GetDeinitializersSPSSig = + SPSExpected(SPSExecutorAddress); + WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] = + EPC.wrapAsyncWithSPS( + this, &MachOPlatform::rt_getDeinitializers); + + using LookupSymbolSPSSig = + SPSExpected(SPSExecutorAddress, SPSString); + WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = + EPC.wrapAsyncWithSPS(this, + &MachOPlatform::rt_lookupSymbol); + + return EPC.associateJITSideWrapperFunctions(PlatformJD, std::move(WFs)); +} + +void MachOPlatform::getInitializersBuildSequencePhase( + SendInitializerSequenceFn SendResult, JITDylib &JD, + std::vector DFSLinkOrder) { + MachOJITDylibInitializerSequence FullInitSeq; { - std::lock_guard Lock(InitSeqsMutex); + std::lock_guard Lock(PlatformMutex); for (auto &InitJD : reverse(DFSLinkOrder)) { LLVM_DEBUG({ dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() @@ -111,160 +325,267 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { }); auto ISItr = InitSeqs.find(InitJD.get()); if (ISItr != InitSeqs.end()) { - FullInitSeq.emplace_back(InitJD.get(), std::move(ISItr->second)); + FullInitSeq.emplace_back(std::move(ISItr->second)); InitSeqs.erase(ISItr); } } } - return FullInitSeq; + SendResult(std::move(FullInitSeq)); } -Expected -MachOPlatform::getDeinitializerSequence(JITDylib &JD) { - std::vector DFSLinkOrder = JD.getDFSLinkOrder(); +void MachOPlatform::getInitializersLookupPhase( + SendInitializerSequenceFn SendResult, JITDylib &JD) { - DeinitializerSequence FullDeinitSeq; - { - std::lock_guard Lock(InitSeqsMutex); - for (auto &DeinitJD : DFSLinkOrder) { - FullDeinitSeq.emplace_back(DeinitJD.get(), MachOJITDylibDeinitializers()); + auto DFSLinkOrder = JD.getDFSLinkOrder(); + DenseMap NewInitSymbols; + ES.runSessionLocked([&]() { + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } } + }); + + // If there are no further init symbols to look up then move on to the next + // phase. + if (NewInitSymbols.empty()) { + getInitializersBuildSequencePhase(std::move(SendResult), JD, + std::move(DFSLinkOrder)); + return; } - return FullDeinitSeq; + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + getInitializersLookupPhase(std::move(SendResult), JD); + }, + ES, std::move(NewInitSymbols)); } -void MachOPlatform::registerInitInfo(JITDylib &JD, - JITTargetAddress ObjCImageInfoAddr, - ExecutorAddressRange ModInits, - ExecutorAddressRange ObjCSelRefs, - ExecutorAddressRange ObjCClassList) { - std::lock_guard Lock(InitSeqsMutex); +void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getInitializers(\"" << JDName << "\")\n"; + }); - auto &InitSeq = InitSeqs[&JD]; + JITDylib *JD = ES.getJITDylibByName(JDName); + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; + }); + SendResult(make_error("No JITDylib named " + JDName, + inconvertibleErrorCode())); + return; + } - InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); - - if (ModInits.StartAddress) - InitSeq.addModInitsSection(std::move(ModInits)); - - if (ObjCSelRefs.StartAddress) - InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); - - if (ObjCClassList.StartAddress) - InitSeq.addObjCClassListSection(std::move(ObjCClassList)); + getInitializersLookupPhase(std::move(SendResult), *JD); } -static Expected getSectionExtent(jitlink::LinkGraph &G, - StringRef SectionName) { - auto *Sec = G.findSectionByName(SectionName); - if (!Sec) - return ExecutorAddressRange(); - jitlink::SectionRange R(*Sec); - if (R.getSize() % G.getPointerSize() != 0) - return make_error(SectionName + " section size is not a " - "multiple of the pointer size", +void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getDeinitializers(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + SendResult(MachOJITDylibDeinitializerSequence()); +} + +void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddress Handle, + StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + // FIXME: Proper mangling. + auto MangledName = ("_" + SymbolName).str(); + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready, + [SendResult = std::move(SendResult)](Expected Result) mutable { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddress(Result->begin()->second.getAddress())); + } else + SendResult(Result.takeError()); + }, + NoDependenciesToRegister); +} + +Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { + + std::pair Symbols[] = { + {"___orc_rt_macho_platform_bootstrap", &orc_rt_macho_platform_bootstrap}, + {"___orc_rt_macho_platform_shutdown", &orc_rt_macho_platform_shutdown}, + {"___orc_rt_macho_register_object_sections", + &orc_rt_macho_register_object_sections}}; + + SymbolLookupSet RuntimeSymbols; + std::vector> AddrsToRecord; + for (const auto &KV : Symbols) { + auto Name = ES.intern(KV.first); + RuntimeSymbols.add(Name); + AddrsToRecord.push_back({std::move(Name), KV.second}); + } + + auto RuntimeSymbolAddrs = ES.lookup( + {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); + if (!RuntimeSymbolAddrs) + return RuntimeSymbolAddrs.takeError(); + + for (const auto &KV : AddrsToRecord) { + auto &Name = KV.first; + assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); + KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + } + + if (auto Err = + EPC.runSPSWrapper(orc_rt_macho_platform_bootstrap.getValue())) + return Err; + + // FIXME: Ordering is fuzzy here. We're probably best off saying + // "behavior is undefined if code that uses the runtime is added before + // the platform constructor returns", then move all this to the constructor. + RuntimeBootstrapped = true; + std::vector DeferredPOSRs; + { + std::lock_guard Lock(PlatformMutex); + DeferredPOSRs = std::move(BootstrapPOSRs); + } + + for (auto &D : DeferredPOSRs) + if (auto Err = registerPerObjectSections(D)) + return Err; + + return Error::success(); +} + +Error MachOPlatform::registerInitInfo( + JITDylib &JD, ArrayRef InitSections) { + + std::unique_lock Lock(PlatformMutex); + + MachOJITDylibInitializers *InitSeq = nullptr; + { + auto I = InitSeqs.find(&JD); + if (I == InitSeqs.end()) { + // If there's no init sequence entry yet then we need to look up the + // header symbol to force creation of one. + Lock.unlock(); + + auto SearchOrder = + JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); + if (auto Err = ES.lookup(SearchOrder, MachOHeaderStartSymbol).takeError()) + return Err; + + Lock.lock(); + I = InitSeqs.find(&JD); + assert(I != InitSeqs.end() && + "Entry missing after header symbol lookup?"); + } + InitSeq = &I->second; + } + + for (auto *Sec : InitSections) { + // FIXME: Avoid copy here. + jitlink::SectionRange R(*Sec); + InitSeq->InitSections[Sec->getName()].push_back( + {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())}); + } + + return Error::success(); +} + +Error MachOPlatform::registerPerObjectSections( + const MachOPerObjectSectionsToRegister &POSR) { + + if (!orc_rt_macho_register_object_sections) + return make_error("Attempting to register per-object " + "sections, but runtime support has not " + "been loaded yet", inconvertibleErrorCode()); - return ExecutorAddressRange(ExecutorAddress(R.getStart()), - ExecutorAddress(R.getEnd())); + + Error ErrResult = Error::success(); + if (auto Err = EPC.runSPSWrapper( + orc_rt_macho_register_object_sections.getValue(), ErrResult, POSR)) + return Err; + return ErrResult; } -void MachOPlatform::InitScraperPlugin::modifyPassConfig( +void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { - if (!MR.getInitializerSymbol()) + // If the initializer symbol is the MachOHeader start symbol then just add + // the macho header support passes. + if (MR.getInitializerSymbol() == MP.MachOHeaderStartSymbol) { + addMachOHeaderSupportPasses(MR, Config); + // The header materialization unit doesn't require any other support, so we + // can bail out early. return; + } - Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { - JITLinkSymbolSet InitSectionSyms; - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__mod_init_func"); - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__objc_selrefs"); - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__objc_classlist"); + // If the object contains initializers then add passes to record them. + if (MR.getInitializerSymbol()) + addInitializerSupportPasses(MR, Config); - if (!InitSectionSyms.empty()) { - std::lock_guard Lock(InitScraperMutex); - InitSymbolDeps[&MR] = std::move(InitSectionSyms); - } - - if (auto Err = processObjCImageInfo(G, MR)) - return Err; - - return Error::success(); - }); - - Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( - jitlink::LinkGraph &G) -> Error { - ExecutorAddressRange ModInits, ObjCSelRefs, ObjCClassList; - - JITTargetAddress ObjCImageInfoAddr = 0; - if (auto *ObjCImageInfoSec = - G.findSectionByName("__DATA,__objc_image_info")) { - if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) - ObjCImageInfoAddr = Addr; - } - - // Record __mod_init_func. - if (auto ModInitsOrErr = getSectionExtent(G, "__DATA,__mod_init_func")) - ModInits = std::move(*ModInitsOrErr); - else - return ModInitsOrErr.takeError(); - - // Record __objc_selrefs. - if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__DATA,__objc_selrefs")) - ObjCSelRefs = std::move(*ObjCSelRefsOrErr); - else - return ObjCSelRefsOrErr.takeError(); - - // Record __objc_classlist. - if (auto ObjCClassListOrErr = - getSectionExtent(G, "__DATA,__objc_classlist")) - ObjCClassList = std::move(*ObjCClassListOrErr); - else - return ObjCClassListOrErr.takeError(); - - // Dump the scraped inits. - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; - dbgs() << " __objc_selrefs: "; - auto NumObjCSelRefs = ObjCSelRefs.size().getValue() / sizeof(uintptr_t); - if (NumObjCSelRefs) - dbgs() << NumObjCSelRefs << " pointer(s) at " - << formatv("{0:x16}", ObjCSelRefs.StartAddress.getValue()) - << "\n"; - else - dbgs() << "none\n"; - - dbgs() << " __objc_classlist: "; - auto NumObjCClasses = ObjCClassList.size().getValue() / sizeof(uintptr_t); - if (NumObjCClasses) - dbgs() << NumObjCClasses << " pointer(s) at " - << formatv("{0:x16}", ObjCClassList.StartAddress.getValue()) - << "\n"; - else - dbgs() << "none\n"; - - dbgs() << " __mod_init_func: "; - auto NumModInits = ModInits.size().getValue() / sizeof(uintptr_t); - if (NumModInits) - dbgs() << NumModInits << " pointer(s) at " - << formatv("{0:x16}", ModInits.StartAddress.getValue()) << "\n"; - else - dbgs() << "none\n"; - }); - - MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), - std::move(ObjCSelRefs), std::move(ObjCClassList)); - - return Error::success(); - }); + // Add passes for eh-frame support. + addEHSupportPasses(MR, Config); } ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap -MachOPlatform::InitScraperPlugin::getSyntheticSymbolDependencies( +MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( MaterializationResponsibility &MR) { - std::lock_guard Lock(InitScraperMutex); + std::lock_guard Lock(PluginMutex); auto I = InitSymbolDeps.find(&MR); if (I != InitSymbolDeps.end()) { SyntheticSymbolDependenciesMap Result; @@ -275,93 +596,135 @@ MachOPlatform::InitScraperPlugin::getSyntheticSymbolDependencies( return SyntheticSymbolDependenciesMap(); } -void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( - JITLinkSymbolSet &Symbols, jitlink::LinkGraph &G, StringRef SectionName) { - if (auto *Sec = G.findSectionByName(SectionName)) { - auto SecBlocks = Sec->blocks(); - if (!llvm::empty(SecBlocks)) - Symbols.insert( - &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); - } +void MachOPlatform::MachOPlatformPlugin::addInitializerSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + /// Preserve init sections. + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + return preserveInitSections(G, MR); + }); + + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerInitSections(G, JD); + }); } -Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( +void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *MP.MachOHeaderStartSymbol; + }); + assert(I != G.defined_symbols().end() && + "Missing MachO header start symbol"); + { + std::lock_guard Lock(MP.PlatformMutex); + JITTargetAddress HeaderAddr = (*I)->getAddress(); + MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); + MP.InitSeqs.insert( + std::make_pair(&JD, MachOJITDylibInitializers( + JD.getName(), ExecutorAddress(HeaderAddr)))); + } + return Error::success(); + }); +} + +void MachOPlatform::MachOPlatformPlugin::addEHSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + // Add a pass to register the final addresses of the eh-frame sections + // with the runtime. + Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { + MachOPerObjectSectionsToRegister POSR; + + if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + jitlink::SectionRange R(*EHFrameSection); + if (!R.empty()) + POSR.EHFrameSection = {ExecutorAddress(R.getStart()), + ExecutorAddress(R.getEnd())}; + } + + if (POSR.EHFrameSection.StartAddress) { + + // If we're still bootstrapping the runtime then just record this + // frame for now. + if (!MP.RuntimeBootstrapped) { + std::lock_guard Lock(MP.PlatformMutex); + MP.BootstrapPOSRs.push_back(POSR); + return Error::success(); + } + + // Otherwise register it immediately. + if (auto Err = MP.registerPerObjectSections(POSR)) + return Err; + } + + return Error::success(); + }); +} + +Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { - // If there's an ObjC imagine info then either - // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In - // this case we name and record it. - // OR - // (2) We already have a recorded __objc_imageinfo for this JITDylib, - // in which case we just verify it. - auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); - if (!ObjCImageInfo) - return Error::success(); + JITLinkSymbolSet InitSectionSymbols; + for (auto &InitSectionName : InitSectionNames) { + // Skip non-init sections. + auto *InitSection = G.findSectionByName(InitSectionName); + if (!InitSection) + continue; - auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); + // Make a pass over live symbols in the section: those blocks are already + // preserved. + DenseSet AlreadyLiveBlocks; + for (auto &Sym : InitSection->symbols()) { + auto &B = Sym->getBlock(); + if (Sym->isLive() && Sym->getOffset() == 0 && + Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) { + InitSectionSymbols.insert(Sym); + AlreadyLiveBlocks.insert(&B); + } + } - // Check that the section is not empty if present. - if (llvm::empty(ObjCImageInfoBlocks)) - return make_error("Empty __objc_imageinfo section in " + - G.getName(), - inconvertibleErrorCode()); - - // Check that there's only one block in the section. - if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) - return make_error("Multiple blocks in __objc_imageinfo " - "section in " + - G.getName(), - inconvertibleErrorCode()); - - // Check that the __objc_imageinfo section is unreferenced. - // FIXME: We could optimize this check if Symbols had a ref-count. - for (auto &Sec : G.sections()) { - if (&Sec != ObjCImageInfo) - for (auto *B : Sec.blocks()) - for (auto &E : B->edges()) - if (E.getTarget().isDefined() && - &E.getTarget().getBlock().getSection() == ObjCImageInfo) - return make_error("__objc_imageinfo is referenced " - "within file " + - G.getName(), - inconvertibleErrorCode()); + // Add anonymous symbols to preserve any not-already-preserved blocks. + for (auto *B : InitSection->blocks()) + if (!AlreadyLiveBlocks.count(B)) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true)); } - auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); - auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); - auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness()); - auto Flags = - support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); - - // Lock the mutex while we verify / update the ObjCImageInfos map. - std::lock_guard Lock(InitScraperMutex); - - auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); - if (ObjCImageInfoItr != ObjCImageInfos.end()) { - // We've already registered an __objc_imageinfo section. Verify the - // content of this new section matches, then delete it. - if (ObjCImageInfoItr->second.first != Version) - return make_error( - "ObjC version in " + G.getName() + - " does not match first registered version", - inconvertibleErrorCode()); - if (ObjCImageInfoItr->second.second != Flags) - return make_error("ObjC flags in " + G.getName() + - " do not match first registered flags", - inconvertibleErrorCode()); - - // __objc_imageinfo is valid. Delete the block. - for (auto *S : ObjCImageInfo->symbols()) - G.removeDefinedSymbol(*S); - G.removeBlock(ObjCImageInfoBlock); - } else { - // We haven't registered an __objc_imageinfo section yet. Register and - // move on. The section should already be marked no-dead-strip. - ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + if (!InitSectionSymbols.empty()) { + std::lock_guard Lock(PluginMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); } return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::registerInitSections( + jitlink::LinkGraph &G, JITDylib &JD) { + + SmallVector InitSections; + + for (auto InitSectionName : InitSectionNames) + if (auto *Sec = G.findSectionByName(InitSectionName)) + InitSections.push_back(Sec); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + for (auto *Sec : InitSections) { + jitlink::SectionRange R(*Sec); + dbgs() << " " << Sec->getName() << ": " + << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + } + }); + + return MP.registerInitInfo(JD, InitSections); +} + } // End namespace orc. } // End namespace llvm. diff --git a/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp b/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp index 9a92581157c..7f197a50c39 100644 --- a/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp +++ b/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp @@ -141,7 +141,7 @@ int main(int argc, char *argv[]) { ExitOnErr.setBanner(std::string(argv[0]) + ":"); using JITLinkExecutorEndpoint = - shared::MultiThreadedRPCEndpoint; + shared::SingleThreadedRPCEndpoint; shared::registerStringError(); diff --git a/tools/llvm-jitlink/llvm-jitlink.cpp b/tools/llvm-jitlink/llvm-jitlink.cpp index f8298317fee..ec91597bc35 100644 --- a/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/tools/llvm-jitlink/llvm-jitlink.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/MC/MCAsmInfo.h" @@ -159,6 +160,15 @@ static cl::opt OutOfProcessExecutorConnect( "oop-executor-connect", cl::desc("Connect to an out-of-process executor via TCP")); +// TODO: Default to false if compiler-rt is not built. +static cl::opt UseOrcRuntime("use-orc-runtime", + cl::desc("Do not required/load ORC runtime"), + cl::init(true)); + +static cl::opt + OrcRuntimePath("orc-runtime-path", cl::desc("Add orc runtime to session"), + cl::init("")); + ExitOnError ExitOnErr; LLVM_ATTRIBUTE_USED void linkComponents() { @@ -167,6 +177,14 @@ LLVM_ATTRIBUTE_USED void linkComponents() { << (void *)&llvm_orc_registerJITLoaderGDBWrapper; } +static bool UseTestResultOverride = false; +static int64_t TestResultOverride = 0; + +extern "C" void llvm_jitlink_setTestResultOverride(int64_t Value) { + TestResultOverride = Value; + UseTestResultOverride = true; +} + namespace llvm { static raw_ostream & @@ -588,6 +606,31 @@ Error LLVMJITLinkObjectLinkingLayer::add(ResourceTrackerSP RT, return JD.define(std::move(MU), std::move(RT)); } +static Error loadProcessSymbols(Session &S) { + auto FilterMainEntryPoint = + [EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) { + return Name != EPName; + }; + S.MainJD->addGenerator( + ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess( + *S.EPC, std::move(FilterMainEntryPoint)))); + + return Error::success(); +} + +static Error loadDylibs(Session &S) { + LLVM_DEBUG(dbgs() << "Loading dylibs...\n"); + for (const auto &Dylib : Dylibs) { + LLVM_DEBUG(dbgs() << " " << Dylib << "\n"); + auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str()); + if (!G) + return G.takeError(); + S.MainJD->addGenerator(std::move(*G)); + } + + return Error::success(); +} + Expected> LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor() { #ifndef LLVM_ON_UNIX @@ -796,40 +839,46 @@ Expected> Session::Create(Triple TT) { if (!PageSize) return PageSize.takeError(); - /// If -oop-executor is passed then launch the executor. std::unique_ptr EPC; if (OutOfProcessExecutor.getNumOccurrences()) { + /// If -oop-executor is passed then launch the executor. if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor()) EPC = std::move(*REPC); else return REPC.takeError(); } else if (OutOfProcessExecutorConnect.getNumOccurrences()) { + /// If -oop-executor-connect is passed then connect to the executor. if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::ConnectToExecutor()) EPC = std::move(*REPC); else return REPC.takeError(); - } else + } else { + /// Otherwise use SelfExecutorProcessControl to target the current process. EPC = std::make_unique( std::make_shared(), std::move(TT), *PageSize, createMemoryManager()); + } Error Err = Error::success(); std::unique_ptr S(new Session(std::move(EPC), Err)); - if (Err) - return std::move(Err); - return std::move(S); + + // FIXME: Errors destroy the session, leaving the SymbolStringPtrs dangling, + // so just exit here. We could fix this by having errors keep the pool alive. + ExitOnErr(std::move(Err)); + return S; } Session::~Session() { if (auto Err = ES.endSession()) ES.reportError(std::move(Err)); + if (auto Err = EPC->disconnect()) + ES.reportError(std::move(Err)); } -// FIXME: Move to createJITDylib if/when we start using Platform support in -// llvm-jitlink. Session::Session(std::unique_ptr EPC, Error &Err) - : EPC(std::move(EPC)), ObjLayer(*this, this->EPC->getMemMgr()) { + : EPC(std::move(EPC)), ES(this->EPC->getSymbolStringPool()), + ObjLayer(*this, this->EPC->getMemMgr()) { /// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the /// Session. @@ -863,7 +912,21 @@ Session::Session(std::unique_ptr EPC, Error &Err) return; } - if (!NoExec && !this->EPC->getTargetTriple().isOSWindows()) { + if (!NoProcessSymbols) + ExitOnErr(loadProcessSymbols(*this)); + ExitOnErr(loadDylibs(*this)); + + // Set up the platform. + if (this->EPC->getTargetTriple().isOSBinFormatMachO() && UseOrcRuntime) { + if (auto P = MachOPlatform::Create(ES, ObjLayer, *this->EPC, *MainJD, + OrcRuntimePath.c_str())) + ES.setPlatform(std::move(*P)); + else { + Err = P.takeError(); + return; + } + } else if (!NoExec && !this->EPC->getTargetTriple().isOSWindows() && + !this->EPC->getTargetTriple().isOSBinFormatMachO()) { ObjLayer.addPlugin(std::make_unique( ES, ExitOnErr(EPCEHFrameRegistrar::Create(*this->EPC)))); ObjLayer.addPlugin(std::make_unique( @@ -1044,18 +1107,41 @@ static Triple getFirstFileTriple() { return FirstTT; } -static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { - // Set the entry point name if not specified. - if (EntryPointName.empty()) { - if (TT.getObjectFormat() == Triple::MachO) - EntryPointName = "_main"; - else - EntryPointName = "main"; +static bool isOrcRuntimeSupported(const Triple &TT) { + switch (TT.getObjectFormat()) { + case Triple::MachO: + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } + default: + return false; } +} + +static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { + + // If we're in noexec mode and the user didn't explicitly specify + // -use-orc-runtime then don't use it. + if (NoExec && UseOrcRuntime.getNumOccurrences() == 0) + UseOrcRuntime = false; // -noexec and --args should not be used together. if (NoExec && !InputArgv.empty()) - outs() << "Warning: --args passed to -noexec run will be ignored.\n"; + errs() << "Warning: --args passed to -noexec run will be ignored.\n"; + + // Turn off UseOrcRuntime on platforms where it's not supported. + if (UseOrcRuntime && !isOrcRuntimeSupported(TT)) { + errs() << "Warning: Orc runtime not available for target platform. " + "Use -use-orc-runtime=false to suppress this warning.\n"; + UseOrcRuntime = false; + } + + // Set the entry point name if not specified. + if (EntryPointName.empty()) + EntryPointName = TT.getObjectFormat() == Triple::MachO ? "_main" : "main"; // If -slab-address is passed, require -slab-allocate and -noexec if (SlabAddress != ~0ULL) { @@ -1080,35 +1166,27 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { SmallString<256> OOPExecutorPath(sys::fs::getMainExecutable( ArgV0, reinterpret_cast(&sanitizeArguments))); sys::path::remove_filename(OOPExecutorPath); - if (OOPExecutorPath.back() != '/') - OOPExecutorPath += '/'; - OOPExecutorPath += "llvm-jitlink-executor"; + sys::path::append(OOPExecutorPath, "llvm-jitlink-executor"); OutOfProcessExecutor = OOPExecutorPath.str().str(); } - return Error::success(); -} - -static Error loadProcessSymbols(Session &S) { - auto FilterMainEntryPoint = - [EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) { - return Name != EPName; - }; - S.MainJD->addGenerator( - ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess( - *S.EPC, std::move(FilterMainEntryPoint)))); - - return Error::success(); -} - -static Error loadDylibs(Session &S) { - for (const auto &Dylib : Dylibs) { - auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str()); - if (!G) - return G.takeError(); - S.MainJD->addGenerator(std::move(*G)); + // If we're loading the Orc runtime then determine the path for it. + if (UseOrcRuntime) { + if (OrcRuntimePath.empty()) { + SmallString<256> DefaultOrcRuntimePath(sys::fs::getMainExecutable( + ArgV0, reinterpret_cast(&sanitizeArguments))); + sys::path::remove_filename( + DefaultOrcRuntimePath); // remove 'llvm-jitlink' + while (!DefaultOrcRuntimePath.empty() && + DefaultOrcRuntimePath.back() == '/') + DefaultOrcRuntimePath.pop_back(); + if (DefaultOrcRuntimePath.endswith("bin")) + sys::path::remove_filename(DefaultOrcRuntimePath); // remove 'bin' + sys::path::append(DefaultOrcRuntimePath, + "lib/clang/13.0.0/lib/darwin/libclang_rt.orc_osx.a"); + OrcRuntimePath = DefaultOrcRuntimePath.str().str(); + } } - return Error::success(); } @@ -1227,6 +1305,11 @@ static Error loadObjects(Session &S) { static Error runChecks(Session &S) { + if (CheckFiles.empty()) + return Error::success(); + + LLVM_DEBUG(dbgs() << "Running checks...\n"); + auto TripleName = S.EPC->getTargetTriple().str(); std::string ErrorStr; const Target *TheTarget = TargetRegistry::lookupTarget(TripleName, ErrorStr); @@ -1310,9 +1393,15 @@ static Error runChecks(Session &S) { } static void dumpSessionStats(Session &S) { + if (!ShowSizes) + return; + if (UseOrcRuntime) + outs() << "Note: Session stats include runtime and entry point lookup, but " + "not JITDylib initialization/deinitialization.\n"; if (ShowSizes) - outs() << "Total size of all blocks before pruning: " << S.SizeBeforePruning - << "\nTotal size of all blocks after fixups: " << S.SizeAfterFixups + outs() << " Total size of all blocks before pruning: " + << S.SizeBeforePruning + << "\n Total size of all blocks after fixups: " << S.SizeAfterFixups << "\n"; } @@ -1320,6 +1409,34 @@ static Expected getMainEntryPoint(Session &S) { return S.ES.lookup(S.JDSearchOrder, EntryPointName); } +static Expected getOrcRuntimeEntryPoint(Session &S) { + std::string RuntimeEntryPoint = "__orc_rt_run_program_wrapper"; + if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO) + RuntimeEntryPoint = '_' + RuntimeEntryPoint; + return S.ES.lookup(S.JDSearchOrder, RuntimeEntryPoint); +} + +static Expected runWithRuntime(Session &S, + JITTargetAddress EntryPointAddress) { + StringRef DemangledEntryPoint = EntryPointName; + if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO && + DemangledEntryPoint.front() == '_') + DemangledEntryPoint = DemangledEntryPoint.drop_front(); + using SPSRunProgramSig = + int64_t(SPSString, SPSString, SPSSequence); + int64_t Result; + if (auto Err = S.EPC->runSPSWrapper( + EntryPointAddress, Result, S.MainJD->getName(), DemangledEntryPoint, + static_cast &>(InputArgv))) + return std::move(Err); + return Result; +} + +static Expected runWithoutRuntime(Session &S, + JITTargetAddress EntryPointAddress) { + return S.EPC->runAsMain(EntryPointAddress, InputArgv); +} + namespace { struct JITLinkTimers { TimerGroup JITLinkTG{"llvm-jitlink timers", "timers for llvm-jitlink phases"}; @@ -1352,21 +1469,23 @@ int main(int argc, char *argv[]) { ExitOnErr(loadObjects(*S)); } - if (!NoProcessSymbols) - ExitOnErr(loadProcessSymbols(*S)); - ExitOnErr(loadDylibs(*S)); - if (PhonyExternals) addPhonyExternalsGenerator(*S); - if (ShowInitialExecutionSessionState) S->ES.dump(outs()); JITEvaluatedSymbol EntryPoint = 0; { TimeRegion TR(Timers ? &Timers->LinkTimer : nullptr); + // Find the entry-point function unconditionally, since we want to force + // it to be materialized to collect stats. EntryPoint = ExitOnErr(getMainEntryPoint(*S)); + + // If we're running with the ORC runtime then replace the entry-point + // with the __orc_rt_run_program symbol. + if (UseOrcRuntime) + EntryPoint = ExitOnErr(getOrcRuntimeEntryPoint(*S)); } if (ShowAddrs) @@ -1381,12 +1500,20 @@ int main(int argc, char *argv[]) { int Result = 0; { + LLVM_DEBUG(dbgs() << "Running \"" << EntryPointName << "\"...\n"); TimeRegion TR(Timers ? &Timers->RunTimer : nullptr); - Result = ExitOnErr(S->EPC->runAsMain(EntryPoint.getAddress(), InputArgv)); + if (UseOrcRuntime) + Result = ExitOnErr(runWithRuntime(*S, EntryPoint.getAddress())); + else + Result = ExitOnErr(runWithoutRuntime(*S, EntryPoint.getAddress())); } - ExitOnErr(S->ES.endSession()); - ExitOnErr(S->EPC->disconnect()); + // Destroy the session. + S.reset(); + + // If the executing code set a test result override then use that. + if (UseTestResultOverride) + Result = TestResultOverride; return Result; } diff --git a/tools/llvm-jitlink/llvm-jitlink.h b/tools/llvm-jitlink/llvm-jitlink.h index 750e543fba1..5050b2fef75 100644 --- a/tools/llvm-jitlink/llvm-jitlink.h +++ b/tools/llvm-jitlink/llvm-jitlink.h @@ -111,7 +111,7 @@ private: struct Session { std::unique_ptr EPC; orc::ExecutionSession ES; - orc::JITDylib *MainJD; + orc::JITDylib *MainJD = nullptr; LLVMJITLinkObjectLinkingLayer ObjLayer; std::vector JDSearchOrder; diff --git a/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp b/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp index 5c784c16a4c..f4a6b458889 100644 --- a/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp +++ b/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h" #include "gtest/gtest.h" using namespace llvm; @@ -158,3 +159,9 @@ TEST(SimplePackedSerializationTest, ArgListSerialization) { EXPECT_EQ(Arg2, ArgOut2); EXPECT_EQ(Arg3, ArgOut3); } + +TEST(SimplePackedSerialization, StringMap) { + StringMap M({{"A", 1}, {"B", 2}}); + blobSerializationRoundTrip>, + StringMap>(M); +}