diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index 15b5286b869..81d8dbee480 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -916,6 +916,12 @@ public: const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); } + /// Allocate a mutable buffer of the given size using the LinkGraph's + /// allocator. + MutableArrayRef allocateBuffer(size_t Size) { + return {Allocator.Allocate(Size), Size}; + } + /// Allocate a copy of the given string using the LinkGraph's allocator. /// This can be useful when renaming symbols or adding new content to the /// graph. diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 9219e288015..159553b5559 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -1224,9 +1224,19 @@ public: /// A utility function for looking up initializer symbols. Performs a blocking /// lookup for the given symbols in each of the given JITDylibs. + /// + /// Note: This function is deprecated and will be removed in the near future. static Expected> lookupInitSymbols(ExecutionSession &ES, const DenseMap &InitSyms); + + /// Performs an async lookup for the the given symbols in each of the given + /// JITDylibs, calling the given handler with the compound result map once + /// all lookups have completed. + static void + lookupInitSymbolsAsync(unique_function OnComplete, + ExecutionSession &ES, + const DenseMap &InitSyms); }; /// Represents an abstract task for ORC to run. diff --git a/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h index 566637e1044..f8d6192706a 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h @@ -18,6 +18,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" #include "llvm/Support/DynamicLibrary.h" @@ -113,6 +114,13 @@ public: const SymbolLookupSet &Symbols; }; + /// Contains the address of the dispatch function and context that the ORC + /// runtime can use to call functions in the JIT. + struct JITDispatchInfo { + ExecutorAddress JITDispatchFunctionAddress; + ExecutorAddress JITDispatchContextAddress; + }; + virtual ~ExecutorProcessControl(); /// Intern a symbol name in the SymbolStringPool. @@ -127,6 +135,9 @@ public: /// Get the page size for the target process. unsigned getPageSize() const { return PageSize; } + /// Get the JIT dispatch function and context address for the executor. + const JITDispatchInfo &getJITDispatchInfo() const { return JDI; } + /// Return a MemoryAccess object for the target process. MemoryAccess &getMemoryAccess() const { return *MemAccess; } @@ -198,14 +209,17 @@ public: /// Run a wrapper function using SPS to serialize the arguments and /// deserialize the results. - template - Error runSPSWrapper(JITTargetAddress WrapperFnAddr, RetT &RetVal, - const ArgTs &...Args) { + /// + /// If SPSSignature is a non-void function signature then the second argument + /// (the first in the Args list) should be a reference to a return value. + template + Error runSPSWrapper(JITTargetAddress WrapperFnAddr, + WrapperCallArgTs &&...WrapperCallArgs) { return shared::WrapperFunction::call( [this, WrapperFnAddr](const char *ArgData, size_t ArgSize) { return runWrapper(WrapperFnAddr, ArrayRef(ArgData, ArgSize)); }, - RetVal, Args...); + std::forward(WrapperCallArgs)...); } /// Wrap a handler that takes concrete argument types (and a sender for a @@ -223,6 +237,15 @@ public: }; } + template + static AsyncWrapperFunction + wrapAsyncWithSPS(ClassT *Instance, void (ClassT::*Method)(MethodArgTs...)) { + return wrapAsyncWithSPS( + [Instance, Method](MethodArgTs &&...MethodArgs) { + (Instance->*Method)(std::forward(MethodArgs)...); + }); + } + /// For each symbol name, associate the AsyncWrapperFunction implementation /// value with the address of that symbol. /// @@ -250,6 +273,7 @@ protected: std::shared_ptr SSP; Triple TargetTriple; unsigned PageSize = 0; + JITDispatchInfo JDI; MemoryAccess *MemAccess = nullptr; jitlink::JITLinkMemoryManager *MemMgr = nullptr; @@ -318,6 +342,10 @@ private: void writeBuffers(ArrayRef Ws, WriteResultFn OnWriteComplete) override; + static shared::detail::CWrapperFunctionResult + jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag, + const char *Data, size_t Size); + std::unique_ptr OwnedMemMgr; char GlobalManglingPrefix = 0; std::vector> DynamicLibraries; diff --git a/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h b/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h new file mode 100644 index 00000000000..f3d616deae8 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h @@ -0,0 +1,69 @@ +//===-- LLVMSPSSerializers.h - SPS serialization for LLVM types -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// SPS Serialization for common LLVM types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H +#define LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" + +namespace llvm { +namespace orc { +namespace shared { + +template +class SPSSerializationTraits>, + StringMap> { +public: + static size_t size(const StringMap &M) { + size_t Sz = SPSArgList::size(static_cast(M.size())); + for (auto &E : M) + Sz += SPSArgList::size(E.first(), E.second); + return Sz; + } + + static bool serialize(SPSOutputBuffer &OB, const StringMap &M) { + if (!SPSArgList::serialize(OB, static_cast(M.size()))) + return false; + + for (auto &E : M) + if (!SPSArgList::serialize(OB, E.first(), E.second)) + return false; + + return true; + } + + static bool deserialize(SPSInputBuffer &IB, StringMap &M) { + uint64_t Size; + assert(M.empty() && "M already contains elements"); + + if (!SPSArgList::deserialize(IB, Size)) + return false; + + while (Size--) { + StringRef S; + ValueT V; + if (!SPSArgList::deserialize(IB, S, V)) + return false; + if (!M.insert(std::make_pair(S, V)).second) + return false; + } + + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LLVMSPSSERIALIZERS_H diff --git a/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 10d0f90510c..50f26b399a6 100644 --- a/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -15,7 +15,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" @@ -26,83 +27,106 @@ namespace llvm { namespace orc { +struct MachOPerObjectSectionsToRegister { + ExecutorAddressRange EHFrameSection; +}; -class MachOJITDylibInitializers { -public: - using RawPointerSectionList = std::vector; +struct MachOJITDylibInitializers { + using SectionList = std::vector; - void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) { - this->ObjCImageInfoAddr = ObjCImageInfoAddr; - } + MachOJITDylibInitializers(std::string Name, + ExecutorAddress MachOHeaderAddress) + : Name(std::move(Name)), + MachOHeaderAddress(std::move(MachOHeaderAddress)) {} - void addModInitsSection(ExecutorAddressRange ModInit) { - ModInitSections.push_back(std::move(ModInit)); - } + std::string Name; + ExecutorAddress MachOHeaderAddress; - const RawPointerSectionList &getModInitsSections() const { - return ModInitSections; - } - - void addObjCSelRefsSection(ExecutorAddressRange ObjCSelRefs) { - ObjCSelRefsSections.push_back(std::move(ObjCSelRefs)); - } - - const RawPointerSectionList &getObjCSelRefsSections() const { - return ObjCSelRefsSections; - } - - void addObjCClassListSection(ExecutorAddressRange ObjCClassList) { - ObjCClassListSections.push_back(std::move(ObjCClassList)); - } - - const RawPointerSectionList &getObjCClassListSections() const { - return ObjCClassListSections; - } - - void runModInits() const; - void registerObjCSelectors() const; - Error registerObjCClasses() const; - -private: - - JITTargetAddress ObjCImageInfoAddr; - RawPointerSectionList ModInitSections; - RawPointerSectionList ObjCSelRefsSections; - RawPointerSectionList ObjCClassListSections; + StringMap InitSections; }; class MachOJITDylibDeinitializers {}; +using MachOJITDylibInitializerSequence = std::vector; + +using MachOJITDylibDeinitializerSequence = + std::vector; + /// Mediates between MachO initialization and ExecutionSession state. class MachOPlatform : public Platform { public: - using InitializerSequence = - std::vector>; - - using DeinitializerSequence = - std::vector>; - - MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr StandardSymbolsObject); + /// Try to create a MachOPlatform instance, adding the ORC runtime to the + /// given JITDylib. + /// + /// The ORC runtime requires access to a number of symbols in libc++, and + /// requires access to symbols in libobjc, and libswiftCore to support + /// Objective-C and Swift code. It is up to the caller to ensure that the + /// requried symbols can be referenced by code added to PlatformJD. The + /// standard way to achieve this is to first attach dynamic library search + /// generators for either the given process, or for the specific required + /// libraries, to PlatformJD, then to create the platform instance: + /// + /// \code{.cpp} + /// auto &PlatformJD = ES.createBareJITDylib("stdlib"); + /// PlatformJD.addGenerator( + /// ExitOnErr(EPCDynamicLibrarySearchGenerator + /// ::GetForTargetProcess(EPC))); + /// ES.setPlatform( + /// ExitOnErr(MachOPlatform::Create(ES, ObjLayer, EPC, PlatformJD, + /// "/path/to/orc/runtime"))); + /// \endcode + /// + /// Alternatively, these symbols could be added to another JITDylib that + /// PlatformJD links against. + /// + /// Clients are also responsible for ensuring that any JIT'd code that + /// depends on runtime functions (including any code using TLV or static + /// destructors) can reference the runtime symbols. This is usually achieved + /// by linking any JITDylibs containing regular code against + /// PlatformJD. + /// + /// By default, MachOPlatform will add the set of aliases returned by the + /// standardPlatformAliases function. This includes both required aliases + /// (e.g. __cxa_atexit -> __orc_rt_macho_cxa_atexit for static destructor + /// support), and optional aliases that provide JIT versions of common + /// functions (e.g. dlopen -> __orc_rt_macho_jit_dlopen). Clients can + /// override these defaults by passing a non-None value for the + /// RuntimeAliases function, in which case the client is responsible for + /// setting up all aliases (including the required ones). + static Expected> + Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + const char *OrcRuntimePath, + Optional RuntimeAliases = None); ExecutionSession &getExecutionSession() const { return ES; } + ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + ExecutorProcessControl &getExecutorProcessControl() const { return EPC; } Error setupJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) override; Error notifyRemoving(ResourceTracker &RT) override; - Expected getInitializerSequence(JITDylib &JD); + /// Returns an AliasMap containing the default aliases for the MachOPlatform. + /// This can be modified by clients when constructing the platform to add + /// or remove aliases. + static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); - Expected getDeinitializerSequence(JITDylib &JD); + /// Returns the array of required CXX aliases. + static ArrayRef> requiredCXXAliases(); + + /// Returns the array of standard runtime utility aliases for MachO. + static ArrayRef> + standardRuntimeUtilityAliases(); private: - // This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func, - // __objc_classlist and __sel_ref sections and records their extents so that - // they can be run in the target process. - class InitScraperPlugin : public ObjectLinkingLayer::Plugin { + // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO + // platform features including initializers, exceptions, TLV, and language + // runtime registration. + class MachOPlatformPlugin : public ObjectLinkingLayer::Plugin { public: - InitScraperPlugin(MachOPlatform &MP) : MP(MP) {} + MachOPlatformPlugin(MachOPlatform &MP) : MP(MP) {} void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, @@ -128,36 +152,173 @@ private: using InitSymbolDepMap = DenseMap; - void preserveInitSectionIfPresent(JITLinkSymbolSet &Symbols, - jitlink::LinkGraph &G, - StringRef SectionName); + void addInitializerSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); - Error processObjCImageInfo(jitlink::LinkGraph &G, + void addMachOHeaderSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + void addEHSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + Error preserveInitSections(jitlink::LinkGraph &G, MaterializationResponsibility &MR); - std::mutex InitScraperMutex; + Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); + + std::mutex PluginMutex; MachOPlatform &MP; - DenseMap> ObjCImageInfos; InitSymbolDepMap InitSymbolDeps; }; - void registerInitInfo(JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, - ExecutorAddressRange ModInits, - ExecutorAddressRange ObjCSelRefs, - ExecutorAddressRange ObjCClassList); + using SendInitializerSequenceFn = + unique_function)>; + + using SendDeinitializerSequenceFn = + unique_function)>; + + using SendSymbolAddressFn = unique_function)>; + + static bool supportedTarget(const Triple &TT); + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, + Error &Err); + + // Associate MachOPlatform JIT-side runtime support functions with handlers. + Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); + + void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, + JITDylib &JD, + std::vector DFSLinkOrder); + + void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, + JITDylib &JD); + + void rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName); + + void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle); + + void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddress Handle, + StringRef SymbolName); + + // Records the addresses of runtime symbols used by the platform. + Error bootstrapMachORuntime(JITDylib &PlatformJD); + + Error registerInitInfo(JITDylib &JD, + ArrayRef InitSections); + + Error registerPerObjectSections(const MachOPerObjectSectionsToRegister &POSR); ExecutionSession &ES; ObjectLinkingLayer &ObjLinkingLayer; - std::unique_ptr StandardSymbolsObject; + ExecutorProcessControl &EPC; + + SymbolStringPtr MachOHeaderStartSymbol; + std::atomic RuntimeBootstrapped{false}; + + ExecutorAddress orc_rt_macho_platform_bootstrap; + ExecutorAddress orc_rt_macho_platform_shutdown; + ExecutorAddress orc_rt_macho_register_object_sections; DenseMap RegisteredInitSymbols; // InitSeqs gets its own mutex to avoid locking the whole session when // aggregating data from the jitlink. - std::mutex InitSeqsMutex; + std::mutex PlatformMutex; DenseMap InitSeqs; + std::vector BootstrapPOSRs; + + DenseMap HeaderAddrToJITDylib; }; +namespace shared { + +using SPSMachOPerObjectSectionsToRegister = SPSTuple; + +template <> +class SPSSerializationTraits { + +public: + static size_t size(const MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOPerObjectSectionsToRegister &MOPOSR) { + return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection); + } +}; + +using SPSNamedExecutorAddressRangeSequenceMap = + SPSSequence>; + +using SPSMachOJITDylibInitializers = + SPSTuple; + +using SPSMachOJITDylibInitializerSequence = + SPSSequence; + +/// Serialization traits for MachOJITDylibInitializers. +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.InitSections); + } +}; + +using SPSMachOJITDylibDeinitializers = SPSEmpty; + +using SPSMachOJITDylibDeinitializerSequence = + SPSSequence; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibDeinitializers &MOJDDs) { + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibDeinitializers &MOJDDs) { + MOJDDs = MachOJITDylibDeinitializers(); + return true; + } +}; + +} // end namespace shared } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h b/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h index 69e37f9af9e..fedd79c9314 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h @@ -283,7 +283,11 @@ public: OrcRPCExecutorProcessControlBase(std::shared_ptr SSP, RPCEndpointT &EP, ErrorReporter ReportError) : ExecutorProcessControl(std::move(SSP)), - ReportError(std::move(ReportError)), EP(EP) {} + ReportError(std::move(ReportError)), EP(EP) { + using ThisT = OrcRPCExecutorProcessControlBase; + EP.template addAsyncHandler(*this, + &ThisT::runWrapperInJIT); + } void reportError(Error Err) { ReportError(std::move(Err)); } @@ -396,20 +400,32 @@ protected: /// Subclasses must call this during construction to initialize the /// TargetTriple and PageSize members. Error initializeORCRPCEPCBase() { - if (auto TripleOrErr = EP.template callB()) - TargetTriple = Triple(*TripleOrErr); - else - return TripleOrErr.takeError(); - - if (auto PageSizeOrErr = EP.template callB()) - PageSize = *PageSizeOrErr; - else - return PageSizeOrErr.takeError(); - - return Error::success(); + if (auto EPI = EP.template callB()) { + this->TargetTriple = Triple(EPI->Triple); + this->PageSize = PageSize; + this->JDI = {ExecutorAddress(EPI->DispatchFuncAddr), + ExecutorAddress(EPI->DispatchCtxAddr)}; + return Error::success(); + } else + return EPI.takeError(); } private: + Error runWrapperInJIT( + std::function)> SendResult, + JITTargetAddress FunctionTag, std::vector ArgBuffer) { + + runJITSideWrapperFunction( + [this, SendResult = std::move(SendResult)]( + Expected R) { + if (auto Err = SendResult(std::move(R))) + ReportError(std::move(Err)); + }, + FunctionTag, + {reinterpret_cast(ArgBuffer.data()), ArgBuffer.size()}); + return Error::success(); + } + ErrorReporter ReportError; RPCEndpointT &EP; }; diff --git a/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h index a2ad84647d2..854f1098d5a 100644 --- a/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h +++ b/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -85,6 +85,7 @@ public: bool skip(size_t Size) { if (Size > Remaining) return false; + Buffer += Size; Remaining -= Size; return true; } diff --git a/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h b/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h index ceaea1d2b20..2f14a1c7633 100644 --- a/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h @@ -298,7 +298,7 @@ public: SendWFR(ResultSerializer::serialize(std::move(Result))); }; - callAsync(std::forward(H), std::move(SendResult), Args, + callAsync(std::forward(H), std::move(SendResult), std::move(Args), ArgIndices{}); } @@ -314,9 +314,9 @@ private: typename ArgTupleT, std::size_t... I> static void callAsync(HandlerT &&H, SerializeAndSendResultT &&SerializeAndSendResult, - ArgTupleT &Args, std::index_sequence) { + ArgTupleT Args, std::index_sequence) { return std::forward(H)(std::move(SerializeAndSendResult), - std::get(Args)...); + std::move(std::get(Args))...); } }; diff --git a/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h b/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h index 458947cc4d4..96e4341fce6 100644 --- a/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h +++ b/include/llvm/ExecutionEngine/Orc/TargetProcess/OrcRPCTPCServer.h @@ -41,6 +41,13 @@ enum WireProtectionFlags : uint8_t { LLVM_MARK_AS_BITMASK_ENUM(WPF_Exec) }; +struct ExecutorProcessInfo { + std::string Triple; + unsigned PageSize; + JITTargetAddress DispatchFuncAddr; + JITTargetAddress DispatchCtxAddr; +}; + /// Convert from sys::Memory::ProtectionFlags inline WireProtectionFlags toWireProtectionFlags(sys::Memory::ProtectionFlags PF) { @@ -95,6 +102,41 @@ using ReleaseOrFinalizeMemRequest = namespace shared { +template <> class SerializationTypeName { +public: + static const char *getName() { return "WrapperFunctionResult"; } +}; + +template +class SerializationTraits< + ChannelT, WrapperFunctionResult, WrapperFunctionResult, + std::enable_if_t::value>> { +public: + static Error serialize(ChannelT &C, const WrapperFunctionResult &E) { + if (auto Err = serializeSeq(C, static_cast(E.size()))) + return Err; + if (E.size() == 0) + return Error::success(); + return C.appendBytes(E.data(), E.size()); + } + + static Error deserialize(ChannelT &C, WrapperFunctionResult &E) { + uint64_t Size; + if (auto Err = deserializeSeq(C, Size)) + return Err; + + WrapperFunctionResult Tmp; + char *Data = WrapperFunctionResult::allocate(Tmp, Size); + + if (auto Err = C.readBytes(Data, Size)) + return Err; + + E = std::move(Tmp); + + return Error::success(); + } +}; + template <> class SerializationTypeName { public: static const char *getName() { return "UInt8Write"; } @@ -136,9 +178,9 @@ public: static const char *getName() { return "ReleaseOrFinalizeMemRequestElement"; } }; -template <> class SerializationTypeName { +template <> class SerializationTypeName { public: - static const char *getName() { return "WrapperFunctionResult"; } + static const char *getName() { return "ExecutorProcessInfo"; } }; template @@ -234,26 +276,17 @@ public: }; template -class SerializationTraits< - ChannelT, shared::WrapperFunctionResult, shared::WrapperFunctionResult, - std::enable_if_t::value>> { +class SerializationTraits { public: - static Error serialize(ChannelT &C, const shared::WrapperFunctionResult &E) { - if (auto Err = serializeSeq(C, static_cast(E.size()))) - return Err; - if (E.size() == 0) - return Error::success(); - return C.appendBytes(E.data(), E.size()); + static Error serialize(ChannelT &C, + const orcrpctpc::ExecutorProcessInfo &EPI) { + return serializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr, + EPI.DispatchCtxAddr); } - static Error deserialize(ChannelT &C, shared::WrapperFunctionResult &E) { - - uint64_t Size; - if (auto Err = deserializeSeq(C, Size)) - return Err; - - char *DataPtr = shared::WrapperFunctionResult::allocate(E, Size); - return C.readBytes(DataPtr, E.size()); + static Error deserialize(ChannelT &C, orcrpctpc::ExecutorProcessInfo &EPI) { + return deserializeSeq(C, EPI.Triple, EPI.PageSize, EPI.DispatchFuncAddr, + EPI.DispatchCtxAddr); } }; @@ -265,15 +298,11 @@ using RemoteSymbolLookupSet = std::vector>; using RemoteLookupRequest = std::pair; -class GetTargetTriple - : public shared::RPCFunction { +class GetExecutorProcessInfo + : public shared::RPCFunction { public: - static const char *getName() { return "GetTargetTriple"; } -}; - -class GetPageSize : public shared::RPCFunction { -public: - static const char *getName() { return "GetPageSize"; } + static const char *getName() { return "GetJITDispatchInfo"; } }; class ReserveMem @@ -349,7 +378,7 @@ public: class RunMain : public shared::RPCFunction Args)> { public: static const char *getName() { return "RunMain"; } @@ -372,18 +401,18 @@ public: /// TargetProcessControl for a process connected via an ORC RPC Endpoint. template class OrcRPCTPCServer { +private: + using ThisT = OrcRPCTPCServer; + public: /// Create an OrcRPCTPCServer from the given endpoint. OrcRPCTPCServer(RPCEndpointT &EP) : EP(EP) { - using ThisT = OrcRPCTPCServer; TripleStr = sys::getProcessTriple(); PageSize = sys::Process::getPageSizeEstimate(); - EP.template addHandler(*this, - &ThisT::getTargetTriple); - EP.template addHandler(*this, &ThisT::getPageSize); - + EP.template addHandler( + *this, &ThisT::getExecutorProcessInfo); EP.template addHandler(*this, &ThisT::reserveMemory); EP.template addHandler(*this, &ThisT::finalizeMemory); @@ -428,9 +457,34 @@ public: return Error::success(); } + Expected + runWrapperInJIT(JITTargetAddress FunctionId, ArrayRef ArgBuffer) { + return EP.template callB( + FunctionId, + ArrayRef(reinterpret_cast(ArgBuffer.data()), + ArgBuffer.size())); + } + private: - std::string getTargetTriple() { return TripleStr; } - uint64_t getPageSize() { return PageSize; } + static shared::detail::CWrapperFunctionResult + jitDispatchViaOrcRPCTPCServer(void *Ctx, const void *FnTag, const char *Data, + size_t Size) { + assert(Ctx && "Attempt to dispatch with null context ptr"); + auto R = static_cast(Ctx)->runWrapperInJIT( + pointerToJITTargetAddress(FnTag), {Data, Size}); + if (!R) { + auto ErrMsg = toString(R.takeError()); + return shared::WrapperFunctionResult::createOutOfBandError(ErrMsg.data()) + .release(); + } + return R->release(); + } + + orcrpctpc::ExecutorProcessInfo getExecutorProcessInfo() { + return {TripleStr, static_cast(PageSize), + pointerToJITTargetAddress(jitDispatchViaOrcRPCTPCServer), + pointerToJITTargetAddress(this)}; + } template static void handleWriteUInt(const std::vector &Ws) { @@ -569,7 +623,7 @@ private: return Result; } - int32_t runMain(JITTargetAddress MainFnAddr, + int64_t runMain(JITTargetAddress MainFnAddr, const std::vector &Args) { Optional ProgramNameOverride; if (ProgramName) diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index f6ebfc318bf..c613c57c15f 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -1750,6 +1750,49 @@ Expected> Platform::lookupInitSymbols( return std::move(CompoundResult); } +void Platform::lookupInitSymbolsAsync( + unique_function OnComplete, ExecutionSession &ES, + const DenseMap &InitSyms) { + + class TriggerOnComplete { + public: + using OnCompleteFn = unique_function; + TriggerOnComplete(OnCompleteFn OnComplete) + : OnComplete(std::move(OnComplete)) {} + ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); } + void reportResult(Error Err) { + std::lock_guard Lock(ResultMutex); + LookupResult = joinErrors(std::move(LookupResult), std::move(Err)); + } + + private: + std::mutex ResultMutex; + Error LookupResult{Error::success()}; + OnCompleteFn OnComplete; + }; + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + auto TOC = std::make_shared(std::move(OnComplete)); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [TOC](Expected Result) { + TOC->reportResult(Result.takeError()); + }, + NoDependenciesToRegister); + } +} + void Task::anchor() {} void MaterializationTask::printDescription(raw_ostream &OS) { diff --git a/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 12fa42ccdef..9442eab2997 100644 --- a/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -14,6 +14,8 @@ #include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#define DEBUG_TYPE "orc" + namespace llvm { namespace orc { @@ -47,6 +49,10 @@ Error ExecutorProcessControl::associateJITSideWrapperFunctions( "AsyncWrapperFunction implementation missing"); TagToFunc[KV.second.getAddress()] = std::make_shared(std::move(I->second)); + LLVM_DEBUG({ + dbgs() << "Associated function tag \"" << *KV.first << "\" (" + << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n"; + }); } return Error::success(); } @@ -84,6 +90,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( this->PageSize = PageSize; this->MemMgr = OwnedMemMgr.get(); this->MemAccess = this; + this->JDI = {ExecutorAddress::fromPtr(jitDispatchViaWrapperFunctionManager), + ExecutorAddress::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; } @@ -198,5 +206,26 @@ void SelfExecutorProcessControl::writeBuffers( OnWriteComplete(Error::success()); } +shared::detail::CWrapperFunctionResult +SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( + void *Ctx, const void *FnTag, const char *Data, size_t Size) { + + LLVM_DEBUG({ + dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size + << " byte payload.\n"; + }); + + std::promise ResultP; + auto ResultF = ResultP.get_future(); + static_cast(Ctx)->runJITSideWrapperFunction( + [ResultP = + std::move(ResultP)](shared::WrapperFunctionResult Result) mutable { + ResultP.set_value(std::move(Result)); + }, + pointerToJITTargetAddress(FnTag), {Data, Size}); + + return ResultF.get().release(); +} + } // end namespace orc } // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 0fe4f00a1b6..686c3e8c165 100644 --- a/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -9,27 +9,182 @@ #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "orc" +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace { + +class MachOHeaderMaterializationUnit : public MaterializationUnit { +public: + MachOHeaderMaterializationUnit(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) + : MaterializationUnit(createHeaderSymbols(MOP, HeaderStartSymbol), + HeaderStartSymbol), + MOP(MOP) {} + + StringRef getName() const override { return "MachOHeaderMU"; } + + void materialize(std::unique_ptr R) override { + unsigned PointerSize; + support::endianness Endianness; + + switch (MOP.getExecutorProcessControl().getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto G = std::make_unique( + "", MOP.getExecutorProcessControl().getTargetTriple(), + PointerSize, Endianness, jitlink::getGenericEdgeKindName); + auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ); + auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + + // Init symbol is header-start symbol. + G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(), + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + for (auto &HS : AdditionalHeaderSymbols) + G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { + {"___mh_executable_header", 0}}; + + static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + MachO::mach_header_64 Hdr; + Hdr.magic = MachO::MH_MAGIC_64; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + Hdr.filetype = MachO::MH_DYLIB; // Custom file type? + Hdr.ncmds = 0; + Hdr.sizeofcmds = 0; + Hdr.flags = 0; + Hdr.reserved = 0; + + if (G.getEndianness() != support::endian::system_endianness()) + MachO::swapStruct(Hdr); + + auto HeaderContent = G.allocateString( + StringRef(reinterpret_cast(&Hdr), sizeof(Hdr))); + + return G.createContentBlock(HeaderSection, HeaderContent, 0, 8, 0); + } + + static SymbolFlagsMap + createHeaderSymbols(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; + + return HeaderSymbolFlags; + } + + MachOPlatform &MOP; +}; + +constexpr MachOHeaderMaterializationUnit::HeaderSymbol + MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; + +StringRef EHFrameSectionName = "__TEXT,__eh_frame"; +StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; + +StringRef InitSectionNames[] = {ModInitFuncSectionName}; + +} // end anonymous namespace + namespace llvm { namespace orc { -MachOPlatform::MachOPlatform( - ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr StandardSymbolsObject) - : ES(ES), ObjLinkingLayer(ObjLinkingLayer), - StandardSymbolsObject(std::move(StandardSymbolsObject)) { - ObjLinkingLayer.addPlugin(std::make_unique(*this)); +Expected> +MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + const char *OrcRuntimePath, + Optional RuntimeAliases) { + + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error("Unsupported MachOPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); + + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + // Add JIT-dispatch function support symbols. + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern("___orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("___orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( + ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr( + new MachOPlatform(ES, ObjLinkingLayer, EPC, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), Err)); + if (Err) + return std::move(Err); + return std::move(P); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - auto ObjBuffer = MemoryBuffer::getMemBuffer( - StandardSymbolsObject->getMemBufferRef(), false); - return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); + return JD.define(std::make_unique( + *this, MachOHeaderStartSymbol)); } Error MachOPlatform::notifyAdding(ResourceTracker &RT, @@ -52,58 +207,117 @@ Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { llvm_unreachable("Not supported yet"); } -Expected -MachOPlatform::getInitializerSequence(JITDylib &JD) { +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Building initializer sequence for " - << JD.getName() << "\n"; - }); +SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, requiredCXXAliases()); + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} - std::vector DFSLinkOrder; +ArrayRef> +MachOPlatform::requiredCXXAliases() { + static const std::pair RequiredCXXAliases[] = { + {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}}; - while (true) { + return RequiredCXXAliases; +} - DenseMap NewInitSymbols; +ArrayRef> +MachOPlatform::standardRuntimeUtilityAliases() { + static const std::pair + StandardRuntimeUtilityAliases[] = { + {"___orc_rt_run_program", "___orc_rt_macho_run_program"}, + {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}}; - ES.runSessionLocked([&]() { - DFSLinkOrder = JD.getDFSLinkOrder(); + return StandardRuntimeUtilityAliases; +} - for (auto &InitJD : DFSLinkOrder) { - auto RISItr = RegisteredInitSymbols.find(InitJD.get()); - if (RISItr != RegisteredInitSymbols.end()) { - NewInitSymbols[InitJD.get()] = std::move(RISItr->second); - RegisteredInitSymbols.erase(RISItr); - } - } - }); +bool MachOPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} - if (NewInitSymbols.empty()) - break; +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + ExecutorProcessControl &EPC, JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), EPC(EPC), + MachOHeaderStartSymbol(ES.intern("___dso_handle")) { + ErrorAsOutParameter _(&Err); - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Issuing lookups for new init symbols: " - "(lookup may require multiple rounds)\n"; - for (auto &KV : NewInitSymbols) - dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; - }); + ObjLinkingLayer.addPlugin(std::make_unique(*this)); - // Outside the lock, issue the lookup. - if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) - ; // Nothing to do in the success case. - else - return R.takeError(); + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); + + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; } - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Init symbol lookup complete, building init " - "sequence\n"; - }); + RegisteredInitSymbols[&PlatformJD].add( + MachOHeaderStartSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); - // Lock again to collect the initializers. - InitializerSequence FullInitSeq; + // Associate wrapper function tags with JIT-side function implementations. + if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { + Err = std::move(E2); + return; + } + + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto E2 = bootstrapMachORuntime(PlatformJD)) { + Err = std::move(E2); + return; + } +} + +Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutorProcessControl::WrapperFunctionAssociationMap WFs; + + using GetInitializersSPSSig = + SPSExpected(SPSString); + WFs[ES.intern("___orc_rt_macho_get_initializers_tag")] = + EPC.wrapAsyncWithSPS( + this, &MachOPlatform::rt_getInitializers); + + using GetDeinitializersSPSSig = + SPSExpected(SPSExecutorAddress); + WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] = + EPC.wrapAsyncWithSPS( + this, &MachOPlatform::rt_getDeinitializers); + + using LookupSymbolSPSSig = + SPSExpected(SPSExecutorAddress, SPSString); + WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = + EPC.wrapAsyncWithSPS(this, + &MachOPlatform::rt_lookupSymbol); + + return EPC.associateJITSideWrapperFunctions(PlatformJD, std::move(WFs)); +} + +void MachOPlatform::getInitializersBuildSequencePhase( + SendInitializerSequenceFn SendResult, JITDylib &JD, + std::vector DFSLinkOrder) { + MachOJITDylibInitializerSequence FullInitSeq; { - std::lock_guard Lock(InitSeqsMutex); + std::lock_guard Lock(PlatformMutex); for (auto &InitJD : reverse(DFSLinkOrder)) { LLVM_DEBUG({ dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() @@ -111,160 +325,267 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { }); auto ISItr = InitSeqs.find(InitJD.get()); if (ISItr != InitSeqs.end()) { - FullInitSeq.emplace_back(InitJD.get(), std::move(ISItr->second)); + FullInitSeq.emplace_back(std::move(ISItr->second)); InitSeqs.erase(ISItr); } } } - return FullInitSeq; + SendResult(std::move(FullInitSeq)); } -Expected -MachOPlatform::getDeinitializerSequence(JITDylib &JD) { - std::vector DFSLinkOrder = JD.getDFSLinkOrder(); +void MachOPlatform::getInitializersLookupPhase( + SendInitializerSequenceFn SendResult, JITDylib &JD) { - DeinitializerSequence FullDeinitSeq; - { - std::lock_guard Lock(InitSeqsMutex); - for (auto &DeinitJD : DFSLinkOrder) { - FullDeinitSeq.emplace_back(DeinitJD.get(), MachOJITDylibDeinitializers()); + auto DFSLinkOrder = JD.getDFSLinkOrder(); + DenseMap NewInitSymbols; + ES.runSessionLocked([&]() { + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } } + }); + + // If there are no further init symbols to look up then move on to the next + // phase. + if (NewInitSymbols.empty()) { + getInitializersBuildSequencePhase(std::move(SendResult), JD, + std::move(DFSLinkOrder)); + return; } - return FullDeinitSeq; + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + getInitializersLookupPhase(std::move(SendResult), JD); + }, + ES, std::move(NewInitSymbols)); } -void MachOPlatform::registerInitInfo(JITDylib &JD, - JITTargetAddress ObjCImageInfoAddr, - ExecutorAddressRange ModInits, - ExecutorAddressRange ObjCSelRefs, - ExecutorAddressRange ObjCClassList) { - std::lock_guard Lock(InitSeqsMutex); +void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getInitializers(\"" << JDName << "\")\n"; + }); - auto &InitSeq = InitSeqs[&JD]; + JITDylib *JD = ES.getJITDylibByName(JDName); + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; + }); + SendResult(make_error("No JITDylib named " + JDName, + inconvertibleErrorCode())); + return; + } - InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); - - if (ModInits.StartAddress) - InitSeq.addModInitsSection(std::move(ModInits)); - - if (ObjCSelRefs.StartAddress) - InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); - - if (ObjCClassList.StartAddress) - InitSeq.addObjCClassListSection(std::move(ObjCClassList)); + getInitializersLookupPhase(std::move(SendResult), *JD); } -static Expected getSectionExtent(jitlink::LinkGraph &G, - StringRef SectionName) { - auto *Sec = G.findSectionByName(SectionName); - if (!Sec) - return ExecutorAddressRange(); - jitlink::SectionRange R(*Sec); - if (R.getSize() % G.getPointerSize() != 0) - return make_error(SectionName + " section size is not a " - "multiple of the pointer size", +void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getDeinitializers(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + SendResult(MachOJITDylibDeinitializerSequence()); +} + +void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddress Handle, + StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + // FIXME: Proper mangling. + auto MangledName = ("_" + SymbolName).str(); + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready, + [SendResult = std::move(SendResult)](Expected Result) mutable { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddress(Result->begin()->second.getAddress())); + } else + SendResult(Result.takeError()); + }, + NoDependenciesToRegister); +} + +Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { + + std::pair Symbols[] = { + {"___orc_rt_macho_platform_bootstrap", &orc_rt_macho_platform_bootstrap}, + {"___orc_rt_macho_platform_shutdown", &orc_rt_macho_platform_shutdown}, + {"___orc_rt_macho_register_object_sections", + &orc_rt_macho_register_object_sections}}; + + SymbolLookupSet RuntimeSymbols; + std::vector> AddrsToRecord; + for (const auto &KV : Symbols) { + auto Name = ES.intern(KV.first); + RuntimeSymbols.add(Name); + AddrsToRecord.push_back({std::move(Name), KV.second}); + } + + auto RuntimeSymbolAddrs = ES.lookup( + {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); + if (!RuntimeSymbolAddrs) + return RuntimeSymbolAddrs.takeError(); + + for (const auto &KV : AddrsToRecord) { + auto &Name = KV.first; + assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); + KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + } + + if (auto Err = + EPC.runSPSWrapper(orc_rt_macho_platform_bootstrap.getValue())) + return Err; + + // FIXME: Ordering is fuzzy here. We're probably best off saying + // "behavior is undefined if code that uses the runtime is added before + // the platform constructor returns", then move all this to the constructor. + RuntimeBootstrapped = true; + std::vector DeferredPOSRs; + { + std::lock_guard Lock(PlatformMutex); + DeferredPOSRs = std::move(BootstrapPOSRs); + } + + for (auto &D : DeferredPOSRs) + if (auto Err = registerPerObjectSections(D)) + return Err; + + return Error::success(); +} + +Error MachOPlatform::registerInitInfo( + JITDylib &JD, ArrayRef InitSections) { + + std::unique_lock Lock(PlatformMutex); + + MachOJITDylibInitializers *InitSeq = nullptr; + { + auto I = InitSeqs.find(&JD); + if (I == InitSeqs.end()) { + // If there's no init sequence entry yet then we need to look up the + // header symbol to force creation of one. + Lock.unlock(); + + auto SearchOrder = + JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); + if (auto Err = ES.lookup(SearchOrder, MachOHeaderStartSymbol).takeError()) + return Err; + + Lock.lock(); + I = InitSeqs.find(&JD); + assert(I != InitSeqs.end() && + "Entry missing after header symbol lookup?"); + } + InitSeq = &I->second; + } + + for (auto *Sec : InitSections) { + // FIXME: Avoid copy here. + jitlink::SectionRange R(*Sec); + InitSeq->InitSections[Sec->getName()].push_back( + {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())}); + } + + return Error::success(); +} + +Error MachOPlatform::registerPerObjectSections( + const MachOPerObjectSectionsToRegister &POSR) { + + if (!orc_rt_macho_register_object_sections) + return make_error("Attempting to register per-object " + "sections, but runtime support has not " + "been loaded yet", inconvertibleErrorCode()); - return ExecutorAddressRange(ExecutorAddress(R.getStart()), - ExecutorAddress(R.getEnd())); + + Error ErrResult = Error::success(); + if (auto Err = EPC.runSPSWrapper( + orc_rt_macho_register_object_sections.getValue(), ErrResult, POSR)) + return Err; + return ErrResult; } -void MachOPlatform::InitScraperPlugin::modifyPassConfig( +void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { - if (!MR.getInitializerSymbol()) + // If the initializer symbol is the MachOHeader start symbol then just add + // the macho header support passes. + if (MR.getInitializerSymbol() == MP.MachOHeaderStartSymbol) { + addMachOHeaderSupportPasses(MR, Config); + // The header materialization unit doesn't require any other support, so we + // can bail out early. return; + } - Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { - JITLinkSymbolSet InitSectionSyms; - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__mod_init_func"); - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__objc_selrefs"); - preserveInitSectionIfPresent(InitSectionSyms, G, "__DATA,__objc_classlist"); + // If the object contains initializers then add passes to record them. + if (MR.getInitializerSymbol()) + addInitializerSupportPasses(MR, Config); - if (!InitSectionSyms.empty()) { - std::lock_guard Lock(InitScraperMutex); - InitSymbolDeps[&MR] = std::move(InitSectionSyms); - } - - if (auto Err = processObjCImageInfo(G, MR)) - return Err; - - return Error::success(); - }); - - Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( - jitlink::LinkGraph &G) -> Error { - ExecutorAddressRange ModInits, ObjCSelRefs, ObjCClassList; - - JITTargetAddress ObjCImageInfoAddr = 0; - if (auto *ObjCImageInfoSec = - G.findSectionByName("__DATA,__objc_image_info")) { - if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) - ObjCImageInfoAddr = Addr; - } - - // Record __mod_init_func. - if (auto ModInitsOrErr = getSectionExtent(G, "__DATA,__mod_init_func")) - ModInits = std::move(*ModInitsOrErr); - else - return ModInitsOrErr.takeError(); - - // Record __objc_selrefs. - if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__DATA,__objc_selrefs")) - ObjCSelRefs = std::move(*ObjCSelRefsOrErr); - else - return ObjCSelRefsOrErr.takeError(); - - // Record __objc_classlist. - if (auto ObjCClassListOrErr = - getSectionExtent(G, "__DATA,__objc_classlist")) - ObjCClassList = std::move(*ObjCClassListOrErr); - else - return ObjCClassListOrErr.takeError(); - - // Dump the scraped inits. - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; - dbgs() << " __objc_selrefs: "; - auto NumObjCSelRefs = ObjCSelRefs.size().getValue() / sizeof(uintptr_t); - if (NumObjCSelRefs) - dbgs() << NumObjCSelRefs << " pointer(s) at " - << formatv("{0:x16}", ObjCSelRefs.StartAddress.getValue()) - << "\n"; - else - dbgs() << "none\n"; - - dbgs() << " __objc_classlist: "; - auto NumObjCClasses = ObjCClassList.size().getValue() / sizeof(uintptr_t); - if (NumObjCClasses) - dbgs() << NumObjCClasses << " pointer(s) at " - << formatv("{0:x16}", ObjCClassList.StartAddress.getValue()) - << "\n"; - else - dbgs() << "none\n"; - - dbgs() << " __mod_init_func: "; - auto NumModInits = ModInits.size().getValue() / sizeof(uintptr_t); - if (NumModInits) - dbgs() << NumModInits << " pointer(s) at " - << formatv("{0:x16}", ModInits.StartAddress.getValue()) << "\n"; - else - dbgs() << "none\n"; - }); - - MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), - std::move(ObjCSelRefs), std::move(ObjCClassList)); - - return Error::success(); - }); + // Add passes for eh-frame support. + addEHSupportPasses(MR, Config); } ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap -MachOPlatform::InitScraperPlugin::getSyntheticSymbolDependencies( +MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( MaterializationResponsibility &MR) { - std::lock_guard Lock(InitScraperMutex); + std::lock_guard Lock(PluginMutex); auto I = InitSymbolDeps.find(&MR); if (I != InitSymbolDeps.end()) { SyntheticSymbolDependenciesMap Result; @@ -275,93 +596,135 @@ MachOPlatform::InitScraperPlugin::getSyntheticSymbolDependencies( return SyntheticSymbolDependenciesMap(); } -void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( - JITLinkSymbolSet &Symbols, jitlink::LinkGraph &G, StringRef SectionName) { - if (auto *Sec = G.findSectionByName(SectionName)) { - auto SecBlocks = Sec->blocks(); - if (!llvm::empty(SecBlocks)) - Symbols.insert( - &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); - } +void MachOPlatform::MachOPlatformPlugin::addInitializerSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + /// Preserve init sections. + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + return preserveInitSections(G, MR); + }); + + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerInitSections(G, JD); + }); } -Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( +void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *MP.MachOHeaderStartSymbol; + }); + assert(I != G.defined_symbols().end() && + "Missing MachO header start symbol"); + { + std::lock_guard Lock(MP.PlatformMutex); + JITTargetAddress HeaderAddr = (*I)->getAddress(); + MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); + MP.InitSeqs.insert( + std::make_pair(&JD, MachOJITDylibInitializers( + JD.getName(), ExecutorAddress(HeaderAddr)))); + } + return Error::success(); + }); +} + +void MachOPlatform::MachOPlatformPlugin::addEHSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + // Add a pass to register the final addresses of the eh-frame sections + // with the runtime. + Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { + MachOPerObjectSectionsToRegister POSR; + + if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + jitlink::SectionRange R(*EHFrameSection); + if (!R.empty()) + POSR.EHFrameSection = {ExecutorAddress(R.getStart()), + ExecutorAddress(R.getEnd())}; + } + + if (POSR.EHFrameSection.StartAddress) { + + // If we're still bootstrapping the runtime then just record this + // frame for now. + if (!MP.RuntimeBootstrapped) { + std::lock_guard Lock(MP.PlatformMutex); + MP.BootstrapPOSRs.push_back(POSR); + return Error::success(); + } + + // Otherwise register it immediately. + if (auto Err = MP.registerPerObjectSections(POSR)) + return Err; + } + + return Error::success(); + }); +} + +Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { - // If there's an ObjC imagine info then either - // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In - // this case we name and record it. - // OR - // (2) We already have a recorded __objc_imageinfo for this JITDylib, - // in which case we just verify it. - auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); - if (!ObjCImageInfo) - return Error::success(); + JITLinkSymbolSet InitSectionSymbols; + for (auto &InitSectionName : InitSectionNames) { + // Skip non-init sections. + auto *InitSection = G.findSectionByName(InitSectionName); + if (!InitSection) + continue; - auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); + // Make a pass over live symbols in the section: those blocks are already + // preserved. + DenseSet AlreadyLiveBlocks; + for (auto &Sym : InitSection->symbols()) { + auto &B = Sym->getBlock(); + if (Sym->isLive() && Sym->getOffset() == 0 && + Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) { + InitSectionSymbols.insert(Sym); + AlreadyLiveBlocks.insert(&B); + } + } - // Check that the section is not empty if present. - if (llvm::empty(ObjCImageInfoBlocks)) - return make_error("Empty __objc_imageinfo section in " + - G.getName(), - inconvertibleErrorCode()); - - // Check that there's only one block in the section. - if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) - return make_error("Multiple blocks in __objc_imageinfo " - "section in " + - G.getName(), - inconvertibleErrorCode()); - - // Check that the __objc_imageinfo section is unreferenced. - // FIXME: We could optimize this check if Symbols had a ref-count. - for (auto &Sec : G.sections()) { - if (&Sec != ObjCImageInfo) - for (auto *B : Sec.blocks()) - for (auto &E : B->edges()) - if (E.getTarget().isDefined() && - &E.getTarget().getBlock().getSection() == ObjCImageInfo) - return make_error("__objc_imageinfo is referenced " - "within file " + - G.getName(), - inconvertibleErrorCode()); + // Add anonymous symbols to preserve any not-already-preserved blocks. + for (auto *B : InitSection->blocks()) + if (!AlreadyLiveBlocks.count(B)) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true)); } - auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); - auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); - auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness()); - auto Flags = - support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); - - // Lock the mutex while we verify / update the ObjCImageInfos map. - std::lock_guard Lock(InitScraperMutex); - - auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); - if (ObjCImageInfoItr != ObjCImageInfos.end()) { - // We've already registered an __objc_imageinfo section. Verify the - // content of this new section matches, then delete it. - if (ObjCImageInfoItr->second.first != Version) - return make_error( - "ObjC version in " + G.getName() + - " does not match first registered version", - inconvertibleErrorCode()); - if (ObjCImageInfoItr->second.second != Flags) - return make_error("ObjC flags in " + G.getName() + - " do not match first registered flags", - inconvertibleErrorCode()); - - // __objc_imageinfo is valid. Delete the block. - for (auto *S : ObjCImageInfo->symbols()) - G.removeDefinedSymbol(*S); - G.removeBlock(ObjCImageInfoBlock); - } else { - // We haven't registered an __objc_imageinfo section yet. Register and - // move on. The section should already be marked no-dead-strip. - ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + if (!InitSectionSymbols.empty()) { + std::lock_guard Lock(PluginMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); } return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::registerInitSections( + jitlink::LinkGraph &G, JITDylib &JD) { + + SmallVector InitSections; + + for (auto InitSectionName : InitSectionNames) + if (auto *Sec = G.findSectionByName(InitSectionName)) + InitSections.push_back(Sec); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + for (auto *Sec : InitSections) { + jitlink::SectionRange R(*Sec); + dbgs() << " " << Sec->getName() << ": " + << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + } + }); + + return MP.registerInitInfo(JD, InitSections); +} + } // End namespace orc. } // End namespace llvm. diff --git a/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp b/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp index 9a92581157c..7f197a50c39 100644 --- a/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp +++ b/tools/llvm-jitlink/llvm-jitlink-executor/llvm-jitlink-executor.cpp @@ -141,7 +141,7 @@ int main(int argc, char *argv[]) { ExitOnErr.setBanner(std::string(argv[0]) + ":"); using JITLinkExecutorEndpoint = - shared::MultiThreadedRPCEndpoint; + shared::SingleThreadedRPCEndpoint; shared::registerStringError(); diff --git a/tools/llvm-jitlink/llvm-jitlink.cpp b/tools/llvm-jitlink/llvm-jitlink.cpp index f8298317fee..ec91597bc35 100644 --- a/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/tools/llvm-jitlink/llvm-jitlink.cpp @@ -20,6 +20,7 @@ #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/MC/MCAsmInfo.h" @@ -159,6 +160,15 @@ static cl::opt OutOfProcessExecutorConnect( "oop-executor-connect", cl::desc("Connect to an out-of-process executor via TCP")); +// TODO: Default to false if compiler-rt is not built. +static cl::opt UseOrcRuntime("use-orc-runtime", + cl::desc("Do not required/load ORC runtime"), + cl::init(true)); + +static cl::opt + OrcRuntimePath("orc-runtime-path", cl::desc("Add orc runtime to session"), + cl::init("")); + ExitOnError ExitOnErr; LLVM_ATTRIBUTE_USED void linkComponents() { @@ -167,6 +177,14 @@ LLVM_ATTRIBUTE_USED void linkComponents() { << (void *)&llvm_orc_registerJITLoaderGDBWrapper; } +static bool UseTestResultOverride = false; +static int64_t TestResultOverride = 0; + +extern "C" void llvm_jitlink_setTestResultOverride(int64_t Value) { + TestResultOverride = Value; + UseTestResultOverride = true; +} + namespace llvm { static raw_ostream & @@ -588,6 +606,31 @@ Error LLVMJITLinkObjectLinkingLayer::add(ResourceTrackerSP RT, return JD.define(std::move(MU), std::move(RT)); } +static Error loadProcessSymbols(Session &S) { + auto FilterMainEntryPoint = + [EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) { + return Name != EPName; + }; + S.MainJD->addGenerator( + ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess( + *S.EPC, std::move(FilterMainEntryPoint)))); + + return Error::success(); +} + +static Error loadDylibs(Session &S) { + LLVM_DEBUG(dbgs() << "Loading dylibs...\n"); + for (const auto &Dylib : Dylibs) { + LLVM_DEBUG(dbgs() << " " << Dylib << "\n"); + auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str()); + if (!G) + return G.takeError(); + S.MainJD->addGenerator(std::move(*G)); + } + + return Error::success(); +} + Expected> LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor() { #ifndef LLVM_ON_UNIX @@ -796,40 +839,46 @@ Expected> Session::Create(Triple TT) { if (!PageSize) return PageSize.takeError(); - /// If -oop-executor is passed then launch the executor. std::unique_ptr EPC; if (OutOfProcessExecutor.getNumOccurrences()) { + /// If -oop-executor is passed then launch the executor. if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::LaunchExecutor()) EPC = std::move(*REPC); else return REPC.takeError(); } else if (OutOfProcessExecutorConnect.getNumOccurrences()) { + /// If -oop-executor-connect is passed then connect to the executor. if (auto REPC = LLVMJITLinkRemoteExecutorProcessControl::ConnectToExecutor()) EPC = std::move(*REPC); else return REPC.takeError(); - } else + } else { + /// Otherwise use SelfExecutorProcessControl to target the current process. EPC = std::make_unique( std::make_shared(), std::move(TT), *PageSize, createMemoryManager()); + } Error Err = Error::success(); std::unique_ptr S(new Session(std::move(EPC), Err)); - if (Err) - return std::move(Err); - return std::move(S); + + // FIXME: Errors destroy the session, leaving the SymbolStringPtrs dangling, + // so just exit here. We could fix this by having errors keep the pool alive. + ExitOnErr(std::move(Err)); + return S; } Session::~Session() { if (auto Err = ES.endSession()) ES.reportError(std::move(Err)); + if (auto Err = EPC->disconnect()) + ES.reportError(std::move(Err)); } -// FIXME: Move to createJITDylib if/when we start using Platform support in -// llvm-jitlink. Session::Session(std::unique_ptr EPC, Error &Err) - : EPC(std::move(EPC)), ObjLayer(*this, this->EPC->getMemMgr()) { + : EPC(std::move(EPC)), ES(this->EPC->getSymbolStringPool()), + ObjLayer(*this, this->EPC->getMemMgr()) { /// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the /// Session. @@ -863,7 +912,21 @@ Session::Session(std::unique_ptr EPC, Error &Err) return; } - if (!NoExec && !this->EPC->getTargetTriple().isOSWindows()) { + if (!NoProcessSymbols) + ExitOnErr(loadProcessSymbols(*this)); + ExitOnErr(loadDylibs(*this)); + + // Set up the platform. + if (this->EPC->getTargetTriple().isOSBinFormatMachO() && UseOrcRuntime) { + if (auto P = MachOPlatform::Create(ES, ObjLayer, *this->EPC, *MainJD, + OrcRuntimePath.c_str())) + ES.setPlatform(std::move(*P)); + else { + Err = P.takeError(); + return; + } + } else if (!NoExec && !this->EPC->getTargetTriple().isOSWindows() && + !this->EPC->getTargetTriple().isOSBinFormatMachO()) { ObjLayer.addPlugin(std::make_unique( ES, ExitOnErr(EPCEHFrameRegistrar::Create(*this->EPC)))); ObjLayer.addPlugin(std::make_unique( @@ -1044,18 +1107,41 @@ static Triple getFirstFileTriple() { return FirstTT; } -static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { - // Set the entry point name if not specified. - if (EntryPointName.empty()) { - if (TT.getObjectFormat() == Triple::MachO) - EntryPointName = "_main"; - else - EntryPointName = "main"; +static bool isOrcRuntimeSupported(const Triple &TT) { + switch (TT.getObjectFormat()) { + case Triple::MachO: + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } + default: + return false; } +} + +static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { + + // If we're in noexec mode and the user didn't explicitly specify + // -use-orc-runtime then don't use it. + if (NoExec && UseOrcRuntime.getNumOccurrences() == 0) + UseOrcRuntime = false; // -noexec and --args should not be used together. if (NoExec && !InputArgv.empty()) - outs() << "Warning: --args passed to -noexec run will be ignored.\n"; + errs() << "Warning: --args passed to -noexec run will be ignored.\n"; + + // Turn off UseOrcRuntime on platforms where it's not supported. + if (UseOrcRuntime && !isOrcRuntimeSupported(TT)) { + errs() << "Warning: Orc runtime not available for target platform. " + "Use -use-orc-runtime=false to suppress this warning.\n"; + UseOrcRuntime = false; + } + + // Set the entry point name if not specified. + if (EntryPointName.empty()) + EntryPointName = TT.getObjectFormat() == Triple::MachO ? "_main" : "main"; // If -slab-address is passed, require -slab-allocate and -noexec if (SlabAddress != ~0ULL) { @@ -1080,35 +1166,27 @@ static Error sanitizeArguments(const Triple &TT, const char *ArgV0) { SmallString<256> OOPExecutorPath(sys::fs::getMainExecutable( ArgV0, reinterpret_cast(&sanitizeArguments))); sys::path::remove_filename(OOPExecutorPath); - if (OOPExecutorPath.back() != '/') - OOPExecutorPath += '/'; - OOPExecutorPath += "llvm-jitlink-executor"; + sys::path::append(OOPExecutorPath, "llvm-jitlink-executor"); OutOfProcessExecutor = OOPExecutorPath.str().str(); } - return Error::success(); -} - -static Error loadProcessSymbols(Session &S) { - auto FilterMainEntryPoint = - [EPName = S.ES.intern(EntryPointName)](SymbolStringPtr Name) { - return Name != EPName; - }; - S.MainJD->addGenerator( - ExitOnErr(orc::EPCDynamicLibrarySearchGenerator::GetForTargetProcess( - *S.EPC, std::move(FilterMainEntryPoint)))); - - return Error::success(); -} - -static Error loadDylibs(Session &S) { - for (const auto &Dylib : Dylibs) { - auto G = orc::EPCDynamicLibrarySearchGenerator::Load(*S.EPC, Dylib.c_str()); - if (!G) - return G.takeError(); - S.MainJD->addGenerator(std::move(*G)); + // If we're loading the Orc runtime then determine the path for it. + if (UseOrcRuntime) { + if (OrcRuntimePath.empty()) { + SmallString<256> DefaultOrcRuntimePath(sys::fs::getMainExecutable( + ArgV0, reinterpret_cast(&sanitizeArguments))); + sys::path::remove_filename( + DefaultOrcRuntimePath); // remove 'llvm-jitlink' + while (!DefaultOrcRuntimePath.empty() && + DefaultOrcRuntimePath.back() == '/') + DefaultOrcRuntimePath.pop_back(); + if (DefaultOrcRuntimePath.endswith("bin")) + sys::path::remove_filename(DefaultOrcRuntimePath); // remove 'bin' + sys::path::append(DefaultOrcRuntimePath, + "lib/clang/13.0.0/lib/darwin/libclang_rt.orc_osx.a"); + OrcRuntimePath = DefaultOrcRuntimePath.str().str(); + } } - return Error::success(); } @@ -1227,6 +1305,11 @@ static Error loadObjects(Session &S) { static Error runChecks(Session &S) { + if (CheckFiles.empty()) + return Error::success(); + + LLVM_DEBUG(dbgs() << "Running checks...\n"); + auto TripleName = S.EPC->getTargetTriple().str(); std::string ErrorStr; const Target *TheTarget = TargetRegistry::lookupTarget(TripleName, ErrorStr); @@ -1310,9 +1393,15 @@ static Error runChecks(Session &S) { } static void dumpSessionStats(Session &S) { + if (!ShowSizes) + return; + if (UseOrcRuntime) + outs() << "Note: Session stats include runtime and entry point lookup, but " + "not JITDylib initialization/deinitialization.\n"; if (ShowSizes) - outs() << "Total size of all blocks before pruning: " << S.SizeBeforePruning - << "\nTotal size of all blocks after fixups: " << S.SizeAfterFixups + outs() << " Total size of all blocks before pruning: " + << S.SizeBeforePruning + << "\n Total size of all blocks after fixups: " << S.SizeAfterFixups << "\n"; } @@ -1320,6 +1409,34 @@ static Expected getMainEntryPoint(Session &S) { return S.ES.lookup(S.JDSearchOrder, EntryPointName); } +static Expected getOrcRuntimeEntryPoint(Session &S) { + std::string RuntimeEntryPoint = "__orc_rt_run_program_wrapper"; + if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO) + RuntimeEntryPoint = '_' + RuntimeEntryPoint; + return S.ES.lookup(S.JDSearchOrder, RuntimeEntryPoint); +} + +static Expected runWithRuntime(Session &S, + JITTargetAddress EntryPointAddress) { + StringRef DemangledEntryPoint = EntryPointName; + if (S.EPC->getTargetTriple().getObjectFormat() == Triple::MachO && + DemangledEntryPoint.front() == '_') + DemangledEntryPoint = DemangledEntryPoint.drop_front(); + using SPSRunProgramSig = + int64_t(SPSString, SPSString, SPSSequence); + int64_t Result; + if (auto Err = S.EPC->runSPSWrapper( + EntryPointAddress, Result, S.MainJD->getName(), DemangledEntryPoint, + static_cast &>(InputArgv))) + return std::move(Err); + return Result; +} + +static Expected runWithoutRuntime(Session &S, + JITTargetAddress EntryPointAddress) { + return S.EPC->runAsMain(EntryPointAddress, InputArgv); +} + namespace { struct JITLinkTimers { TimerGroup JITLinkTG{"llvm-jitlink timers", "timers for llvm-jitlink phases"}; @@ -1352,21 +1469,23 @@ int main(int argc, char *argv[]) { ExitOnErr(loadObjects(*S)); } - if (!NoProcessSymbols) - ExitOnErr(loadProcessSymbols(*S)); - ExitOnErr(loadDylibs(*S)); - if (PhonyExternals) addPhonyExternalsGenerator(*S); - if (ShowInitialExecutionSessionState) S->ES.dump(outs()); JITEvaluatedSymbol EntryPoint = 0; { TimeRegion TR(Timers ? &Timers->LinkTimer : nullptr); + // Find the entry-point function unconditionally, since we want to force + // it to be materialized to collect stats. EntryPoint = ExitOnErr(getMainEntryPoint(*S)); + + // If we're running with the ORC runtime then replace the entry-point + // with the __orc_rt_run_program symbol. + if (UseOrcRuntime) + EntryPoint = ExitOnErr(getOrcRuntimeEntryPoint(*S)); } if (ShowAddrs) @@ -1381,12 +1500,20 @@ int main(int argc, char *argv[]) { int Result = 0; { + LLVM_DEBUG(dbgs() << "Running \"" << EntryPointName << "\"...\n"); TimeRegion TR(Timers ? &Timers->RunTimer : nullptr); - Result = ExitOnErr(S->EPC->runAsMain(EntryPoint.getAddress(), InputArgv)); + if (UseOrcRuntime) + Result = ExitOnErr(runWithRuntime(*S, EntryPoint.getAddress())); + else + Result = ExitOnErr(runWithoutRuntime(*S, EntryPoint.getAddress())); } - ExitOnErr(S->ES.endSession()); - ExitOnErr(S->EPC->disconnect()); + // Destroy the session. + S.reset(); + + // If the executing code set a test result override then use that. + if (UseTestResultOverride) + Result = TestResultOverride; return Result; } diff --git a/tools/llvm-jitlink/llvm-jitlink.h b/tools/llvm-jitlink/llvm-jitlink.h index 750e543fba1..5050b2fef75 100644 --- a/tools/llvm-jitlink/llvm-jitlink.h +++ b/tools/llvm-jitlink/llvm-jitlink.h @@ -111,7 +111,7 @@ private: struct Session { std::unique_ptr EPC; orc::ExecutionSession ES; - orc::JITDylib *MainJD; + orc::JITDylib *MainJD = nullptr; LLVMJITLinkObjectLinkingLayer ObjLayer; std::vector JDSearchOrder; diff --git a/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp b/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp index 5c784c16a4c..f4a6b458889 100644 --- a/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp +++ b/unittests/ExecutionEngine/Orc/SimplePackedSerializationTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/ExecutionEngine/Orc/LLVMSPSSerializers.h" #include "gtest/gtest.h" using namespace llvm; @@ -158,3 +159,9 @@ TEST(SimplePackedSerializationTest, ArgListSerialization) { EXPECT_EQ(Arg2, ArgOut2); EXPECT_EQ(Arg3, ArgOut3); } + +TEST(SimplePackedSerialization, StringMap) { + StringMap M({{"A", 1}, {"B", 2}}); + blobSerializationRoundTrip>, + StringMap>(M); +}