From 900dc7edc7f38ae687c2bff8468450a079d5d6df Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Mon, 16 Dec 2019 02:50:40 -0800 Subject: [PATCH] [ORC] Add generic initializer/deinitializer support. Initializers and deinitializers are used to implement C++ static constructors and destructors, runtime registration for some languages (e.g. with the Objective-C runtime for Objective-C/C++ code) and other tasks that would typically be performed when a shared-object/dylib is loaded or unloaded by a statically compiled program. MCJIT and ORC have historically provided limited support for discovering and running initializers/deinitializers by scanning the llvm.global_ctors and llvm.global_dtors variables and recording the functions to be run. This approach suffers from several drawbacks: (1) It only works for IR inputs, not for object files (including cached JIT'd objects). (2) It only works for initializers described by llvm.global_ctors and llvm.global_dtors, however not all initializers are described in this way (Objective-C, for example, describes initializers via specially named metadata sections). (3) To make the initializer/deinitializer functions described by llvm.global_ctors and llvm.global_dtors searchable they must be promoted to extern linkage, polluting the JIT symbol table (extra care must be taken to ensure this promotion does not result in symbol name clashes). This patch introduces several interdependent changes to ORCv2 to support the construction of new initialization schemes, and includes an implementation of a backwards-compatible llvm.global_ctor/llvm.global_dtor scanning scheme, and a MachO specific scheme that handles Objective-C runtime registration (if the Objective-C runtime is available) enabling execution of LLVM IR compiled from Objective-C and Swift. The major changes included in this patch are: (1) The MaterializationUnit and MaterializationResponsibility classes are extended to describe an optional "initializer" symbol for the module (see the getInitializerSymbol method on each class). The presence or absence of this symbol indicates whether the module contains any initializers or deinitializers. The initializer symbol otherwise behaves like any other: searching for it triggers materialization. (2) A new Platform interface is introduced in llvm/ExecutionEngine/Orc/Core.h which provides the following callback interface: - Error setupJITDylib(JITDylib &JD): Can be used to install standard symbols in JITDylibs upon creation. E.g. __dso_handle. - Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU): Generally used to record initializer symbols. - Error notifyRemoving(JITDylib &JD, VModuleKey K): Used to notify a platform that a module is being removed. Platform implementations can use these callbacks to track outstanding initializers and implement a platform-specific approach for executing them. For example, the MachOPlatform installs a plugin in the JIT linker to scan for both __mod_inits sections (for C++ static constructors) and ObjC metadata sections. If discovered, these are processed in the usual platform order: Objective-C registration is carried out first, then static initializers are executed, ensuring that calls to Objective-C from static initializers will be safe. This patch updates LLJIT to use the new scheme for initialization. Two LLJIT::PlatformSupport classes are implemented: A GenericIR platform and a MachO platform. The GenericIR platform implements a modified version of the previous llvm.global-ctor scraping scheme to provide support for Windows and Linux. LLJIT's MachO platform uses the MachOPlatform class to provide MachO specific initialization as described above. Reviewers: sgraenitz, dblaikie Subscribers: mgorny, hiraditya, mgrang, ributzka, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D74300 --- .../BuildingAJIT/Chapter1/KaleidoscopeJIT.h | 2 +- .../BuildingAJIT/Chapter2/KaleidoscopeJIT.h | 2 +- .../llvm/ExecutionEngine/JITLink/JITLink.h | 2 + .../Orc/CompileOnDemandLayer.h | 1 + .../llvm/ExecutionEngine/Orc/CompileUtils.h | 4 +- include/llvm/ExecutionEngine/Orc/Core.h | 122 ++- .../llvm/ExecutionEngine/Orc/ExecutionUtils.h | 64 ++ .../llvm/ExecutionEngine/Orc/IRCompileLayer.h | 10 +- .../ExecutionEngine/Orc/IRTransformLayer.h | 7 +- .../ExecutionEngine/Orc/IndirectionUtils.h | 2 +- include/llvm/ExecutionEngine/Orc/LLJIT.h | 103 ++- include/llvm/ExecutionEngine/Orc/Layer.h | 33 +- .../llvm/ExecutionEngine/Orc/MachOPlatform.h | 147 +++ include/llvm/ExecutionEngine/Orc/Mangling.h | 66 ++ .../ExecutionEngine/Orc/ObjectLinkingLayer.h | 14 + .../ExecutionEngine/Orc/SymbolStringPool.h | 3 + lib/ExecutionEngine/Orc/CMakeLists.txt | 2 + .../Orc/CompileOnDemandLayer.cpp | 49 +- lib/ExecutionEngine/Orc/CompileUtils.cpp | 4 +- lib/ExecutionEngine/Orc/Core.cpp | 118 ++- lib/ExecutionEngine/Orc/ExecutionUtils.cpp | 44 + lib/ExecutionEngine/Orc/IndirectionUtils.cpp | 2 +- lib/ExecutionEngine/Orc/LLJIT.cpp | 873 +++++++++++++++++- lib/ExecutionEngine/Orc/Layer.cpp | 82 +- lib/ExecutionEngine/Orc/LazyReexports.cpp | 7 +- lib/ExecutionEngine/Orc/MachOPlatform.cpp | 459 +++++++++ lib/ExecutionEngine/Orc/Mangling.cpp | 131 +++ .../Orc/ObjectLinkingLayer.cpp | 75 +- .../Orc/RTDyldObjectLinkingLayer.cpp | 6 +- test/ExecutionEngine/OrcLazy/objc-minimal.ll | 73 ++ tools/lli/lli.cpp | 63 +- tools/llvm-jitlink/llvm-jitlink.cpp | 58 +- tools/llvm-jitlink/llvm-jitlink.h | 6 +- .../ExecutionEngine/Orc/CoreAPIsTest.cpp | 15 +- unittests/ExecutionEngine/Orc/OrcTestCommon.h | 6 +- .../Orc/RTDyldObjectLinkingLayerTest.cpp | 6 +- 36 files changed, 2408 insertions(+), 253 deletions(-) create mode 100644 include/llvm/ExecutionEngine/Orc/MachOPlatform.h create mode 100644 include/llvm/ExecutionEngine/Orc/Mangling.h create mode 100644 lib/ExecutionEngine/Orc/MachOPlatform.cpp create mode 100644 lib/ExecutionEngine/Orc/Mangling.cpp create mode 100644 test/ExecutionEngine/OrcLazy/objc-minimal.ll diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h index b7404baf1ff..adb6be43666 100644 --- a/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h +++ b/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h @@ -49,7 +49,7 @@ public: std::make_unique(std::move(JTMB))), DL(std::move(DL)), Mangle(ES, this->DL), Ctx(std::make_unique()), - MainJD(ES.createJITDylib("
")) { + MainJD(ES.createBareJITDylib("
")) { MainJD.addGenerator( cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess( DL.getGlobalPrefix()))); diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h index efb19349e3e..28649ffe491 100644 --- a/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h +++ b/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h @@ -55,7 +55,7 @@ public: std::make_unique(std::move(JTMB))), OptimizeLayer(ES, CompileLayer, optimizeModule), DL(std::move(DL)), Mangle(ES, this->DL), Ctx(std::make_unique()), - MainJD(ES.createJITDylib("
")) { + MainJD(ES.createBareJITDylib("
")) { MainJD.addGenerator( cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess( DL.getGlobalPrefix()))); diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h index 1a8f5cacbcd..d673a89386d 100644 --- a/include/llvm/ExecutionEngine/JITLink/JITLink.h +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -488,6 +488,8 @@ public: /// Set the visibility for this Symbol. void setScope(Scope S) { + assert((!Name.empty() || S == Scope::Local) && + "Can not set anonymous symbol to non-local scope"); assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && "Invalid visibility for symbol type"); this->S = static_cast(S); diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 65295f97e2c..ef251ac302d 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -94,6 +94,7 @@ public: /// Sets the ImplSymbolMap void setImplMap(ImplSymbolMap *Imp); + /// Emits the given module. This should not be called by clients: it will be /// called by the JIT when a definition added via the add method is requested. void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; diff --git a/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/include/llvm/ExecutionEngine/Orc/CompileUtils.h index 218afda1b54..8376d163d57 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -30,7 +30,7 @@ namespace orc { class JITTargetMachineBuilder; -IRMaterializationUnit::ManglingOptions +IRSymbolMapper::ManglingOptions irManglingOptionsFromTargetOptions(const TargetOptions &Opts); /// Simple compile functor: Takes a single IR module and returns an ObjectFile. @@ -52,7 +52,7 @@ public: Expected operator()(Module &M) override; private: - IRMaterializationUnit::ManglingOptions + IRSymbolMapper::ManglingOptions manglingOptionsForTargetMachine(const TargetMachine &TM); CompileResult tryToLoadFromObjectCache(const Module &M); diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 1a55801b9d8..94600f2286b 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -14,11 +14,11 @@ #define LLVM_EXECUTIONENGINE_ORC_CORE_H #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/FunctionExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/ExecutionEngine/OrcV1Deprecation.h" -#include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include @@ -456,6 +456,11 @@ public: /// before using. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + /// Returns the initialization pseudo-symbol, if any. This symbol will also + /// be present in the SymbolFlagsMap for this MaterializationResponsibility + /// object. + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + /// Returns the names of any symbols covered by this /// MaterializationResponsibility object that have queries pending. This /// information can be used to return responsibility for unrequested symbols @@ -532,10 +537,15 @@ private: /// Create a MaterializationResponsibility for the given JITDylib and /// initial symbols. MaterializationResponsibility(JITDylib &JD, SymbolFlagsMap SymbolFlags, - VModuleKey K); + SymbolStringPtr InitSymbol, VModuleKey K) + : JD(JD), SymbolFlags(std::move(SymbolFlags)), + InitSymbol(std::move(InitSymbol)), K(std::move(K)) { + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); + } JITDylib &JD; SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; VModuleKey K; }; @@ -549,8 +559,13 @@ private: /// stronger definition is added or already present. class MaterializationUnit { public: - MaterializationUnit(SymbolFlagsMap InitalSymbolFlags, VModuleKey K) - : SymbolFlags(std::move(InitalSymbolFlags)), K(std::move(K)) {} + MaterializationUnit(SymbolFlagsMap InitalSymbolFlags, + SymbolStringPtr InitSymbol, VModuleKey K) + : SymbolFlags(std::move(InitalSymbolFlags)), + InitSymbol(std::move(InitSymbol)), K(std::move(K)) { + assert((!this->InitSymbol || this->SymbolFlags.count(this->InitSymbol)) && + "If set, InitSymbol should appear in InitialSymbolFlags map"); + } virtual ~MaterializationUnit() {} @@ -561,12 +576,15 @@ public: /// Return the set of symbols that this source provides. const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + /// Returns the initialization symbol for this MaterializationUnit (if any). + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + /// Called by materialization dispatchers (see /// ExecutionSession::DispatchMaterializationFunction) to trigger /// materialization of this MaterializationUnit. void doMaterialize(JITDylib &JD) { - materialize(MaterializationResponsibility(JD, std::move(SymbolFlags), - std::move(K))); + materialize(MaterializationResponsibility( + JD, std::move(SymbolFlags), std::move(InitSymbol), std::move(K))); } /// Called by JITDylibs to notify MaterializationUnits that the given symbol @@ -578,6 +596,7 @@ public: protected: SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; VModuleKey K; private: @@ -774,6 +793,7 @@ private: class JITDylib { friend class AsynchronousSymbolQuery; friend class ExecutionSession; + friend class Platform; friend class MaterializationResponsibility; public: /// Definition generators can be attached to JITDylibs to generate new @@ -1054,6 +1074,35 @@ private: JITDylibSearchOrder SearchOrder; }; +/// Platforms set up standard symbols and mediate interactions between dynamic +/// initializers (e.g. C++ static constructors) and ExecutionSession state. +/// Note that Platforms do not automatically run initializers: clients are still +/// responsible for doing this. +class Platform { +public: + virtual ~Platform(); + + /// This method will be called outside the session lock each time a JITDylib + /// is created (unless it is created with EmptyJITDylib set) to allow the + /// Platform to install any JITDylib specific standard symbols (e.g + /// __dso_handle). + virtual Error setupJITDylib(JITDylib &JD) = 0; + + /// This method will be called under the ExecutionSession lock each time a + /// MaterializationUnit is added to a JITDylib. + virtual Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) = 0; + + /// This method will be called under the ExecutionSession lock when a + /// VModuleKey is removed. + virtual Error notifyRemoving(JITDylib &JD, VModuleKey K) = 0; + + /// A utility function for looking up initializer symbols. Performs a blocking + /// lookup for the given symbols in each of the given JITDylibs. + static Expected> + lookupInitSymbols(ExecutionSession &ES, + const DenseMap &InitSyms); +}; + /// An ExecutionSession represents a running JIT program. class ExecutionSession { // FIXME: Remove this when we remove the old ORC layers. @@ -1078,6 +1127,13 @@ public: /// Returns a shared_ptr to the SymbolStringPool for this ExecutionSession. std::shared_ptr getSymbolStringPool() const { return SSP; } + /// Set the Platform for this ExecutionSession. + void setPlatform(std::unique_ptr P) { this->P = std::move(P); } + + /// Get the Platform for this session. + /// Will return null if no Platform has been set for this ExecutionSession. + Platform *getPlatform() { return P.get(); } + /// Run the given lambda with the session mutex locked. template decltype(auto) runSessionLocked(Func &&F) { std::lock_guard Lock(SessionMutex); @@ -1088,12 +1144,26 @@ public: /// Ownership of JITDylib remains within Execution Session JITDylib *getJITDylibByName(StringRef Name); + /// Add a new bare JITDylib to this ExecutionSession. + /// + /// The JITDylib Name is required to be unique. Clients should verify that + /// names are not being re-used (E.g. by calling getJITDylibByName) if names + /// are based on user input. + /// + /// This call does not install any library code or symbols into the newly + /// created JITDylib. The client is responsible for all configuration. + JITDylib &createBareJITDylib(std::string Name); + /// Add a new JITDylib to this ExecutionSession. /// /// The JITDylib Name is required to be unique. Clients should verify that /// names are not being re-used (e.g. by calling getJITDylibByName) if names /// are based on user input. - JITDylib &createJITDylib(std::string Name); + /// + /// If a Platform is attached then Platform::setupJITDylib will be called to + /// install standard platform symbols (e.g. standard library interposes). + /// If no Platform is attached this call is equivalent to createBareJITDylib. + Expected createJITDylib(std::string Name); /// Allocate a module key for a new module to add to the JIT. VModuleKey allocateVModule() { @@ -1177,20 +1247,23 @@ public: /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. - Expected lookup(const JITDylibSearchOrder &SearchOrder, - SymbolStringPtr Symbol); + Expected + lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. The search will not find non-exported symbols. - Expected lookup(ArrayRef SearchOrder, - SymbolStringPtr Symbol); + Expected + lookup(ArrayRef SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given /// symbol. The search will not find non-exported symbols. - Expected lookup(ArrayRef SearchOrder, - StringRef Symbol); + Expected + lookup(ArrayRef SearchOrder, StringRef Symbol, + SymbolState RequiredState = SymbolState::Ready); /// Materialize the given unit. void dispatchMaterialization(JITDylib &JD, @@ -1221,6 +1294,7 @@ private: mutable std::recursive_mutex SessionMutex; std::shared_ptr SSP; + std::unique_ptr P; VModuleKey LastKey = 0; ErrorReporter ReportError = logErrorsToStdErr; DispatchMaterializationFunction DispatchMaterialization = @@ -1256,6 +1330,11 @@ Error JITDylib::define(std::unique_ptr &&MU) { if (auto Err = defineImpl(*MU)) return Err; + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*this, *MU)) + return Err; + } + /// defineImpl succeeded. auto UMI = std::make_shared(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) @@ -1273,6 +1352,11 @@ Error JITDylib::define(std::unique_ptr &MU) { if (auto Err = defineImpl(*MU)) return Err; + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*this, *MU)) + return Err; + } + /// defineImpl succeeded. auto UMI = std::make_shared(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) @@ -1305,18 +1389,6 @@ private: SymbolPredicate Allow; }; -/// Mangles symbol names then uniques them in the context of an -/// ExecutionSession. -class MangleAndInterner { -public: - MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); - SymbolStringPtr operator()(StringRef Name); - -private: - ExecutionSession &ES; - const DataLayout &DL; -}; - } // End namespace orc } // End namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index f7255c5af84..49ab0d628b0 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -17,6 +17,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/Object/Archive.h" @@ -104,6 +105,53 @@ iterator_range getConstructors(const Module &M); /// array. iterator_range getDestructors(const Module &M); +/// This iterator provides a convenient way to iterate over GlobalValues that +/// have initialization effects. +class StaticInitGVIterator { +public: + StaticInitGVIterator() = default; + + StaticInitGVIterator(Module &M) + : I(M.global_values().begin()), E(M.global_values().end()), + ObjFmt(Triple(M.getTargetTriple()).getObjectFormat()) { + if (I != E) { + if (!isStaticInitGlobal(*I)) + moveToNextStaticInitGlobal(); + } else + I = E = Module::global_value_iterator(); + } + + bool operator==(const StaticInitGVIterator &O) const { return I == O.I; } + bool operator!=(const StaticInitGVIterator &O) const { return I != O.I; } + + StaticInitGVIterator &operator++() { + assert(I != E && "Increment past end of range"); + moveToNextStaticInitGlobal(); + return *this; + } + + GlobalValue &operator*() { return *I; } + +private: + bool isStaticInitGlobal(GlobalValue &GV); + void moveToNextStaticInitGlobal() { + ++I; + while (I != E && !isStaticInitGlobal(*I)) + ++I; + if (I == E) + I = E = Module::global_value_iterator(); + } + + Module::global_value_iterator I, E; + Triple::ObjectFormatType ObjFmt; +}; + +/// Create an iterator range over the GlobalValues that contribute to static +/// initialization. +inline iterator_range getStaticInitGVs(Module &M) { + return make_range(StaticInitGVIterator(M), StaticInitGVIterator()); +} + /// Convenience class for recording constructor/destructor names for /// later execution. template @@ -246,6 +294,22 @@ public: Error enable(JITDylib &JD, MangleAndInterner &Mangler); }; +/// An interface for Itanium __cxa_atexit interposer implementations. +class ItaniumCXAAtExitSupport { +public: + struct AtExitRecord { + void (*F)(void *); + void *Ctx; + }; + + void registerAtExit(void (*F)(void *), void *Ctx, void *DSOHandle); + void runAtExits(void *DSOHandle); + +private: + std::mutex AtExitsMutex; + DenseMap> AtExitRecords; +}; + /// A utility class to expose symbols found via dlsym to the JIT. /// /// If an instance of this class is attached to a JITDylib as a fallback diff --git a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h index bb8270fe80a..eb74d283f04 100644 --- a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -31,18 +31,18 @@ class IRCompileLayer : public IRLayer { public: class IRCompiler { public: - IRCompiler(IRMaterializationUnit::ManglingOptions MO) : MO(std::move(MO)) {} + IRCompiler(IRSymbolMapper::ManglingOptions MO) : MO(std::move(MO)) {} virtual ~IRCompiler(); - const IRMaterializationUnit::ManglingOptions &getManglingOptions() const { + const IRSymbolMapper::ManglingOptions &getManglingOptions() const { return MO; } virtual Expected> operator()(Module &M) = 0; protected: - IRMaterializationUnit::ManglingOptions &manglingOptions() { return MO; } + IRSymbolMapper::ManglingOptions &manglingOptions() { return MO; } private: - IRMaterializationUnit::ManglingOptions MO; + IRSymbolMapper::ManglingOptions MO; }; using NotifyCompiledFunction = @@ -61,7 +61,7 @@ private: mutable std::mutex IRLayerMutex; ObjectLayer &BaseLayer; std::unique_ptr Compile; - const IRMaterializationUnit::ManglingOptions *ManglingOpts; + const IRSymbolMapper::ManglingOptions *ManglingOpts; NotifyCompiledFunction NotifyCompiled = NotifyCompiledFunction(); }; diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index b71e5b33971..296d74ae6b8 100644 --- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -28,7 +28,7 @@ namespace orc { class IRTransformLayer : public IRLayer { public: using TransformFunction = std::function( - ThreadSafeModule, const MaterializationResponsibility &R)>; + ThreadSafeModule, MaterializationResponsibility &R)>; IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, TransformFunction Transform = identityTransform); @@ -39,9 +39,8 @@ public: void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; - static ThreadSafeModule - identityTransform(ThreadSafeModule TSM, - const MaterializationResponsibility &R) { + static ThreadSafeModule identityTransform(ThreadSafeModule TSM, + MaterializationResponsibility &R) { return TSM; } diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index a9ab3a630a6..880c8ff44c5 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -201,7 +201,7 @@ protected: ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) : TP(std::move(TP)), ES(ES), - CallbacksJD(ES.createJITDylib("")), + CallbacksJD(ES.createBareJITDylib("")), ErrorHandlerAddress(ErrorHandlerAddress) {} void setTrampolinePool(std::unique_ptr TP) { diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h index 3374a29f04d..c0b5012f801 100644 --- a/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -35,7 +35,23 @@ class LLLazyJITBuilderState; class LLJIT { template friend class LLJITBuilderSetters; + friend void setUpGenericLLVMIRPlatform(LLJIT &J); + public: + /// Initializer support for LLJIT. + class PlatformSupport { + public: + virtual ~PlatformSupport(); + + virtual Error initialize(JITDylib &JD) = 0; + + virtual Error deinitialize(JITDylib &JD) = 0; + + protected: + static void setInitTransform(LLJIT &J, + IRTransformLayer::TransformFunction T); + }; + static Expected> Create(LLJITBuilderState &S); /// Destruct this instance. If a multi-threaded instance, waits for all @@ -52,7 +68,7 @@ public: const DataLayout &getDataLayout() const { return DL; } /// Returns a reference to the JITDylib representing the JIT'd main program. - JITDylib &getMainJITDylib() { return Main; } + JITDylib &getMainJITDylib() { return *Main; } /// Returns the JITDylib with the given name, or nullptr if no JITDylib with /// that name exists. @@ -66,7 +82,7 @@ public: /// input or elsewhere in the environment then the client should check /// (e.g. by calling getJITDylibByName) that the given name is not already in /// use. - JITDylib &createJITDylib(std::string Name) { + Expected createJITDylib(std::string Name) { return ES->createJITDylib(std::move(Name)); } @@ -78,7 +94,7 @@ public: /// Adds an IR module to the Main JITDylib. Error addIRModule(ThreadSafeModule TSM) { - return addIRModule(Main, std::move(TSM)); + return addIRModule(*Main, std::move(TSM)); } /// Adds an object file to the given JITDylib. @@ -86,7 +102,7 @@ public: /// Adds an object file to the given JITDylib. Error addObjectFile(std::unique_ptr Obj) { - return addObjectFile(Main, std::move(Obj)); + return addObjectFile(*Main, std::move(Obj)); } /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to @@ -98,7 +114,7 @@ public: /// (to look up symbols based on their IR name use the lookup function /// instead). Expected lookupLinkerMangled(StringRef Name) { - return lookupLinkerMangled(Main, Name); + return lookupLinkerMangled(*Main, Name); } /// Look up a symbol in JITDylib JD based on its IR symbol name. @@ -108,14 +124,28 @@ public: /// Look up a symbol in the main JITDylib based on its IR symbol name. Expected lookup(StringRef UnmangledName) { - return lookup(Main, UnmangledName); + return lookup(*Main, UnmangledName); } - /// Runs all not-yet-run static constructors. - Error runConstructors() { return CtorRunner.run(); } + /// Set the PlatformSupport instance. + void setPlatformSupport(std::unique_ptr PS) { + this->PS = std::move(PS); + } - /// Runs all not-yet-run static destructors. - Error runDestructors() { return DtorRunner.run(); } + /// Get the PlatformSupport instance. + PlatformSupport *getPlatformSupport() { return PS.get(); } + + /// Run the initializers for the given JITDylib. + Error initialize(JITDylib &JD) { + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->initialize(JD); + } + + /// Run the deinitializers for the given JITDylib. + Error deinitialize(JITDylib &JD) { + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->deinitialize(JD); + } /// Returns a reference to the ObjLinkingLayer ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; } @@ -126,6 +156,9 @@ public: /// Returns a reference to the IR transform layer. IRTransformLayer &getIRTransformLayer() { return *TransformLayer; } + /// Returns a reference to the IR compile layer. + IRCompileLayer &getIRCompileLayer() { return *CompileLayer; } + protected: static std::unique_ptr createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES); @@ -143,7 +176,9 @@ protected: void recordCtorDtors(Module &M); std::unique_ptr ES; - JITDylib &Main; + std::unique_ptr PS; + + JITDylib *Main = nullptr; DataLayout DL; Triple TT; @@ -153,8 +188,7 @@ protected: ObjectTransformLayer ObjTransformLayer; std::unique_ptr CompileLayer; std::unique_ptr TransformLayer; - - CtorDtorRunner CtorRunner, DtorRunner; + std::unique_ptr InitHelperTransformLayer; }; /// An extended version of LLJIT that supports lazy function-at-a-time @@ -175,7 +209,7 @@ public: /// Add a module to be lazily compiled to the main JITDylib. Error addLazyIRModule(ThreadSafeModule M) { - return addLazyIRModule(Main, std::move(M)); + return addLazyIRModule(*Main, std::move(M)); } private: @@ -196,10 +230,14 @@ public: std::function>( JITTargetMachineBuilder JTMB)>; + using PlatformSetupFunction = std::function; + std::unique_ptr ES; Optional JTMB; + Optional DL; ObjectLinkingLayerCreator CreateObjectLinkingLayer; CompileFunctionCreator CreateCompileFunction; + PlatformSetupFunction SetUpPlatform; unsigned NumCompileThreads = 0; /// Called prior to JIT class construcion to fix up defaults. @@ -224,6 +262,13 @@ public: return impl().JTMB; } + /// Set a DataLayout for this instance. If no data layout is specified then + /// the target's default data layout will be used. + SetterImpl &setDataLayout(Optional DL) { + impl().DL = std::move(DL); + return impl(); + } + /// Set an ObjectLinkingLayer creation function. /// /// If this method is not called, a default creation function will be used @@ -246,6 +291,16 @@ public: return impl(); } + /// Set up an PlatformSetupFunction. + /// + /// If this method is not called then setUpGenericLLVMIRPlatform + /// will be used to configure the JIT's platform support. + SetterImpl & + setPlatformSetUp(LLJITBuilderState::PlatformSetupFunction SetUpPlatform) { + impl().SetUpPlatform = std::move(SetUpPlatform); + return impl(); + } + /// Set the number of compile threads to use. /// /// If set to zero, compilation will be performed on the execution thread when @@ -334,6 +389,26 @@ class LLLazyJITBuilder public LLLazyJITBuilderSetters {}; +/// Configure the LLJIT instance to scrape modules for llvm.global_ctors and +/// llvm.global_dtors variables and (if present) build initialization and +/// deinitialization functions. Platform specific initialization configurations +/// should be preferred where available. +void setUpGenericLLVMIRPlatform(LLJIT &J); + +/// Configure the LLJIT instance to use MachOPlatform support. +/// +/// Warning: MachOPlatform *requires* that LLJIT be configured to use +/// ObjectLinkingLayer (default on platforms supported by JITLink). If +/// MachOPlatform is used with RTDyldObjectLinkingLayer it will result in +/// undefined behavior). +/// +/// MachOPlatform installs an ObjectLinkingLayer plugin to scrape initializers +/// from the __mod_inits section. It also provides interposes for the dlfcn +/// functions (dlopen, dlclose, dlsym, dlerror) that work for JITDylibs as +/// well as regular libraries (JITDylibs will be preferenced, so make sure +/// your JITDylib names do not shadow any real library paths). +Error setUpMachOPlatform(LLJIT &J); + } // End namespace orc } // End namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/Layer.h b/include/llvm/ExecutionEngine/Orc/Layer.h index 95e32b2431a..e843d0f5624 100644 --- a/include/llvm/ExecutionEngine/Orc/Layer.h +++ b/include/llvm/ExecutionEngine/Orc/Layer.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_LAYER_H #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" #include "llvm/IR/Module.h" #include "llvm/Support/MemoryBuffer.h" @@ -27,15 +28,12 @@ namespace orc { /// their linkage is changed to available-externally. class IRMaterializationUnit : public MaterializationUnit { public: - struct ManglingOptions { - bool EmulatedTLS = false; - }; - using SymbolNameToDefinitionMap = std::map; /// Create an IRMaterializationLayer. Scans the module to build the /// SymbolFlags and SymbolToDefinition maps. - IRMaterializationUnit(ExecutionSession &ES, const ManglingOptions &MO, + IRMaterializationUnit(ExecutionSession &ES, + const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K); /// Create an IRMaterializationLayer from a module, and pre-existing @@ -44,12 +42,13 @@ public: /// This constructor is useful for delegating work from one /// IRMaterializationUnit to another. IRMaterializationUnit(ThreadSafeModule TSM, VModuleKey K, - SymbolFlagsMap SymbolFlags, + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition); /// Return the ModuleIdentifier as the name for this MaterializationUnit. StringRef getName() const override; + /// Return a reference to the contained ThreadSafeModule. const ThreadSafeModule &getModule() const { return TSM; } protected: @@ -57,14 +56,16 @@ protected: SymbolNameToDefinitionMap SymbolToDefinition; private: + static SymbolStringPtr getInitSymbol(ExecutionSession &ES, + const ThreadSafeModule &TSM); + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; }; /// Interface for layers that accept LLVM IR. class IRLayer { public: - IRLayer(ExecutionSession &ES, - const IRMaterializationUnit::ManglingOptions *&MO) + IRLayer(ExecutionSession &ES, const IRSymbolMapper::ManglingOptions *&MO) : ES(ES), MO(MO) {} virtual ~IRLayer(); @@ -73,7 +74,7 @@ public: ExecutionSession &getExecutionSession() { return ES; } /// Get the mangling options for this layer. - const IRMaterializationUnit::ManglingOptions *&getManglingOptions() const { + const IRSymbolMapper::ManglingOptions *&getManglingOptions() const { return MO; } @@ -104,14 +105,15 @@ public: private: bool CloneToNewContextOnEmit = false; ExecutionSession &ES; - const IRMaterializationUnit::ManglingOptions *&MO; + const IRSymbolMapper::ManglingOptions *&MO; }; /// MaterializationUnit that materializes modules by calling the 'emit' method /// on the given IRLayer. class BasicIRLayerMaterializationUnit : public IRMaterializationUnit { public: - BasicIRLayerMaterializationUnit(IRLayer &L, const ManglingOptions &MO, + BasicIRLayerMaterializationUnit(IRLayer &L, + const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K); private: @@ -153,7 +155,8 @@ public: BasicObjectLayerMaterializationUnit(ObjectLayer &L, VModuleKey K, std::unique_ptr O, - SymbolFlagsMap SymbolFlags); + SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol); /// Return the buffer's identifier as the name for this MaterializationUnit. StringRef getName() const override; @@ -167,12 +170,6 @@ private: std::unique_ptr O; }; -/// Returns a SymbolFlagsMap for the object file represented by the given -/// buffer, or an error if the buffer does not contain a valid object file. -// FIXME: Maybe move to Core.h? -Expected getObjectSymbolFlags(ExecutionSession &ES, - MemoryBufferRef ObjBuffer); - } // End namespace orc } // End namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/include/llvm/ExecutionEngine/Orc/MachOPlatform.h new file mode 100644 index 00000000000..92c1a04921b --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -0,0 +1,147 @@ +//===-- MachOPlatform.h - Utilities for executing MachO in Orc --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for executing JIT'd MachO in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H +#define LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include +#include +#include + +namespace llvm { +namespace orc { + +/// Enable registration of JIT'd ObjC classes and selectors. +Error enableObjCRegistration(const char *PathToLibObjC); +bool objCRegistrationEnabled(); + +class MachOJITDylibInitializers { +public: + struct SectionExtent { + SectionExtent() = default; + SectionExtent(JITTargetAddress Address, uint64_t NumPtrs) + : Address(Address), NumPtrs(NumPtrs) {} + JITTargetAddress Address = 0; + uint64_t NumPtrs = 0; + }; + + void setObjCImageInfoAddr(JITTargetAddress ObjCImageInfoAddr) { + this->ObjCImageInfoAddr = ObjCImageInfoAddr; + } + + void addModInitsSection(SectionExtent ModInit) { + ModInitSections.push_back(std::move(ModInit)); + } + + void addObjCSelRefsSection(SectionExtent ObjCSelRefs) { + ObjCSelRefsSections.push_back(std::move(ObjCSelRefs)); + } + + void addObjCClassListSection(SectionExtent ObjCClassList) { + ObjCClassListSections.push_back(std::move(ObjCClassList)); + } + + void runModInits() const; + void registerObjCSelectors() const; + Error registerObjCClasses() const; + + void dump() const; + +private: + using RawPointerSectionList = std::vector; + + JITTargetAddress ObjCImageInfoAddr; + RawPointerSectionList ModInitSections; + RawPointerSectionList ObjCSelRefsSections; + RawPointerSectionList ObjCClassListSections; +}; + +class MachOJITDylibDeinitializers {}; + +/// Mediates between MachO initialization and ExecutionSession state. +class MachOPlatform : public Platform { +public: + using InitializerSequence = + std::vector>; + + using DeinitializerSequence = + std::vector>; + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr StandardSymbolsObject); + + ExecutionSession &getExecutionSession() const { return ES; } + + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; + Error notifyRemoving(JITDylib &JD, VModuleKey K) override; + + Expected getInitializerSequence(JITDylib &JD); + + Expected getDeinitializerSequence(JITDylib &JD); + +private: + // This ObjectLinkingLayer plugin scans JITLink graphs for __mod_init_func, + // __objc_classlist and __sel_ref sections and records their extents so that + // they can be run in the target process. + class InitScraperPlugin : public ObjectLinkingLayer::Plugin { + public: + InitScraperPlugin(MachOPlatform &MP) : MP(MP) {} + + void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) override; + + LocalDependenciesMap getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) override; + + private: + using InitSymbolDepMap = + DenseMap; + + void preserveInitSectionIfPresent(JITLinkSymbolVector &Syms, + jitlink::LinkGraph &G, + StringRef SectionName); + + Error processObjCImageInfo(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + std::mutex InitScraperMutex; + MachOPlatform &MP; + DenseMap> ObjCImageInfos; + InitSymbolDepMap InitSymbolDeps; + }; + + static std::vector getDFSLinkOrder(JITDylib &JD); + + void registerInitInfo(JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList); + + std::mutex PlatformMutex; + ExecutionSession &ES; + ObjectLinkingLayer &ObjLinkingLayer; + std::unique_ptr StandardSymbolsObject; + + DenseMap RegisteredInitSymbols; + DenseMap InitSeqs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H diff --git a/include/llvm/ExecutionEngine/Orc/Mangling.h b/include/llvm/ExecutionEngine/Orc/Mangling.h new file mode 100644 index 00000000000..e0f770a601f --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Mangling.h @@ -0,0 +1,66 @@ +//===------ Mangling.h -- Name Mangling Utilities for ORC -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Name mangling utilities for ORC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MANGLING_H +#define LLVM_EXECUTIONENGINE_ORC_MANGLING_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +/// Mangles symbol names then uniques them in the context of an +/// ExecutionSession. +class MangleAndInterner { +public: + MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); + SymbolStringPtr operator()(StringRef Name); + +private: + ExecutionSession &ES; + const DataLayout &DL; +}; + +/// Maps IR global values to their linker symbol names / flags. +/// +/// This utility can be used when adding new IR globals in the JIT. +class IRSymbolMapper { +public: + struct ManglingOptions { + bool EmulatedTLS = false; + }; + + using SymbolNameToDefinitionMap = std::map; + + /// Add mangled symbols for the given GlobalValues to SymbolFlags. + /// If a SymbolToDefinitionMap pointer is supplied then it will be populated + /// with Name-to-GlobalValue* mappings. Note that this mapping is not + /// necessarily one-to-one: thread-local GlobalValues, for example, may + /// produce more than one symbol, in which case the map will contain duplicate + /// values. + static void add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef GVs, SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition = nullptr); +}; + +/// Returns a SymbolFlagsMap for the object file represented by the given +/// buffer, or an error if the buffer does not contain a valid object file. +Expected> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MANGLING_H diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index 50d25f18891..1b4c03bd2cf 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -35,6 +35,7 @@ namespace llvm { namespace jitlink { class EHFrameRegistrar; +class Symbol; } // namespace jitlink namespace object { @@ -59,10 +60,14 @@ public: /// configured. class Plugin { public: + using JITLinkSymbolVector = std::vector; + using LocalDependenciesMap = DenseMap; + virtual ~Plugin(); virtual void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, jitlink::PassConfiguration &Config) {} + virtual void notifyLoaded(MaterializationResponsibility &MR) {} virtual Error notifyEmitted(MaterializationResponsibility &MR) { return Error::success(); @@ -71,6 +76,15 @@ public: return Error::success(); } virtual Error notifyRemovingAllModules() { return Error::success(); } + + /// Return any dependencies that synthetic symbols (e.g. init symbols) + /// have on locally scoped jitlink::Symbols. This is used by the + /// ObjectLinkingLayer to update the dependencies for the synthetic + /// symbols. + virtual LocalDependenciesMap + getSyntheticSymbolLocalDependencies(MaterializationResponsibility &MR) { + return LocalDependenciesMap(); + } }; using ReturnObjectBufferFunction = diff --git a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h index c354f6c3559..c8462dc3d2c 100644 --- a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -53,6 +53,7 @@ class SymbolStringPtr { public: SymbolStringPtr() = default; + SymbolStringPtr(nullptr_t) {} SymbolStringPtr(const SymbolStringPtr &Other) : S(Other.S) { if (isRealPoolEntry(S)) @@ -85,6 +86,8 @@ public: --S->getValue(); } + explicit operator bool() const { return S; } + StringRef operator*() const { return S->first(); } friend bool operator==(const SymbolStringPtr &LHS, diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt index e3a7a67c8a0..7eb2742d7b5 100644 --- a/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -12,6 +12,8 @@ add_llvm_component_library(LLVMOrcJIT Legacy.cpp Layer.cpp LLJIT.cpp + MachOPlatform.cpp + Mangling.cpp NullResolver.cpp ObjectLinkingLayer.cpp ObjectTransformLayer.cpp diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 29d18b6e4d7..f22ae01a208 100644 --- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" @@ -68,18 +69,18 @@ namespace orc { class PartitioningIRMaterializationUnit : public IRMaterializationUnit { public: PartitioningIRMaterializationUnit(ExecutionSession &ES, - const ManglingOptions &MO, + const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K, CompileOnDemandLayer &Parent) : IRMaterializationUnit(ES, MO, std::move(TSM), std::move(K)), Parent(Parent) {} PartitioningIRMaterializationUnit( - ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition, + ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition, CompileOnDemandLayer &Parent) : IRMaterializationUnit(std::move(TSM), std::move(K), - std::move(SymbolFlags), + std::move(SymbolFlags), std::move(InitSymbol), std::move(SymbolToDefinition)), Parent(Parent) {} @@ -172,21 +173,23 @@ CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { auto I = DylibResources.find(&TargetD); if (I == DylibResources.end()) { auto &ImplD = - getExecutionSession().createJITDylib(TargetD.getName() + ".impl"); + getExecutionSession().createBareJITDylib(TargetD.getName() + ".impl"); + JITDylibSearchOrder NewSearchOrder; TargetD.withSearchOrderDo( [&](const JITDylibSearchOrder &TargetSearchOrder) { - auto NewSearchOrder = TargetSearchOrder; - assert( - !NewSearchOrder.empty() && - NewSearchOrder.front().first == &TargetD && - NewSearchOrder.front().second == - JITDylibLookupFlags::MatchAllSymbols && - "TargetD must be at the front of its own search order and match " - "non-exported symbol"); - NewSearchOrder.insert(std::next(NewSearchOrder.begin()), - {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); - ImplD.setSearchOrder(std::move(NewSearchOrder), false); + NewSearchOrder = TargetSearchOrder; }); + + assert( + !NewSearchOrder.empty() && NewSearchOrder.front().first == &TargetD && + NewSearchOrder.front().second == JITDylibLookupFlags::MatchAllSymbols && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewSearchOrder.insert(std::next(NewSearchOrder.begin()), + {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); + ImplD.setSearchOrder(NewSearchOrder, false); + TargetD.setSearchOrder(std::move(NewSearchOrder), false); + PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; } @@ -251,8 +254,15 @@ void CompileOnDemandLayer::emitPartition( auto &ES = getExecutionSession(); GlobalValueSet RequestedGVs; for (auto &Name : R.getRequestedSymbols()) { - assert(Defs.count(Name) && "No definition for symbol"); - RequestedGVs.insert(Defs[Name]); + if (Name == R.getInitializerSymbol()) + TSM.withModuleDo([&](Module &M) { + for (auto &GV : getStaticInitGVs(M)) + RequestedGVs.insert(&GV); + }); + else { + assert(Defs.count(Name) && "No definition for symbol"); + RequestedGVs.insert(Defs[Name]); + } } /// Perform partitioning with the context lock held, since the partition @@ -272,7 +282,8 @@ void CompileOnDemandLayer::emitPartition( // If the partition is empty, return the whole module to the symbol table. if (GVsToExtract->empty()) { R.replace(std::make_unique( - std::move(TSM), R.getSymbols(), std::move(Defs), *this)); + std::move(TSM), R.getVModuleKey(), R.getSymbols(), + R.getInitializerSymbol(), std::move(Defs), *this)); return; } diff --git a/lib/ExecutionEngine/Orc/CompileUtils.cpp b/lib/ExecutionEngine/Orc/CompileUtils.cpp index 160e5ba5031..f8efed15ede 100644 --- a/lib/ExecutionEngine/Orc/CompileUtils.cpp +++ b/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -24,9 +24,9 @@ namespace llvm { namespace orc { -IRMaterializationUnit::ManglingOptions +IRSymbolMapper::ManglingOptions irManglingOptionsFromTargetOptions(const TargetOptions &Opts) { - IRMaterializationUnit::ManglingOptions MO; + IRSymbolMapper::ManglingOptions MO; MO.EmulatedTLS = Opts.EmulatedTLS; diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index a7af34a2192..6a0df10f7c3 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -11,7 +11,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" -#include "llvm/IR/Mangler.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" @@ -421,12 +420,6 @@ void AsynchronousSymbolQuery::detach() { QueryRegistrations.clear(); } -MaterializationResponsibility::MaterializationResponsibility( - JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) - : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { - assert(!this->SymbolFlags.empty() && "Materializing nothing?"); -} - MaterializationResponsibility::~MaterializationResponsibility() { assert(SymbolFlags.empty() && "All symbols should have been explicitly materialized or failed"); @@ -501,9 +494,13 @@ void MaterializationResponsibility::failMaterialization() { void MaterializationResponsibility::replace( std::unique_ptr MU) { + for (auto &KV : MU->getSymbols()) SymbolFlags.erase(KV.first); + if (MU->getInitializerSymbol() == InitSymbol) + InitSymbol = nullptr; + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { dbgs() << "In " << JD.getName() << " replacing symbols with " << *MU << "\n"; @@ -519,6 +516,7 @@ MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, if (NewKey == VModuleKey()) NewKey = K; + SymbolStringPtr DelegatedInitSymbol; SymbolFlagsMap DelegatedFlags; for (auto &Name : Symbols) { @@ -528,10 +526,14 @@ MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, "instance"); DelegatedFlags[Name] = std::move(I->second); + if (Name == InitSymbol) + std::swap(InitSymbol, DelegatedInitSymbol); + SymbolFlags.erase(I); } return MaterializationResponsibility(JD, std::move(DelegatedFlags), + std::move(DelegatedInitSymbol), std::move(NewKey)); } @@ -550,7 +552,7 @@ void MaterializationResponsibility::addDependenciesForAll( AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( SymbolMap Symbols, VModuleKey K) - : MaterializationUnit(extractFlags(Symbols), std::move(K)), + : MaterializationUnit(extractFlags(Symbols), nullptr, std::move(K)), Symbols(std::move(Symbols)) {} StringRef AbsoluteSymbolsMaterializationUnit::getName() const { @@ -581,7 +583,7 @@ AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { ReExportsMaterializationUnit::ReExportsMaterializationUnit( JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases, VModuleKey K) - : MaterializationUnit(extractFlags(Aliases), std::move(K)), + : MaterializationUnit(extractFlags(Aliases), nullptr, std::move(K)), SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} @@ -972,7 +974,7 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, // Assert that this symbol exists and has not reached the ready state // already. assert(OtherSymI != OtherJITDylib.Symbols.end() && - (OtherSymI->second.getState() != SymbolState::Ready && + (OtherSymI->second.getState() < SymbolState::Ready && "Dependency on emitted/ready symbol")); #endif @@ -1101,6 +1103,7 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { Error JITDylib::emit(const SymbolFlagsMap &Emitted) { AsynchronousSymbolQuerySet CompletedQueries; SymbolNameSet SymbolsInErrorState; + DenseMap ReadySymbols; ES.runSessionLocked([&, this]() { std::vector Worklist; @@ -1145,6 +1148,7 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { // dependencies) then notify any pending queries. for (auto &KV : MI.Dependants) { auto &DependantJD = *KV.first; + auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; for (auto &DependantName : KV.second) { auto DependantMII = DependantJD.MaterializingInfos.find(DependantName); @@ -1184,6 +1188,7 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. DependantSymEntry.setState(SymbolState::Ready); + DependantJDReadySymbols.push_back(DependantName); for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState( @@ -1192,22 +1197,21 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { CompletedQueries.insert(Q); Q->removeQueryDependence(DependantJD, DependantName); } - - DependantJD.MaterializingInfos.erase(DependantMII); } } } + auto &ThisJDReadySymbols = ReadySymbols[this]; MI.Dependants.clear(); if (MI.UnemittedDependencies.empty()) { SymI->second.setState(SymbolState::Ready); + ThisJDReadySymbols.push_back(Name); for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); if (Q->isComplete()) CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } - MaterializingInfos.erase(MII); } } }); @@ -1882,6 +1886,57 @@ void JITDylib::transferEmittedNodeDependencies( } } +Platform::~Platform() {} + +Expected> Platform::lookupInitSymbols( + ExecutionSession &ES, + const DenseMap &InitSyms) { + + DenseMap CompoundResult; + Error CompoundErr = Error::success(); + std::mutex LookupMutex; + std::condition_variable CV; + uint64_t Count = InitSyms.size(); + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [&, JD](Expected Result) { + { + std::lock_guard Lock(LookupMutex); + --Count; + if (Result) { + assert(!CompoundResult.count(JD) && + "Duplicate JITDylib in lookup?"); + CompoundResult[JD] = std::move(*Result); + } else + CompoundErr = + joinErrors(std::move(CompoundErr), Result.takeError()); + } + CV.notify_one(); + }, + NoDependenciesToRegister); + } + + std::unique_lock Lock(LookupMutex); + CV.wait(Lock, [&] { return Count == 0 || CompoundErr; }); + + if (CompoundErr) + return std::move(CompoundErr); + + return std::move(CompoundResult); +} + ExecutionSession::ExecutionSession(std::shared_ptr SSP) : SSP(SSP ? std::move(SSP) : std::make_shared()) { } @@ -1895,7 +1950,7 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { }); } -JITDylib &ExecutionSession::createJITDylib(std::string Name) { +JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( @@ -1904,6 +1959,14 @@ JITDylib &ExecutionSession::createJITDylib(std::string Name) { }); } +Expected ExecutionSession::createJITDylib(std::string Name) { + auto &JD = createBareJITDylib(Name); + if (P) + if (auto Err = P->setupJITDylib(JD)) + return std::move(Err); + return JD; +} + void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { assert(!!Err && "Error should be in failure state"); @@ -2144,11 +2207,11 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, Expected ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, - SymbolStringPtr Name) { + SymbolStringPtr Name, SymbolState RequiredState) { SymbolLookupSet Names({Name}); if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static, - SymbolState::Ready, NoDependenciesToRegister)) { + RequiredState, NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); @@ -2157,14 +2220,15 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, } Expected -ExecutionSession::lookup(ArrayRef SearchOrder, - SymbolStringPtr Name) { - return lookup(makeJITDylibSearchOrder(SearchOrder), Name); +ExecutionSession::lookup(ArrayRef SearchOrder, SymbolStringPtr Name, + SymbolState RequiredState) { + return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); } Expected -ExecutionSession::lookup(ArrayRef SearchOrder, StringRef Name) { - return lookup(SearchOrder, intern(Name)); +ExecutionSession::lookup(ArrayRef SearchOrder, StringRef Name, + SymbolState RequiredState) { + return lookup(SearchOrder, intern(Name), RequiredState); } void ExecutionSession::dump(raw_ostream &OS) { @@ -2195,17 +2259,5 @@ void ExecutionSession::runOutstandingMUs() { } } -MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) - : ES(ES), DL(DL) {} - -SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return ES.intern(MangledName); -} - } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 3d97fe9eeab..a98445a2295 100644 --- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -113,6 +113,26 @@ iterator_range getDestructors(const Module &M) { CtorDtorIterator(DtorsList, true)); } +bool StaticInitGVIterator::isStaticInitGlobal(GlobalValue &GV) { + if (GV.isDeclaration()) + return false; + + if (GV.hasName() && (GV.getName() == "llvm.global_ctors" || + GV.getName() == "llvm.global_dtors")) + return true; + + if (ObjFmt == Triple::MachO) { + // FIXME: These section checks are too strict: We should match first and + // second word split by comma. + if (GV.hasSection() && + (GV.getSection().startswith("__DATA,__objc_classlist") || + GV.getSection().startswith("__DATA,__objc_selrefs"))) + return true; + } + + return false; +} + void CtorDtorRunner::add(iterator_range CtorDtors) { if (CtorDtors.empty()) return; @@ -198,6 +218,30 @@ Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); } +void ItaniumCXAAtExitSupport::registerAtExit(void (*F)(void *), void *Ctx, + void *DSOHandle) { + std::lock_guard Lock(AtExitsMutex); + AtExitRecords[DSOHandle].push_back({F, Ctx}); +} + +void ItaniumCXAAtExitSupport::runAtExits(void *DSOHandle) { + std::vector AtExitsToRun; + + { + std::lock_guard Lock(AtExitsMutex); + auto I = AtExitRecords.find(DSOHandle); + if (I != AtExitRecords.end()) { + AtExitsToRun = std::move(I->second); + AtExitRecords.erase(I); + } + } + + while (!AtExitsToRun.empty()) { + AtExitsToRun.back().F(AtExitsToRun.back().Ctx); + AtExitsToRun.pop_back(); + } +} + DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 1ac9a58aeae..1e991d8f58f 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -28,7 +28,7 @@ public: CompileCallbackMaterializationUnit(SymbolStringPtr Name, CompileFunction Compile, VModuleKey K) : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), - std::move(K)), + nullptr, std::move(K)), Name(std::move(Name)), Compile(std::move(Compile)) {} StringRef getName() const override { return ""; } diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp index 4218ca4e481..5b2f8682335 100644 --- a/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -8,15 +8,827 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/DynamicLibrary.h" + +#include + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +/// Add a reference to the __dso_handle global to the given module. +/// Returns a reference to the __dso_handle IR decl. +GlobalVariable *addDSOHandleDecl(Module &M) { + auto DSOHandleTy = StructType::create(M.getContext(), "lljit.dso_handle"); + return new GlobalVariable(M, DSOHandleTy, true, GlobalValue::ExternalLinkage, + nullptr, "__dso_handle"); +} + +/// Adds helper function decls and wrapper functions that call the helper with +/// some additional prefix arguments. +/// +/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix +/// args i32 4 and i16 12345, this function will add: +/// +/// declare i8 @bar(i32, i16, i8, i64) +/// +/// define i8 @foo(i8, i64) { +/// entry: +/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1) +/// ret i8 %2 +/// } +/// +Function *addHelperAndWrapper(Module &M, StringRef WrapperName, + FunctionType *WrapperFnType, + GlobalValue::VisibilityTypes WrapperVisibility, + StringRef HelperName, + ArrayRef HelperPrefixArgs) { + std::vector HelperArgTypes; + for (auto *Arg : HelperPrefixArgs) + HelperArgTypes.push_back(Arg->getType()); + for (auto *T : WrapperFnType->params()) + HelperArgTypes.push_back(T); + auto *HelperFnType = + FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false); + auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage, + HelperName, M); + + auto *WrapperFn = Function::Create( + WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M); + WrapperFn->setVisibility(WrapperVisibility); + + auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn); + IRBuilder<> IB(EntryBlock); + + std::vector HelperArgs; + for (auto *Arg : HelperPrefixArgs) + HelperArgs.push_back(Arg); + for (auto &Arg : WrapperFn->args()) + HelperArgs.push_back(&Arg); + auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs); + if (HelperFn->getReturnType()->isVoidTy()) + IB.CreateRetVoid(); + else + IB.CreateRet(HelperResult); + + return WrapperFn; +} + +class GenericLLVMIRPlatformSupport; + +/// orc::Platform component of Generic LLVM IR Platform support. +/// Just forwards calls to the GenericLLVMIRPlatformSupport class below. +class GenericLLVMIRPlatform : public Platform { +public: + GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {} + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; + Error notifyRemoving(JITDylib &JD, VModuleKey K) override { + // Noop -- Nothing to do (yet). + return Error::success(); + } + +private: + GenericLLVMIRPlatformSupport &S; +}; + +/// This transform parses llvm.global_ctors to produce a single initialization +/// function for the module, records the function, then deletes +/// llvm.global_ctors. +class GlobalCtorDtorScraper { +public: + GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS) : PS(PS) {} + Expected operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R); + +private: + GenericLLVMIRPlatformSupport &PS; +}; + +/// Generic IR Platform Support +/// +/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with +/// specially named 'init' and 'deinit'. Injects definitions / interposes for +/// some runtime API, including __cxa_atexit, dlopen, and dlclose. +class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { +public: + // GenericLLVMIRPlatform &P) : P(P) { + GenericLLVMIRPlatformSupport(LLJIT &J) : J(J) { + + getExecutionSession().setPlatform( + std::make_unique(*this)); + + setInitTransform(J, GlobalCtorDtorScraper(*this)); + + MangleAndInterner Mangle(getExecutionSession(), J.getDataLayout()); + SymbolMap StdInterposes; + + StdInterposes[Mangle("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); + StdInterposes[Mangle("__lljit.cxa_atexit_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(registerAtExitHelper), JITSymbolFlags()); + StdInterposes[Mangle("__lljit.run_atexits_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(runAtExitsHelper), JITSymbolFlags()); + + cantFail( + J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); + cantFail(setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); + } + + ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } + + /// Adds a module that defines the __dso_handle global. + Error setupJITDylib(JITDylib &JD) { + auto Ctx = std::make_unique(); + auto M = std::make_unique("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(*Ctx); + auto *DSOHandle = new GlobalVariable( + *M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, reinterpret_cast(&JD)), + "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::HiddenVisibility); + DSOHandle->setInitializer( + ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); + return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx))); + } + + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + std::lock_guard Lock(PlatformSupportMutex); + if (auto &InitSym = MU.getInitializerSymbol()) + InitSymbols[&JD].add(InitSym); + return Error::success(); + } + + Error initialize(JITDylib &JD) override { + if (auto Initializers = getInitializers(JD)) { + for (auto InitFnAddr : *Initializers) { + auto *InitFn = jitTargetAddressToFunction(InitFnAddr); + InitFn(); + } + } else + return Initializers.takeError(); + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + if (auto Deinitializers = getDeinitializers(JD)) { + for (auto DeinitFnAddr : *Deinitializers) { + auto *DeinitFn = jitTargetAddressToFunction(DeinitFnAddr); + DeinitFn(); + } + } else + return Deinitializers.takeError(); + + return Error::success(); + } + + void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) { + std::lock_guard Lock(PlatformSupportMutex); + InitFunctions[&JD].add(InitName); + } + +private: + Expected> getInitializers(JITDylib &JD) { + if (auto Err = issueInitLookups(JD)) + return std::move(Err); + + DenseMap LookupSymbols; + std::vector DFSLinkOrder; + + { + std::lock_guard Lock(PlatformSupportMutex); + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto IFItr = InitFunctions.find(NextJD); + if (IFItr != InitFunctions.end()) { + LookupSymbols[NextJD] = std::move(IFItr->second); + InitFunctions.erase(IFItr); + } + } + } + + auto &ES = getExecutionSession(); + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector Initializers; + while (!DFSLinkOrder.empty()) { + auto &NextJD = *DFSLinkOrder.back(); + DFSLinkOrder.pop_back(); + auto InitsItr = LookupResult->find(&NextJD); + if (InitsItr == LookupResult->end()) + continue; + for (auto &KV : InitsItr->second) + Initializers.push_back(KV.second.getAddress()); + } + + return Initializers; + } + + Expected> getDeinitializers(JITDylib &JD) { + auto &ES = getExecutionSession(); + + MangleAndInterner Mangle(getExecutionSession(), J.getDataLayout()); + auto LLJITRunAtExits = Mangle("__lljit_run_atexits"); + + DenseMap LookupSymbols; + std::vector DFSLinkOrder; + + { + std::lock_guard Lock(PlatformSupportMutex); + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto &JDLookupSymbols = LookupSymbols[NextJD]; + auto DIFItr = DeInitFunctions.find(NextJD); + if (DIFItr != DeInitFunctions.end()) { + LookupSymbols[NextJD] = std::move(DIFItr->second); + DeInitFunctions.erase(DIFItr); + } + JDLookupSymbols.add(LLJITRunAtExits, + SymbolLookupFlags::WeaklyReferencedSymbol); + } + } + + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector DeInitializers; + for (auto *NextJD : DFSLinkOrder) { + auto DeInitsItr = LookupResult->find(NextJD); + assert(DeInitsItr != LookupResult->end() && + "Every JD should have at least __lljit_run_atexits"); + + auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits); + if (RunAtExitsItr != DeInitsItr->second.end()) + DeInitializers.push_back(RunAtExitsItr->second.getAddress()); + + for (auto &KV : DeInitsItr->second) + if (KV.first != LLJITRunAtExits) + DeInitializers.push_back(KV.second.getAddress()); + } + + return DeInitializers; + } + + // Returns a DFS traversal order of the JITDylibs reachable (via + // links-against edges) from JD, starting with JD itself. + static std::vector getDFSLinkOrder(JITDylib &JD) { + std::vector DFSLinkOrder; + std::vector WorkStack({&JD}); + DenseSet Visited; + + while (!WorkStack.empty()) { + auto &NextJD = *WorkStack.back(); + WorkStack.pop_back(); + if (Visited.count(&NextJD)) + continue; + Visited.insert(&NextJD); + DFSLinkOrder.push_back(&NextJD); + NextJD.withSearchOrderDo([&](const JITDylibSearchOrder &SearchOrder) { + for (auto &KV : SearchOrder) + WorkStack.push_back(KV.first); + }); + } + + return DFSLinkOrder; + } + + /// Issue lookups for all init symbols required to initialize JD (and any + /// JITDylibs that it depends on). + Error issueInitLookups(JITDylib &JD) { + DenseMap RequiredInitSymbols; + + { + std::lock_guard Lock(PlatformSupportMutex); + + auto DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto ISItr = InitSymbols.find(NextJD); + if (ISItr != InitSymbols.end()) { + RequiredInitSymbols[NextJD] = std::move(ISItr->second); + InitSymbols.erase(ISItr); + } + } + } + + return Platform::lookupInitSymbols(getExecutionSession(), + RequiredInitSymbols) + .takeError(); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + static_cast(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + static_cast(Self)->AtExitMgr.runAtExits( + DSOHandle); + } + + // Constructs an LLVM IR module containing platform runtime globals, + // functions, and interposes. + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique(); + auto M = std::make_unique("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *GenericIRPlatformSupportTy = + StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, + nullptr, "__lljit.platform_support_instance"); + + auto *DSOHandleDecl = addDSOHandleDecl(*M); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::HiddenVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper( + *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false), + GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper", + {PlatformInstanceDecl, DSOHandleDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + std::mutex PlatformSupportMutex; + LLJIT &J; + DenseMap InitSymbols; + DenseMap InitFunctions; + DenseMap DeInitFunctions; + ItaniumCXAAtExitSupport AtExitMgr; +}; + +Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) { + return S.setupJITDylib(JD); +} + +Error GenericLLVMIRPlatform::notifyAdding(JITDylib &JD, + const MaterializationUnit &MU) { + return S.notifyAdding(JD, MU); +} + +Expected +GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R) { + auto Err = TSM.withModuleDo([&](Module &M) -> Error { + auto &Ctx = M.getContext(); + auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors"); + + // If there's no llvm.global_ctors or it's just a decl then skip. + if (!GlobalCtors || GlobalCtors->isDeclaration()) + return Error::success(); + + std::string InitFunctionName; + raw_string_ostream(InitFunctionName) + << "__orc_init." << M.getModuleIdentifier(); + + MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); + auto InternedName = Mangle(InitFunctionName); + if (auto Err = + R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}})) + return Err; + + auto *InitFunc = + Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false), + GlobalValue::ExternalLinkage, InitFunctionName, &M); + InitFunc->setVisibility(GlobalValue::HiddenVisibility); + std::vector> Inits; + for (auto E : getConstructors(M)) + Inits.push_back(std::make_pair(E.Func, E.Priority)); + llvm::sort(Inits, [](const std::pair &LHS, + const std::pair &RHS) { + return LHS.first < RHS.first; + }); + auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc); + IRBuilder<> IB(EntryBlock); + for (auto &KV : Inits) + IB.CreateCall(KV.first); + IB.CreateRetVoid(); + + PS.registerInitFunc(R.getTargetJITDylib(), InternedName); + GlobalCtors->eraseFromParent(); + return Error::success(); + }); + + if (Err) + return std::move(Err); + + return TSM; +} + +class MachOPlatformSupport : public LLJIT::PlatformSupport { +public: + using DLOpenType = void *(*)(const char *Name, int Mode); + using DLCloseType = int (*)(void *Handle); + using DLSymType = void *(*)(void *Handle, const char *Name); + using DLErrorType = const char *(*)(); + + struct DlFcnValues { + Optional RTLDDefault; + DLOpenType dlopen = nullptr; + DLCloseType dlclose = nullptr; + DLSymType dlsym = nullptr; + DLErrorType dlerror = nullptr; + }; + + static Expected> + Create(LLJIT &J, JITDylib &PlatformJITDylib) { + + // Make process symbols visible. + { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); + if (!Lib.isValid()) + return make_error(std::move(ErrMsg), + inconvertibleErrorCode()); + } + + DlFcnValues DlFcn; + + // Add support for RTLDDefault on known platforms. +#ifdef __APPLE__ + DlFcn.RTLDDefault = reinterpret_cast(-2); +#endif // __APPLE__ + + if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror")) + return std::move(Err); + + std::unique_ptr MP( + new MachOPlatformSupport(J, PlatformJITDylib, DlFcn)); + return std::move(MP); + } + + Error initialize(JITDylib &JD) override { + if (auto InitSeq = MP.getInitializerSequence(JD)) { + for (auto &KV : *InitSeq) { + KV.second.registerObjCSelectors(); + if (auto Err = KV.second.registerObjCClasses()) { + // FIXME: Roll back registrations on error? + return Err; + } + } + for (auto &KV : *InitSeq) + KV.second.runModInits(); + } else + return InitSeq.takeError(); + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + auto &ES = J.getExecutionSession(); + if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) { + for (auto &KV : *DeinitSeq) { + auto DSOHandleName = ES.intern("___dso_handle"); + + // FIXME: Run DeInits here. + auto Result = ES.lookup( + {{KV.first, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(DSOHandleName, + SymbolLookupFlags::WeaklyReferencedSymbol)); + if (!Result) + return Result.takeError(); + if (Result->empty()) + continue; + assert(Result->count(DSOHandleName) && + "Result does not contain __dso_handle"); + auto *DSOHandle = jitTargetAddressToPointer( + Result->begin()->second.getAddress()); + AtExitMgr.runAtExits(DSOHandle); + } + } else + return DeinitSeq.takeError(); + return Error::success(); + } + +private: + template + static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) { + if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) { + Fn = reinterpret_cast(Fn); + return Error::success(); + } + + return make_error((Twine("Can not enable MachO JIT Platform: " + "missing function: ") + + Name) + .str(), + inconvertibleErrorCode()); + } + + MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn) + : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) { + + MangleAndInterner Mangle(J.getExecutionSession(), J.getDataLayout()); + SymbolMap HelperSymbols; + + // platform and atexit helpers. + HelperSymbols[Mangle("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.cxa_atexit_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(registerAtExitHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.run_atexits_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(runAtExitsHelper), JITSymbolFlags()); + + // dlfcn helpers. + HelperSymbols[Mangle("__lljit.dlopen_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlopenHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.dlclose_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlcloseHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.dlsym_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlsymHelper), JITSymbolFlags()); + HelperSymbols[Mangle("__lljit.dlerror_helper")] = JITEvaluatedSymbol( + pointerToJITTargetAddress(dlerrorHelper), JITSymbolFlags()); + + cantFail( + PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols)))); + cantFail(MP.setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule())); + } + + static MachOPlatform &setupPlatform(LLJIT &J) { + auto Tmp = std::make_unique( + J.getExecutionSession(), + static_cast(J.getObjLinkingLayer()), + createStandardSymbolsObject(J)); + auto &MP = *Tmp; + J.getExecutionSession().setPlatform(std::move(Tmp)); + return MP; + } + + static std::unique_ptr createStandardSymbolsObject(LLJIT &J) { + LLVMContext Ctx; + Module M("__standard_symbols", Ctx); + M.setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(Ctx); + + auto *DSOHandle = + new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, 0), "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::HiddenVisibility); + + return cantFail(J.getIRCompileLayer().getCompiler()(M)); + } + + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique(); + auto M = std::make_unique("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *MachOPlatformSupportTy = + StructType::create(*Ctx, "lljit.MachOPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr, + "__lljit.platform_support_instance"); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlopen", + FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlopen_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlclose", + FunctionType::get(IntTy, {BytePtrTy}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlclose_helper", {PlatformInstanceDecl}); + + addHelperAndWrapper( + *M, "dlsym", + FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlsym_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlerror_helper", {PlatformInstanceDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + static_cast(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + static_cast(Self)->AtExitMgr.runAtExits(DSOHandle); + } + + void *jit_dlopen(const char *Path, int Mode) { + JITDylib *JDToOpen = nullptr; + // FIXME: Do the right thing with Mode flags. + { + std::lock_guard Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) { + auto I = JDRefCounts.find(JD); + if (I != JDRefCounts.end()) { + ++I->second; + return JD; + } + + JDRefCounts[JD] = 1; + JDToOpen = JD; + } + } + + if (JDToOpen) { + if (auto Err = initialize(*JDToOpen)) { + recordError(std::move(Err)); + return 0; + } + } + + // Fall through to dlopen if no JITDylib found for Path. + return DlFcn.dlopen(Path, Mode); + } + + static void *dlopenHelper(void *Self, const char *Path, int Mode) { + return static_cast(Self)->jit_dlopen(Path, Mode); + } + + int jit_dlclose(void *Handle) { + JITDylib *JDToClose = nullptr; + + { + std::lock_guard Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + auto I = JDRefCounts.find(Handle); + if (I != JDRefCounts.end()) { + --I->second; + if (I->second == 0) { + JDRefCounts.erase(I); + JDToClose = static_cast(Handle); + } else + return 0; + } + } + + if (JDToClose) { + if (auto Err = deinitialize(*JDToClose)) { + recordError(std::move(Err)); + return -1; + } + return 0; + } + + // Fall through to dlclose if no JITDylib found for Path. + return DlFcn.dlclose(Handle); + } + + static int dlcloseHelper(void *Self, void *Handle) { + return static_cast(Self)->jit_dlclose(Handle); + } + + void *jit_dlsym(void *Handle, const char *Name) { + JITDylibSearchOrder JITSymSearchOrder; + + // FIXME: RTLD_NEXT, RTLD_SELF not supported. + { + std::lock_guard Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (JDRefCounts.count(Handle)) { + JITSymSearchOrder.push_back( + {static_cast(Handle), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } else if (Handle == DlFcn.RTLDDefault) { + for (auto &KV : JDRefCounts) + JITSymSearchOrder.push_back( + {static_cast(KV.first), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } + } + + if (!JITSymSearchOrder.empty()) { + MangleAndInterner Mangle(J.getExecutionSession(), J.getDataLayout()); + auto MangledName = Mangle(Name); + SymbolLookupSet Syms(MangledName, + SymbolLookupFlags::WeaklyReferencedSymbol); + if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms, + LookupKind::DLSym)) { + auto I = Result->find(MangledName); + if (I != Result->end()) + return jitTargetAddressToPointer(I->second.getAddress()); + } else { + recordError(Result.takeError()); + return 0; + } + } + + // Fall through to dlsym. + return DlFcn.dlsym(Handle, Name); + } + + static void *dlsymHelper(void *Self, void *Handle, const char *Name) { + return static_cast(Self)->jit_dlsym(Handle, Name); + } + + const char *jit_dlerror() { + { + std::lock_guard Lock(PlatformSupportMutex); + auto I = dlErrorMsgs.find(std::this_thread::get_id()); + if (I != dlErrorMsgs.end()) + return I->second->c_str(); + } + return DlFcn.dlerror(); + } + + static const char *dlerrorHelper(void *Self) { + return static_cast(Self)->jit_dlerror(); + } + + void recordError(Error Err) { + std::lock_guard Lock(PlatformSupportMutex); + dlErrorMsgs[std::this_thread::get_id()] = + std::make_unique(toString(std::move(Err))); + } + + std::mutex PlatformSupportMutex; + LLJIT &J; + MachOPlatform &MP; + DlFcnValues DlFcn; + ItaniumCXAAtExitSupport AtExitMgr; + DenseMap JDRefCounts; + std::map> dlErrorMsgs; +}; + +} // end anonymous namespace namespace llvm { namespace orc { +void LLJIT::PlatformSupport::setInitTransform( + LLJIT &J, IRTransformLayer::TransformFunction T) { + J.InitHelperTransformLayer->setTransform(std::move(T)); +} + +LLJIT::PlatformSupport::~PlatformSupport() {} + Error LLJITBuilderState::prepareForConstruction() { if (!JTMB) { @@ -57,7 +869,7 @@ LLJIT::~LLJIT() { Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { auto InternedName = ES->intern(Name); SymbolMap Symbols({{InternedName, Sym}}); - return Main.define(absoluteSymbols(std::move(Symbols))); + return Main->define(absoluteSymbols(std::move(Symbols))); } Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { @@ -67,7 +879,8 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) return Err; - return TransformLayer->add(JD, std::move(TSM), ES->allocateVModule()); + return InitHelperTransformLayer->add(JD, std::move(TSM), + ES->allocateVModule()); } Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr Obj) { @@ -128,16 +941,23 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : ES(S.ES ? std::move(S.ES) : std::make_unique()), - Main(this->ES->createJITDylib("
")), DL(""), - TT(S.JTMB->getTargetTriple()), + : ES(S.ES ? std::move(S.ES) : std::make_unique()), Main(), + DL(""), TT(S.JTMB->getTargetTriple()), ObjLinkingLayer(createObjectLinkingLayer(S, *ES)), - ObjTransformLayer(*this->ES, *ObjLinkingLayer), CtorRunner(Main), - DtorRunner(Main) { + ObjTransformLayer(*this->ES, *ObjLinkingLayer) { ErrorAsOutParameter _(&Err); - if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + if (auto MainOrErr = this->ES->createJITDylib("main")) + Main = &*MainOrErr; + else { + Err = MainOrErr.takeError(); + return; + } + + if (S.DL) + DL = std::move(*S.DL); + else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) DL = std::move(*DLOrErr); else { Err = DLOrErr.takeError(); @@ -153,10 +973,12 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) CompileLayer = std::make_unique( *ES, ObjTransformLayer, std::move(*CompileFunction)); TransformLayer = std::make_unique(*ES, *CompileLayer); + InitHelperTransformLayer = + std::make_unique(*ES, *TransformLayer); } if (S.NumCompileThreads > 0) { - TransformLayer->setCloneToNewContextOnEmit(true); + InitHelperTransformLayer->setCloneToNewContextOnEmit(true); CompileThreads = std::make_unique(hardware_concurrency(S.NumCompileThreads)); ES->setDispatchMaterialization( @@ -167,6 +989,11 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) CompileThreads->async(std::move(Work)); }); } + + if (S.SetUpPlatform) + Err = S.SetUpPlatform(*this); + else + setUpGenericLLVMIRPlatform(*this); } std::string LLJIT::mangle(StringRef UnmangledName) { @@ -184,15 +1011,24 @@ Error LLJIT::applyDataLayout(Module &M) { if (M.getDataLayout() != DL) return make_error( - "Added modules have incompatible data layouts", + "Added modules have incompatible data layouts: " + + M.getDataLayout().getStringRepresentation() + " (module) vs " + + DL.getStringRepresentation() + " (jit)", inconvertibleErrorCode()); return Error::success(); } -void LLJIT::recordCtorDtors(Module &M) { - CtorRunner.add(getConstructors(M)); - DtorRunner.add(getDestructors(M)); +void setUpGenericLLVMIRPlatform(LLJIT &J) { + J.setPlatformSupport(std::make_unique(J)); +} + +Error setUpMachOPlatform(LLJIT &J) { + auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib()); + if (!MP) + return MP.takeError(); + J.setPlatformSupport(std::move(*MP)); + return Error::success(); } Error LLLazyJITBuilderState::prepareForConstruction() { @@ -205,13 +1041,8 @@ Error LLLazyJITBuilderState::prepareForConstruction() { Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); - if (auto Err = TSM.withModuleDo([&](Module &M) -> Error { - if (auto Err = applyDataLayout(M)) - return Err; - - recordCtorDtors(M); - return Error::success(); - })) + if (auto Err = TSM.withModuleDo( + [&](Module &M) -> Error { return applyDataLayout(M); })) return Err; return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); @@ -256,7 +1087,7 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { // Create the COD layer. CODLayer = std::make_unique( - *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder)); if (S.NumCompileThreads > 0) CODLayer->setCloneToNewContextOnEmit(true); diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp index ebc7801f11f..63b04a0562b 100644 --- a/lib/ExecutionEngine/Orc/Layer.cpp +++ b/lib/ExecutionEngine/Orc/Layer.cpp @@ -7,7 +7,10 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Layer.h" + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/Constants.h" +#include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" @@ -23,10 +26,11 @@ Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) { *this, *getManglingOptions(), std::move(TSM), std::move(K))); } -IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, - const ManglingOptions &MO, - ThreadSafeModule TSM, VModuleKey K) - : MaterializationUnit(SymbolFlagsMap(), std::move(K)), TSM(std::move(TSM)) { +IRMaterializationUnit::IRMaterializationUnit( + ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap(), nullptr, std::move(K)), + TSM(std::move(TSM)) { assert(this->TSM && "Module must not be null"); @@ -34,6 +38,7 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, this->TSM.withModuleDo([&](Module &M) { for (auto &G : M.global_values()) { // Skip globals that don't generate symbols. + if (!G.hasName() || G.isDeclaration() || G.hasLocalLinkage() || G.hasAvailableExternallyLinkage() || G.hasAppendingLinkage()) continue; @@ -72,13 +77,23 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); SymbolToDefinition[MangledName] = &G; } + + // If we need an init symbol for this module then create one. + if (!llvm::empty(getStaticInitGVs(M))) { + std::string InitSymbolName; + raw_string_ostream(InitSymbolName) + << "$." << M.getModuleIdentifier() << ".__inits"; + InitSymbol = ES.intern(InitSymbolName); + SymbolFlags[InitSymbol] = JITSymbolFlags(); + } }); } IRMaterializationUnit::IRMaterializationUnit( ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition) - : MaterializationUnit(std::move(SymbolFlags), std::move(K)), + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), + std::move(K)), TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} StringRef IRMaterializationUnit::getName() const { @@ -105,7 +120,8 @@ void IRMaterializationUnit::discard(const JITDylib &JD, } BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( - IRLayer &L, const ManglingOptions &MO, ThreadSafeModule TSM, VModuleKey K) + IRLayer &L, const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, + VModuleKey K) : IRMaterializationUnit(L.getExecutionSession(), MO, std::move(TSM), std::move(K)), L(L), K(std::move(K)) {} @@ -150,22 +166,26 @@ Error ObjectLayer::add(JITDylib &JD, std::unique_ptr O, Expected> BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, std::unique_ptr O) { - auto SymbolFlags = - getObjectSymbolFlags(L.getExecutionSession(), O->getMemBufferRef()); + auto ObjSymInfo = + getObjectSymbolInfo(L.getExecutionSession(), O->getMemBufferRef()); - if (!SymbolFlags) - return SymbolFlags.takeError(); + if (!ObjSymInfo) + return ObjSymInfo.takeError(); + + auto &SymbolFlags = ObjSymInfo->first; + auto &InitSymbol = ObjSymInfo->second; return std::unique_ptr( - new BasicObjectLayerMaterializationUnit(L, K, std::move(O), - std::move(*SymbolFlags))); + new BasicObjectLayerMaterializationUnit( + L, K, std::move(O), std::move(SymbolFlags), std::move(InitSymbol))); } BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( ObjectLayer &L, VModuleKey K, std::unique_ptr O, - SymbolFlagsMap SymbolFlags) - : MaterializationUnit(std::move(SymbolFlags), std::move(K)), L(L), - O(std::move(O)) {} + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), + std::move(K)), + L(L), O(std::move(O)) {} StringRef BasicObjectLayerMaterializationUnit::getName() const { if (O) @@ -184,35 +204,5 @@ void BasicObjectLayerMaterializationUnit::discard(const JITDylib &JD, // filter to pass to the object layer along with the object itself. } -Expected getObjectSymbolFlags(ExecutionSession &ES, - MemoryBufferRef ObjBuffer) { - auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); - - if (!Obj) - return Obj.takeError(); - - SymbolFlagsMap SymbolFlags; - for (auto &Sym : (*Obj)->symbols()) { - // Skip symbols not defined in this object file. - if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) - continue; - - // Skip symbols that are not global. - if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) - continue; - - auto Name = Sym.getName(); - if (!Name) - return Name.takeError(); - auto InternedName = ES.intern(*Name); - auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); - if (!SymFlags) - return SymFlags.takeError(); - SymbolFlags[InternedName] = std::move(*SymFlags); - } - - return SymbolFlags; -} - } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp index 97f36ede713..27d08e18270 100644 --- a/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -51,7 +51,7 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { auto LookupResult = ES.lookup( makeJITDylibSearchOrder(SourceJD, JITDylibLookupFlags::MatchAllSymbols), - SymbolName); + SymbolName, SymbolState::Ready); if (!LookupResult) { ES.reportError(LookupResult.takeError()); @@ -123,10 +123,9 @@ LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc, VModuleKey K) - : MaterializationUnit(extractFlags(CallableAliases), std::move(K)), + : MaterializationUnit(extractFlags(CallableAliases), nullptr, std::move(K)), LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), - CallableAliases(std::move(CallableAliases)), - AliaseeTable(SrcJDLoc) {} + CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {} StringRef LazyReexportsMaterializationUnit::getName() const { return ""; diff --git a/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/lib/ExecutionEngine/Orc/MachOPlatform.cpp new file mode 100644 index 00000000000..4618d53315e --- /dev/null +++ b/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -0,0 +1,459 @@ +//===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/Support/BinaryByteStream.h" + +namespace { + +struct objc_class; +struct objc_image_info; +struct objc_object; +struct objc_selector; + +using Class = objc_class *; +using id = objc_object *; +using SEL = objc_selector *; + +using ObjCMsgSendTy = id (*)(id, SEL, ...); +using ObjCReadClassPairTy = Class (*)(Class, const objc_image_info *); +using SelRegisterNameTy = SEL (*)(const char *); + +enum class ObjCRegistrationAPI { Uninitialized, Unavailable, Initialized }; + +ObjCRegistrationAPI ObjCRegistrationAPIState = + ObjCRegistrationAPI::Uninitialized; +ObjCMsgSendTy objc_msgSend = nullptr; +ObjCReadClassPairTy objc_readClassPair = nullptr; +SelRegisterNameTy sel_registerName = nullptr; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +template +static Error setUpObjCRegAPIFunc(FnTy &Target, sys::DynamicLibrary &LibObjC, + const char *Name) { + if (void *Addr = LibObjC.getAddressOfSymbol(Name)) + Target = reinterpret_cast(Addr); + else + return make_error( + (Twine("Could not find address for ") + Name).str(), + inconvertibleErrorCode()); + return Error::success(); +} + +Error enableObjCRegistration(const char *PathToLibObjC) { + // If we've already tried to initialize then just bail out. + if (ObjCRegistrationAPIState != ObjCRegistrationAPI::Uninitialized) + return Error::success(); + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Unavailable; + + std::string ErrMsg; + auto LibObjC = + sys::DynamicLibrary::getPermanentLibrary(PathToLibObjC, &ErrMsg); + + if (!LibObjC.isValid()) + return make_error(std::move(ErrMsg), inconvertibleErrorCode()); + + if (auto Err = setUpObjCRegAPIFunc(objc_msgSend, LibObjC, "objc_msgSend")) + return Err; + if (auto Err = setUpObjCRegAPIFunc(objc_readClassPair, LibObjC, + "objc_readClassPair")) + return Err; + if (auto Err = + setUpObjCRegAPIFunc(sel_registerName, LibObjC, "sel_registerName")) + return Err; + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Initialized; + return Error::success(); +} + +bool objcRegistrationEnabled() { + return ObjCRegistrationAPIState == ObjCRegistrationAPI::Initialized; +} + +void MachOJITDylibInitializers::runModInits() const { + for (const auto &ModInit : ModInitSections) { + for (uint64_t I = 0; I != ModInit.NumPtrs; ++I) { + auto *InitializerAddr = jitTargetAddressToPointer( + ModInit.Address + (I * sizeof(uintptr_t))); + auto *Initializer = + jitTargetAddressToFunction(*InitializerAddr); + Initializer(); + } + } +} + +void MachOJITDylibInitializers::registerObjCSelectors() const { + assert(objcRegistrationEnabled() && "ObjC registration not enabled."); + + for (const auto &ObjCSelRefs : ObjCSelRefsSections) { + for (uint64_t I = 0; I != ObjCSelRefs.NumPtrs; ++I) { + auto SelEntryAddr = ObjCSelRefs.Address + (I * sizeof(uintptr_t)); + const auto *SelName = + *jitTargetAddressToPointer(SelEntryAddr); + auto Sel = sel_registerName(SelName); + *jitTargetAddressToPointer(SelEntryAddr) = Sel; + } + } +} + +Error MachOJITDylibInitializers::registerObjCClasses() const { + assert(objcRegistrationEnabled() && "ObjC registration not enabled."); + + struct ObjCClassCompiled { + void *Metaclass; + void *Parent; + void *Cache1; + void *Cache2; + void *Data; + }; + + auto *ImageInfo = + jitTargetAddressToPointer(ObjCImageInfoAddr); + auto ClassSelector = sel_registerName("class"); + + for (const auto &ObjCClassList : ObjCClassListSections) { + for (uint64_t I = 0; I != ObjCClassList.NumPtrs; ++I) { + auto ClassPtrAddr = ObjCClassList.Address + (I * sizeof(uintptr_t)); + auto Cls = *jitTargetAddressToPointer(ClassPtrAddr); + auto *ClassCompiled = + *jitTargetAddressToPointer(ClassPtrAddr); + objc_msgSend(reinterpret_cast(ClassCompiled->Parent), ClassSelector); + auto Registered = objc_readClassPair(Cls, ImageInfo); + + // FIXME: Improve diagnostic by reporting the failed class's name. + if (Registered != Cls) + return make_error("Unable to register Objective-C class", + inconvertibleErrorCode()); + } + } + return Error::success(); +} + +void MachOJITDylibInitializers::dump() const { + for (auto &Extent : ModInitSections) + dbgs() << formatv("{0:x16}", Extent.Address) << " -- " + << formatv("{0:x16}", Extent.Address + 8 * Extent.NumPtrs) << "\n"; +} + +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr StandardSymbolsObject) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + StandardSymbolsObject(std::move(StandardSymbolsObject)) { + ObjLinkingLayer.addPlugin(std::make_unique(*this)); +} + +Error MachOPlatform::setupJITDylib(JITDylib &JD) { + auto ObjBuffer = MemoryBuffer::getMemBuffer( + StandardSymbolsObject->getMemBufferRef(), false); + return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); +} + +Error MachOPlatform::notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + std::lock_guard Lock(PlatformMutex); + RegisteredInitSymbols[&JD].add(InitSym); + return Error::success(); +} + +Error MachOPlatform::notifyRemoving(JITDylib &JD, VModuleKey K) { + llvm_unreachable("Not supported yet"); +} + +Expected +MachOPlatform::getInitializerSequence(JITDylib &JD) { + + std::vector DFSLinkOrder; + + while (true) { + // Lock the platform while we search for any initializer symbols to + // look up. + DenseMap NewInitSymbols; + { + std::lock_guard Lock(PlatformMutex); + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + } + } + + if (NewInitSymbols.empty()) + break; + + // Outside the lock, issue the lookup. + if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) + ; // Nothing to do in the success case. + else + return R.takeError(); + } + + // Lock again to collect the initializers. + InitializerSequence FullInitSeq; + { + std::lock_guard Lock(PlatformMutex); + for (auto *InitJD : reverse(DFSLinkOrder)) { + auto ISItr = InitSeqs.find(InitJD); + if (ISItr != InitSeqs.end()) { + FullInitSeq.emplace_back(InitJD, std::move(ISItr->second)); + InitSeqs.erase(ISItr); + } + } + } + + return FullInitSeq; +} + +Expected +MachOPlatform::getDeinitializerSequence(JITDylib &JD) { + std::vector DFSLinkOrder = getDFSLinkOrder(JD); + + DeinitializerSequence FullDeinitSeq; + { + std::lock_guard Lock(PlatformMutex); + for (auto *DeinitJD : DFSLinkOrder) { + FullDeinitSeq.emplace_back(DeinitJD, MachOJITDylibDeinitializers()); + } + } + + return FullDeinitSeq; +} + +std::vector MachOPlatform::getDFSLinkOrder(JITDylib &JD) { + std::vector Result, WorkStack({&JD}); + DenseSet Visited; + + while (!WorkStack.empty()) { + auto *NextJD = WorkStack.back(); + WorkStack.pop_back(); + if (Visited.count(NextJD)) + continue; + Visited.insert(NextJD); + Result.push_back(NextJD); + NextJD->withSearchOrderDo([&](const JITDylibSearchOrder &SO) { + for (auto &KV : SO) + WorkStack.push_back(KV.first); + }); + } + + return Result; +} + +void MachOPlatform::registerInitInfo( + JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList) { + std::lock_guard Lock(PlatformMutex); + + auto &InitSeq = InitSeqs[&JD]; + + InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); + + if (ModInits.Address) + InitSeq.addModInitsSection(std::move(ModInits)); + + if (ObjCSelRefs.Address) + InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); + + if (ObjCClassList.Address) + InitSeq.addObjCClassListSection(std::move(ObjCClassList)); +} + +static Expected +getSectionExtent(jitlink::LinkGraph &G, StringRef SectionName) { + auto *Sec = G.findSectionByName(SectionName); + if (!Sec) + return MachOJITDylibInitializers::SectionExtent(); + jitlink::SectionRange R(*Sec); + if (R.getSize() % G.getPointerSize() != 0) + return make_error(SectionName + " section size is not a " + "multiple of the pointer size", + inconvertibleErrorCode()); + return MachOJITDylibInitializers::SectionExtent( + R.getStart(), R.getSize() / G.getPointerSize()); +} + +void MachOPlatform::InitScraperPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) { + + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { + JITLinkSymbolVector InitSectionSymbols; + preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist"); + + if (!InitSymbolDeps.empty()) { + std::lock_guard Lock(InitScraperMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); + } + + if (auto Err = processObjCImageInfo(G, MR)) + return Err; + + return Error::success(); + }); + + Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + MachOJITDylibInitializers::SectionExtent ModInits, ObjCSelRefs, + ObjCClassList; + + JITTargetAddress ObjCImageInfoAddr = 0; + if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) { + if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) { + ObjCImageInfoAddr = Addr; + dbgs() << "Recorded __objc_imageinfo @ " << formatv("{0:x16}", Addr); + } + } + + // Record __mod_init_func. + if (auto ModInitsOrErr = getSectionExtent(G, "__mod_init_func")) + ModInits = std::move(*ModInitsOrErr); + else + return ModInitsOrErr.takeError(); + + // Record __objc_selrefs. + if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__objc_selrefs")) + ObjCSelRefs = std::move(*ObjCSelRefsOrErr); + else + return ObjCSelRefsOrErr.takeError(); + + // Record __objc_classlist. + if (auto ObjCClassListOrErr = getSectionExtent(G, "__objc_classlist")) + ObjCClassList = std::move(*ObjCClassListOrErr); + else + return ObjCClassListOrErr.takeError(); + + MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), + std::move(ObjCSelRefs), std::move(ObjCClassList)); + + return Error::success(); + }); +} + +ObjectLinkingLayer::Plugin::LocalDependenciesMap +MachOPlatform::InitScraperPlugin::getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) { + std::lock_guard Lock(InitScraperMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + LocalDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return LocalDependenciesMap(); +} + +void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( + JITLinkSymbolVector &Symbols, jitlink::LinkGraph &G, + StringRef SectionName) { + if (auto *Sec = G.findSectionByName(SectionName)) { + auto SecBlocks = Sec->blocks(); + if (!llvm::empty(SecBlocks)) + Symbols.push_back( + &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); + } +} + +Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + // If there's an ObjC imagine info then either + // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In + // this case we name and record it. + // OR + // (2) We already have a recorded __objc_imageinfo for this JITDylib, + // in which case we just verify it. + auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); + if (!ObjCImageInfo) + return Error::success(); + + auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); + + // Check that the section is not empty if present. + if (llvm::empty(ObjCImageInfoBlocks)) + return make_error("Empty __objc_imageinfo section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that there's only one block in the section. + if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) + return make_error("Multiple blocks in __objc_imageinfo " + "section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that the __objc_imageinfo section is unreferenced. + // FIXME: We could optimize this check if Symbols had a ref-count. + for (auto &Sec : G.sections()) { + if (&Sec != ObjCImageInfo) + for (auto *B : Sec.blocks()) + for (auto &E : B->edges()) + if (E.getTarget().isDefined() && + &E.getTarget().getBlock().getSection() == ObjCImageInfo) + return make_error("__objc_imageinfo is referenced " + "within file " + + G.getName(), + inconvertibleErrorCode()); + } + + auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); + auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); + auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness()); + auto Flags = + support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); + + // Lock the mutex while we verify / update the ObjCImageInfos map. + std::lock_guard Lock(InitScraperMutex); + + auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); + if (ObjCImageInfoItr != ObjCImageInfos.end()) { + // We've already registered an __objc_imageinfo section. Verify the + // content of this new section matches, then delete it. + if (ObjCImageInfoItr->second.first != Version) + return make_error( + "ObjC version in " + G.getName() + + " does not match first registered version", + inconvertibleErrorCode()); + if (ObjCImageInfoItr->second.second != Flags) + return make_error("ObjC flags in " + G.getName() + + " do not match first registered flags", + inconvertibleErrorCode()); + + // __objc_imageinfo is valid. Delete the block. + for (auto *S : ObjCImageInfo->symbols()) + G.removeDefinedSymbol(*S); + G.removeBlock(ObjCImageInfoBlock); + } else { + // We haven't registered an __objc_imageinfo section yet. Register and + // move on. The section should already be marked no-dead-strip. + ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/Mangling.cpp b/lib/ExecutionEngine/Orc/Mangling.cpp new file mode 100644 index 00000000000..7ec7869d00b --- /dev/null +++ b/lib/ExecutionEngine/Orc/Mangling.cpp @@ -0,0 +1,131 @@ +//===----------- Mangling.cpp -- Name Mangling Utilities for ORC ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.intern(MangledName); +} + +void IRSymbolMapper::add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef GVs, + SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition) { + if (GVs.empty()) + return; + + MangleAndInterner Mangle(ES, GVs[0]->getParent()->getDataLayout()); + for (auto *G : GVs) { + assert(G && "GVs cannot contain null elements"); + if (!G->hasName() || G->isDeclaration() || G->hasLocalLinkage() || + G->hasAvailableExternallyLinkage() || G->hasAppendingLinkage()) + continue; + + if (G->isThreadLocal() && MO.EmulatedTLS) { + auto *GV = cast(G); + + auto Flags = JITSymbolFlags::fromGlobalValue(*GV); + + auto EmuTLSV = Mangle(("__emutls_v." + GV->getName()).str()); + SymbolFlags[EmuTLSV] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLSV] = GV; + + // If this GV has a non-zero initializer we'll need to emit an + // __emutls.t symbol too. + if (GV->hasInitializer()) { + const auto *InitVal = GV->getInitializer(); + + // Skip zero-initializers. + if (isa(InitVal)) + continue; + const auto *InitIntValue = dyn_cast(InitVal); + if (InitIntValue && InitIntValue->isZero()) + continue; + + auto EmuTLST = Mangle(("__emutls_t." + GV->getName()).str()); + SymbolFlags[EmuTLST] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLST] = GV; + } + continue; + } + + // Otherwise we just need a normal linker mangling. + auto MangledName = Mangle(G->getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(*G); + if (SymbolToDefinition) + (*SymbolToDefinition)[MangledName] = G; + } +} + +Expected> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + SymbolFlagsMap SymbolFlags; + for (auto &Sym : (*Obj)->symbols()) { + // Skip symbols not defined in this object file. + if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) + continue; + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + SymbolStringPtr InitSymbol; + + if (auto *MachOObj = dyn_cast(Obj->get())) { + for (auto &Sec : MachOObj->sections()) { + auto SecType = MachOObj->getSectionType(Sec); + if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << ObjBuffer.getBufferIdentifier() << ".__inits"; + InitSymbol = ES.intern(InitSymString); + break; + } + } + } + + return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 6575b0a6ccb..db88ec8517b 100644 --- a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -144,6 +144,10 @@ public: if (!ExtraSymbolsToClaim.empty()) if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) return notifyFailed(std::move(Err)); + + if (const auto &InitSym = MR.getInitializerSymbol()) + InternedResult[InitSym] = JITEvaluatedSymbol(); + if (auto Err = MR.notifyResolved(InternedResult)) { Layer.getExecutionSession().reportError(std::move(Err)); MR.failMaterialization(); @@ -184,8 +188,12 @@ public: } private: - using JITLinkSymbolSet = DenseSet; - using LocalToNamedDependenciesMap = DenseMap; + struct LocalSymbolNamedDependencies { + SymbolNameSet Internal, External; + }; + + using LocalSymbolNamedDependenciesMap = + DenseMap; Error externalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); @@ -216,6 +224,7 @@ private: auto &ES = MR.getTargetJITDylib().getExecutionSession(); auto LocalDeps = computeLocalDeps(G); + // Compute dependencies for symbols defined in the JITLink graph. for (auto *Sym : G.defined_symbols()) { // Skip local symbols: we do not track dependencies for these. @@ -239,15 +248,12 @@ private: assert(TargetSym.isDefined() && "local symbols must be defined"); auto I = LocalDeps.find(&TargetSym); - if (I != LocalDeps.end()) - for (auto &S : I->second) { - assert(S->hasName() && - "LocalDeps should only contain named values"); - if (S->isExternal()) - ExternalSymDeps.insert(ES.intern(S->getName())); - else if (S != Sym) - InternalSymDeps.insert(ES.intern(S->getName())); - } + if (I != LocalDeps.end()) { + for (auto &S : I->second.External) + ExternalSymDeps.insert(S); + for (auto &S : I->second.Internal) + InternalSymDeps.insert(S); + } } } @@ -261,11 +267,33 @@ private: InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps); } + for (auto &P : Layer.Plugins) { + auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(MR); + if (SyntheticLocalDeps.empty()) + continue; + + for (auto &KV : SyntheticLocalDeps) { + auto &Name = KV.first; + auto &LocalDepsForName = KV.second; + for (auto *Local : LocalDepsForName) { + assert(Local->getScope() == Scope::Local && + "Dependence on non-local symbol"); + auto LocalNamedDepsItr = LocalDeps.find(Local); + if (LocalNamedDepsItr == LocalDeps.end()) + continue; + for (auto &S : LocalNamedDepsItr->second.Internal) + InternalNamedSymbolDeps[Name].insert(S); + for (auto &S : LocalNamedDepsItr->second.External) + ExternalNamedSymbolDeps[Name].insert(S); + } + } + } + return Error::success(); } - LocalToNamedDependenciesMap computeLocalDeps(LinkGraph &G) { - LocalToNamedDependenciesMap DepMap; + LocalSymbolNamedDependenciesMap computeLocalDeps(LinkGraph &G) { + DenseMap> DepMap; // For all local symbols: // (1) Add their named dependencies. @@ -319,7 +347,26 @@ private: } } while (Changed); - return DepMap; + // Intern the results to produce a mapping of jitlink::Symbol* to internal + // and external symbol names. + auto &ES = Layer.getExecutionSession(); + LocalSymbolNamedDependenciesMap Result; + for (auto &KV : DepMap) { + auto *Local = KV.first; + assert(Local->getScope() == Scope::Local && + "DepMap keys should all be local symbols"); + auto &LocalNamedDeps = Result[Local]; + for (auto *Named : KV.second) { + assert(Named->getScope() != Scope::Local && + "DepMap values should all be non-local symbol sets"); + if (Named->isExternal()) + LocalNamedDeps.External.insert(ES.intern(Named->getName())); + else + LocalNamedDeps.Internal.insert(ES.intern(Named->getName())); + } + } + + return Result; } void registerDependencies(const SymbolDependenceMap &QueryDeps) { diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index ff8289a264c..0eaf13c6e3c 100644 --- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -88,7 +88,8 @@ RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() { void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr O) { assert(O && "Object must not be null"); - + dbgs() << "Emitting via RTDyldObjectLinkingLayer:\n" + << R.getSymbols() << "\n"; // This method launches an asynchronous link step that will fulfill our // materialization responsibility. We need to switch R to be heap // allocated before that happens so it can live as long as the asynchronous @@ -229,6 +230,9 @@ Error RTDyldObjectLinkingLayer::onObjLoad( Symbols.erase(KV.first); } + if (const auto &InitSym = R.getInitializerSymbol()) + Symbols[InitSym] = JITEvaluatedSymbol(); + if (auto Err = R.notifyResolved(Symbols)) { R.failMaterialization(); return Err; diff --git a/test/ExecutionEngine/OrcLazy/objc-minimal.ll b/test/ExecutionEngine/OrcLazy/objc-minimal.ll new file mode 100644 index 00000000000..8a940573dcb --- /dev/null +++ b/test/ExecutionEngine/OrcLazy/objc-minimal.ll @@ -0,0 +1,73 @@ +; REQUIRES: system-darwin +; RUN: lli -jit-kind=orc-lazy %s +; +; Sanity check MachO Platform support: Call a no-op method (returning int 0) on +; an ObjC object. This test checks that we correctly auto-identify this as a +; MachO target, configure MachOPlatform support, and correctly register the +; class metadata and method selector with the Objective-C runtime. + +source_filename = "objc-minimal.mm" +target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.14.0" + +%0 = type opaque +%struct._objc_cache = type opaque +%struct._class_t = type { %struct._class_t*, %struct._class_t*, %struct._objc_cache*, i8* (i8*, i8*)**, %struct._class_ro_t* } +%struct._class_ro_t = type { i32, i32, i32, i8*, i8*, %struct.__method_list_t*, %struct._objc_protocol_list*, %struct._ivar_list_t*, i8*, %struct._prop_list_t* } +%struct.__method_list_t = type { i32, i32, [0 x %struct._objc_method] } +%struct._objc_method = type { i8*, i8*, i8* } +%struct._objc_protocol_list = type { i64, [0 x %struct._protocol_t*] } +%struct._protocol_t = type { i8*, i8*, %struct._objc_protocol_list*, %struct.__method_list_t*, %struct.__method_list_t*, %struct.__method_list_t*, %struct.__method_list_t*, %struct._prop_list_t*, i32, i32, i8**, i8*, %struct._prop_list_t* } +%struct._ivar_list_t = type { i32, i32, [0 x %struct._ivar_t] } +%struct._ivar_t = type { i64*, i8*, i8*, i32, i32 } +%struct._prop_list_t = type { i32, i32, [0 x %struct._prop_t] } +%struct._prop_t = type { i8*, i8* } + +@_objc_empty_cache = external global %struct._objc_cache +@"OBJC_METACLASS_$_NSObject" = external global %struct._class_t +@OBJC_CLASS_NAME_ = private unnamed_addr constant [4 x i8] c"Foo\00", section "__TEXT,__objc_classname,cstring_literals", align 1 +@"\01l_OBJC_METACLASS_RO_$_Foo" = private global %struct._class_ro_t { i32 1, i32 40, i32 40, i8* null, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @OBJC_CLASS_NAME_, i32 0, i32 0), %struct.__method_list_t* null, %struct._objc_protocol_list* null, %struct._ivar_list_t* null, i8* null, %struct._prop_list_t* null }, section "__DATA, __objc_const", align 8 +@"OBJC_METACLASS_$_Foo" = global %struct._class_t { %struct._class_t* @"OBJC_METACLASS_$_NSObject", %struct._class_t* @"OBJC_METACLASS_$_NSObject", %struct._objc_cache* @_objc_empty_cache, i8* (i8*, i8*)** null, %struct._class_ro_t* @"\01l_OBJC_METACLASS_RO_$_Foo" }, section "__DATA, __objc_data", align 8 +@"OBJC_CLASS_$_NSObject" = external global %struct._class_t +@OBJC_METH_VAR_NAME_ = private unnamed_addr constant [4 x i8] c"foo\00", section "__TEXT,__objc_methname,cstring_literals", align 1 +@OBJC_METH_VAR_TYPE_ = private unnamed_addr constant [8 x i8] c"i16@0:8\00", section "__TEXT,__objc_methtype,cstring_literals", align 1 +@"\01l_OBJC_$_INSTANCE_METHODS_Foo" = private global { i32, i32, [1 x %struct._objc_method] } { i32 24, i32 1, [1 x %struct._objc_method] [%struct._objc_method { i8* getelementptr inbounds ([4 x i8], [4 x i8]* @OBJC_METH_VAR_NAME_, i32 0, i32 0), i8* getelementptr inbounds ([8 x i8], [8 x i8]* @OBJC_METH_VAR_TYPE_, i32 0, i32 0), i8* bitcast (i32 (%0*, i8*)* @"\01-[Foo foo]" to i8*) }] }, section "__DATA, __objc_const", align 8 +@"\01l_OBJC_CLASS_RO_$_Foo" = private global %struct._class_ro_t { i32 0, i32 8, i32 8, i8* null, i8* getelementptr inbounds ([4 x i8], [4 x i8]* @OBJC_CLASS_NAME_, i32 0, i32 0), %struct.__method_list_t* bitcast ({ i32, i32, [1 x %struct._objc_method] }* @"\01l_OBJC_$_INSTANCE_METHODS_Foo" to %struct.__method_list_t*), %struct._objc_protocol_list* null, %struct._ivar_list_t* null, i8* null, %struct._prop_list_t* null }, section "__DATA, __objc_const", align 8 +@"OBJC_CLASS_$_Foo" = global %struct._class_t { %struct._class_t* @"OBJC_METACLASS_$_Foo", %struct._class_t* @"OBJC_CLASS_$_NSObject", %struct._objc_cache* @_objc_empty_cache, i8* (i8*, i8*)** null, %struct._class_ro_t* @"\01l_OBJC_CLASS_RO_$_Foo" }, section "__DATA, __objc_data", align 8 +@"OBJC_CLASSLIST_REFERENCES_$_" = private global %struct._class_t* @"OBJC_CLASS_$_Foo", section "__DATA,__objc_classrefs,regular,no_dead_strip", align 8 +@OBJC_METH_VAR_NAME_.1 = private unnamed_addr constant [5 x i8] c"init\00", section "__TEXT,__objc_methname,cstring_literals", align 1 +@OBJC_SELECTOR_REFERENCES_ = private externally_initialized global i8* getelementptr inbounds ([5 x i8], [5 x i8]* @OBJC_METH_VAR_NAME_.1, i64 0, i64 0), section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip", align 8 +@OBJC_SELECTOR_REFERENCES_.2 = private externally_initialized global i8* getelementptr inbounds ([4 x i8], [4 x i8]* @OBJC_METH_VAR_NAME_, i64 0, i64 0), section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip", align 8 +@"OBJC_LABEL_CLASS_$" = private global [1 x i8*] [i8* bitcast (%struct._class_t* @"OBJC_CLASS_$_Foo" to i8*)], section "__DATA,__objc_classlist,regular,no_dead_strip", align 8 +@llvm.compiler.used = appending global [9 x i8*] [i8* bitcast ({ i32, i32, [1 x %struct._objc_method] }* @"\01l_OBJC_$_INSTANCE_METHODS_Foo" to i8*), i8* bitcast (%struct._class_t** @"OBJC_CLASSLIST_REFERENCES_$_" to i8*), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @OBJC_CLASS_NAME_, i32 0, i32 0), i8* bitcast ([1 x i8*]* @"OBJC_LABEL_CLASS_$" to i8*), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @OBJC_METH_VAR_NAME_, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @OBJC_METH_VAR_NAME_.1, i32 0, i32 0), i8* getelementptr inbounds ([8 x i8], [8 x i8]* @OBJC_METH_VAR_TYPE_, i32 0, i32 0), i8* bitcast (i8** @OBJC_SELECTOR_REFERENCES_ to i8*), i8* bitcast (i8** @OBJC_SELECTOR_REFERENCES_.2 to i8*)], section "llvm.metadata" + +; Function Attrs: noinline norecurse nounwind readnone ssp uwtable +define internal i32 @"\01-[Foo foo]"(%0* nocapture readnone, i8* nocapture readnone) { + ret i32 0 +} + +; Function Attrs: noinline norecurse ssp uwtable +define i32 @main(i32, i8** nocapture readnone) local_unnamed_addr { + %3 = load i8*, i8** bitcast (%struct._class_t** @"OBJC_CLASSLIST_REFERENCES_$_" to i8**), align 8 + %4 = tail call i8* @objc_alloc(i8* %3) + %5 = load i8*, i8** @OBJC_SELECTOR_REFERENCES_, align 8 + %6 = tail call i8* bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i8* (i8*, i8*)*)(i8* %4, i8* %5) + %7 = load i8*, i8** @OBJC_SELECTOR_REFERENCES_.2, align 8 + %8 = tail call i32 bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to i32 (i8*, i8*)*)(i8* %6, i8* %7) + ret i32 %8 +} + +declare i8* @objc_alloc(i8*) local_unnamed_addr + +; Function Attrs: nonlazybind +declare i8* @objc_msgSend(i8*, i8*, ...) local_unnamed_addr + +!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6 } + +!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 10, i32 15]} +!1 = !{i32 1, !"Objective-C Version", i32 2} +!2 = !{i32 1, !"Objective-C Image Info Version", i32 0} +!3 = !{i32 1, !"Objective-C Image Info Section", !"__DATA,__objc_imageinfo,regular,no_dead_strip"} +!4 = !{i32 4, !"Objective-C Garbage Collection", i32 0} +!5 = !{i32 1, !"Objective-C Class Properties", i32 64} +!6 = !{i32 1, !"wchar_size", i32 4} diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp index e7251b12f7f..55b9557bddf 100644 --- a/tools/lli/lli.cpp +++ b/tools/lli/lli.cpp @@ -24,9 +24,11 @@ #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/MCJIT.h" #include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h" #include "llvm/ExecutionEngine/OrcMCJITReplacement.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" @@ -206,6 +208,19 @@ namespace { cl::desc("Do not resolve lli process symbols in JIT'd code"), cl::init(false)); + enum class LLJITPlatform { DetectHost, GenericIR, MachO }; + + cl::opt + Platform("lljit-platform", cl::desc("Platform to use with LLJIT"), + cl::init(LLJITPlatform::DetectHost), + cl::values(clEnumValN(LLJITPlatform::DetectHost, "DetectHost", + "Select based on JIT target triple"), + clEnumValN(LLJITPlatform::GenericIR, "GenericIR", + "Use LLJITGenericIRPlatform"), + clEnumValN(LLJITPlatform::MachO, "MachO", + "Use LLJITMachOPlatform")), + cl::Hidden); + enum class DumpKind { NoDump, DumpFuncsToStdOut, @@ -772,16 +787,20 @@ int runOrcLazyJIT(const char *ProgName) { if (!MainModule) reportError(Err, ProgName); - const auto &TT = MainModule->getTargetTriple(); + Triple TT(MainModule->getTargetTriple()); orc::LLLazyJITBuilder Builder; Builder.setJITTargetMachineBuilder( - TT.empty() ? ExitOnErr(orc::JITTargetMachineBuilder::detectHost()) - : orc::JITTargetMachineBuilder(Triple(TT))); + MainModule->getTargetTriple().empty() + ? ExitOnErr(orc::JITTargetMachineBuilder::detectHost()) + : orc::JITTargetMachineBuilder(TT)); if (!MArch.empty()) Builder.getJITTargetMachineBuilder()->getTargetTriple().setArchName(MArch); + if (!MainModule->getDataLayout().isDefault()) + Builder.setDataLayout(MainModule->getDataLayout()); + Builder.getJITTargetMachineBuilder() ->setCPU(getCPUStr()) .addFeatures(getFeatureList()) @@ -796,6 +815,29 @@ int runOrcLazyJIT(const char *ProgName) { pointerToJITTargetAddress(exitOnLazyCallThroughFailure)); Builder.setNumCompileThreads(LazyJITCompileThreads); + // Set up LLJIT platform. + { + LLJITPlatform P = Platform; + if (P == LLJITPlatform::DetectHost) { + if (TT.isOSBinFormatMachO()) + P = LLJITPlatform::MachO; + else + P = LLJITPlatform::GenericIR; + } + + switch (P) { + case LLJITPlatform::GenericIR: + // Nothing to do: LLJITBuilder will use this by default. + break; + case LLJITPlatform::MachO: + Builder.setPlatformSetUp(orc::setUpMachOPlatform); + ExitOnErr(orc::enableObjCRegistration("libobjc.dylib")); + break; + default: + llvm_unreachable("Unrecognized platform value"); + } + } + auto J = ExitOnErr(Builder.create()); if (PerModuleLazy) @@ -828,9 +870,6 @@ int runOrcLazyJIT(const char *ProgName) { return Name != MainName; }))); - orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; - ExitOnErr(CXXRuntimeOverrides.enable(J->getMainJITDylib(), Mangle)); - // Add the main module. ExitOnErr( J->addLazyIRModule(orc::ThreadSafeModule(std::move(MainModule), TSCtx))); @@ -845,8 +884,11 @@ int runOrcLazyJIT(const char *ProgName) { for (auto JDItr = JITDylibs.begin(), JDEnd = JITDylibs.end(); JDItr != JDEnd; ++JDItr) { orc::JITDylib *JD = J->getJITDylibByName(*JDItr); - if (!JD) - JD = &J->createJITDylib(*JDItr); + if (!JD) { + JD = &ExitOnErr(J->createJITDylib(*JDItr)); + J->getMainJITDylib().addToSearchOrder(*JD); + JD->addToSearchOrder(J->getMainJITDylib()); + } IdxToDylib[JITDylibs.getPosition(JDItr - JITDylibs.begin())] = JD; } @@ -882,7 +924,7 @@ int runOrcLazyJIT(const char *ProgName) { } // Run any static constructors. - ExitOnErr(J->runConstructors()); + ExitOnErr(J->initialize(J->getMainJITDylib())); // Run any -thread-entry points. std::vector AltEntryThreads; @@ -907,8 +949,7 @@ int runOrcLazyJIT(const char *ProgName) { AltEntryThread.join(); // Run destructors. - ExitOnErr(J->runDestructors()); - CXXRuntimeOverrides.runDestructors(); + ExitOnErr(J->deinitialize(J->getMainJITDylib())); return Result; } diff --git a/tools/llvm-jitlink/llvm-jitlink.cpp b/tools/llvm-jitlink/llvm-jitlink.cpp index 26bcf46d60a..7765d53f402 100644 --- a/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/tools/llvm-jitlink/llvm-jitlink.cpp @@ -396,9 +396,18 @@ static std::unique_ptr createMemoryManager() { return std::make_unique(); } -Session::Session(Triple TT) - : MainJD(ES.createJITDylib("
")), ObjLayer(ES, createMemoryManager()), - TT(std::move(TT)) { +Expected> Session::Create(Triple TT) { + Error Err = Error::success(); + std::unique_ptr S(new Session(std::move(TT), Err)); + if (Err) + return std::move(Err); + return std::move(S); +} + +// FIXME: Move to createJITDylib if/when we start using Platform support in +// llvm-jitlink. +Session::Session(Triple TT, Error &Err) + : ObjLayer(ES, createMemoryManager()), TT(std::move(TT)) { /// Local ObjectLinkingLayer::Plugin class to forward modifyPassConfig to the /// Session. @@ -414,6 +423,15 @@ Session::Session(Triple TT) Session &S; }; + ErrorAsOutParameter _(&Err); + + if (auto MainJDOrErr = ES.createJITDylib("main")) + MainJD = &*MainJDOrErr; + else { + Err = MainJDOrErr.takeError(); + return; + } + if (!NoExec && !TT.isOSWindows()) ObjLayer.addPlugin(std::make_unique( InProcessEHFrameRegistrar::getInstance())); @@ -561,7 +579,7 @@ Error loadProcessSymbols(Session &S) { auto FilterMainEntryPoint = [InternedEntryPointName](SymbolStringPtr Name) { return Name != InternedEntryPointName; }; - S.MainJD.addGenerator( + S.MainJD->addGenerator( ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( GlobalPrefix, FilterMainEntryPoint))); @@ -590,20 +608,22 @@ Error loadObjects(Session &S) { LLVM_DEBUG(dbgs() << "Creating JITDylibs...\n"); { // Create a "main" JITLinkDylib. - IdxToJLD[0] = &S.MainJD; - S.JDSearchOrder.push_back(&S.MainJD); - LLVM_DEBUG(dbgs() << " 0: " << S.MainJD.getName() << "\n"); + IdxToJLD[0] = S.MainJD; + S.JDSearchOrder.push_back(S.MainJD); + LLVM_DEBUG(dbgs() << " 0: " << S.MainJD->getName() << "\n"); // Add any extra JITLinkDylibs from the command line. std::string JDNamePrefix("lib"); for (auto JLDItr = JITLinkDylibs.begin(), JLDEnd = JITLinkDylibs.end(); JLDItr != JLDEnd; ++JLDItr) { - auto &JD = S.ES.createJITDylib(JDNamePrefix + *JLDItr); + auto JD = S.ES.createJITDylib(JDNamePrefix + *JLDItr); + if (!JD) + return JD.takeError(); unsigned JDIdx = JITLinkDylibs.getPosition(JLDItr - JITLinkDylibs.begin()); - IdxToJLD[JDIdx] = &JD; - S.JDSearchOrder.push_back(&JD); - LLVM_DEBUG(dbgs() << " " << JDIdx << ": " << JD.getName() << "\n"); + IdxToJLD[JDIdx] = &*JD; + S.JDSearchOrder.push_back(&*JD); + LLVM_DEBUG(dbgs() << " " << JDIdx << ": " << JD->getName() << "\n"); } // Set every dylib to link against every other, in command line order. @@ -790,32 +810,32 @@ int main(int argc, char *argv[]) { std::unique_ptr Timers = ShowTimes ? std::make_unique() : nullptr; - Session S(getFirstFileTriple()); + auto S = ExitOnErr(Session::Create(getFirstFileTriple())); - ExitOnErr(sanitizeArguments(S)); + ExitOnErr(sanitizeArguments(*S)); if (!NoProcessSymbols) - ExitOnErr(loadProcessSymbols(S)); + ExitOnErr(loadProcessSymbols(*S)); ExitOnErr(loadDylibs()); { TimeRegion TR(Timers ? &Timers->LoadObjectsTimer : nullptr); - ExitOnErr(loadObjects(S)); + ExitOnErr(loadObjects(*S)); } JITEvaluatedSymbol EntryPoint = 0; { TimeRegion TR(Timers ? &Timers->LinkTimer : nullptr); - EntryPoint = ExitOnErr(getMainEntryPoint(S)); + EntryPoint = ExitOnErr(getMainEntryPoint(*S)); } if (ShowAddrs) - S.dumpSessionInfo(outs()); + S->dumpSessionInfo(outs()); - ExitOnErr(runChecks(S)); + ExitOnErr(runChecks(*S)); - dumpSessionStats(S); + dumpSessionStats(*S); if (NoExec) return 0; diff --git a/tools/llvm-jitlink/llvm-jitlink.h b/tools/llvm-jitlink/llvm-jitlink.h index 0f92d760501..c888baec9ad 100644 --- a/tools/llvm-jitlink/llvm-jitlink.h +++ b/tools/llvm-jitlink/llvm-jitlink.h @@ -26,12 +26,13 @@ namespace llvm { struct Session { orc::ExecutionSession ES; - orc::JITDylib &MainJD; + orc::JITDylib *MainJD; orc::ObjectLinkingLayer ObjLayer; std::vector JDSearchOrder; Triple TT; Session(Triple TT); + static Expected> Create(Triple TT); void dumpSessionInfo(raw_ostream &OS); void modifyPassConfig(const Triple &FTT, jitlink::PassConfiguration &PassConfig); @@ -63,6 +64,9 @@ struct Session { FileInfoMap FileInfos; uint64_t SizeBeforePruning = 0; uint64_t SizeAfterFixups = 0; + +private: + Session(Triple TT, Error &Err); }; Error registerMachOStubsAndGOT(Session &S, jitlink::LinkGraph &G); diff --git a/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp index 68a6d2ed2ca..171ff37f006 100644 --- a/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp +++ b/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp @@ -113,6 +113,7 @@ TEST_F(CoreAPIsStandardTest, RemoveSymbolsTest) { cantFail(R.notifyResolved({{Bar, BarSym}})); cantFail(R.notifyEmitted()); }, + nullptr, [&](const JITDylib &JD, const SymbolStringPtr &Name) { EXPECT_EQ(Name, Bar) << "Expected \"Bar\" to be discarded"; if (Name == Bar) @@ -126,6 +127,7 @@ TEST_F(CoreAPIsStandardTest, RemoveSymbolsTest) { cantFail(JD.define(std::make_unique( SymbolFlagsMap({{Baz, BazSym.getFlags()}}), [&](MaterializationResponsibility R) { BazR.emplace(std::move(R)); }, + nullptr, [](const JITDylib &JD, const SymbolStringPtr &Name) { ADD_FAILURE() << "\"Baz\" discarded unexpectedly"; }))); @@ -176,7 +178,7 @@ TEST_F(CoreAPIsStandardTest, RemoveSymbolsTest) { TEST_F(CoreAPIsStandardTest, ChainedJITDylibLookup) { cantFail(JD.define(absoluteSymbols({{Foo, FooSym}}))); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); bool OnCompletionRun = false; @@ -198,7 +200,7 @@ TEST_F(CoreAPIsStandardTest, LookupWithHiddenSymbols) { cantFail(JD.define(absoluteSymbols({{Foo, FooSym}, {Bar, BarHiddenSym}}))); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(absoluteSymbols({{Bar, QuxSym}}))); /// Try a blocking lookup. @@ -307,7 +309,7 @@ TEST_F(CoreAPIsStandardTest, TestBasicReExports) { // JITDylib works. cantFail(JD.define(absoluteSymbols({{Foo, FooSym}}))); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(reexports(JD, {{Bar, {Foo, BarSym.getFlags()}}}))); @@ -332,7 +334,7 @@ TEST_F(CoreAPIsStandardTest, TestThatReExportsDontUnnecessarilyMaterialize) { cantFail(JD.define(BarMU)); - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(reexports( JD, {{Baz, {Foo, BazSym.getFlags()}}, {Qux, {Bar, QuxSym.getFlags()}}}))); @@ -347,7 +349,7 @@ TEST_F(CoreAPIsStandardTest, TestThatReExportsDontUnnecessarilyMaterialize) { TEST_F(CoreAPIsStandardTest, TestReexportsGenerator) { // Test that a re-exports generator can dynamically generate reexports. - auto &JD2 = ES.createJITDylib("JD2"); + auto &JD2 = ES.createBareJITDylib("JD2"); cantFail(JD2.define(absoluteSymbols({{Foo, FooSym}, {Bar, BarSym}}))); auto Filter = [this](SymbolStringPtr Name) { return Name != Bar; }; @@ -838,6 +840,7 @@ TEST_F(CoreAPIsStandardTest, DropMaterializerWhenEmpty) { [](MaterializationResponsibility R) { llvm_unreachable("Unexpected call to materialize"); }, + nullptr, [&](const JITDylib &JD, SymbolStringPtr Name) { EXPECT_TRUE(Name == Foo || Name == Bar) << "Discard of unexpected symbol?"; @@ -872,6 +875,7 @@ TEST_F(CoreAPIsStandardTest, AddAndMaterializeLazySymbol) { cantFail(R.notifyEmitted()); FooMaterialized = true; }, + nullptr, [&](const JITDylib &JD, SymbolStringPtr Name) { EXPECT_EQ(Name, Bar) << "Expected Name to be Bar"; BarDiscarded = true; @@ -920,6 +924,7 @@ TEST_F(CoreAPIsStandardTest, TestBasicWeakSymbolMaterialization) { ADD_FAILURE() << "Attempt to materialize Bar from the wrong unit"; R.failMaterialization(); }, + nullptr, [&](const JITDylib &JD, SymbolStringPtr Name) { EXPECT_EQ(Name, Bar) << "Expected \"Bar\" to be discarded"; DuplicateBarDiscarded = true; diff --git a/unittests/ExecutionEngine/Orc/OrcTestCommon.h b/unittests/ExecutionEngine/Orc/OrcTestCommon.h index d82012b5a85..b25851d8f79 100644 --- a/unittests/ExecutionEngine/Orc/OrcTestCommon.h +++ b/unittests/ExecutionEngine/Orc/OrcTestCommon.h @@ -47,7 +47,7 @@ class CoreAPIsBasedStandardTest : public testing::Test { protected: std::shared_ptr SSP = std::make_shared(); ExecutionSession ES{SSP}; - JITDylib &JD = ES.createJITDylib("JD"); + JITDylib &JD = ES.createBareJITDylib("JD"); SymbolStringPtr Foo = ES.intern("foo"); SymbolStringPtr Bar = ES.intern("bar"); SymbolStringPtr Baz = ES.intern("baz"); @@ -93,9 +93,11 @@ public: SimpleMaterializationUnit( orc::SymbolFlagsMap SymbolFlags, MaterializeFunction Materialize, + orc::SymbolStringPtr InitSym = nullptr, DiscardFunction Discard = DiscardFunction(), DestructorFunction Destructor = DestructorFunction()) - : MaterializationUnit(std::move(SymbolFlags), orc::VModuleKey()), + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSym), + orc::VModuleKey()), Materialize(std::move(Materialize)), Discard(std::move(Discard)), Destructor(std::move(Destructor)) {} diff --git a/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp b/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp index 0c66841e9af..9a50571b196 100644 --- a/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp +++ b/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp @@ -50,7 +50,7 @@ static bool testSetProcessAllSections(std::unique_ptr Obj, bool DebugSectionSeen = false; ExecutionSession ES; - auto &JD = ES.createJITDylib("main"); + auto &JD = ES.createBareJITDylib("main"); auto Foo = ES.intern("foo"); RTDyldObjectLinkingLayer ObjLayer(ES, [&DebugSectionSeen]() { @@ -151,7 +151,7 @@ TEST(RTDyldObjectLinkingLayerTest, TestOverrideObjectFlags) { // Create a simple stack and set the override flags option. ExecutionSession ES; - auto &JD = ES.createJITDylib("main"); + auto &JD = ES.createBareJITDylib("main"); auto Foo = ES.intern("foo"); RTDyldObjectLinkingLayer ObjLayer( ES, []() { return std::make_unique(); }); @@ -218,7 +218,7 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) { // Create a simple stack and set the override flags option. ExecutionSession ES; - auto &JD = ES.createJITDylib("main"); + auto &JD = ES.createBareJITDylib("main"); auto Foo = ES.intern("foo"); RTDyldObjectLinkingLayer ObjLayer( ES, []() { return std::make_unique(); });