From 857b27372df9483b53c99d3c464c8f32645ab076 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Tue, 26 Jun 2018 21:35:48 +0000
Subject: [PATCH] [ORC] Add LLJIT and LLLazyJIT, and replace OrcLazyJIT in LLI
 with LLLazyJIT.

LLJIT is a prefabricated ORC based JIT class that is meant to be the go-to
replacement for MCJIT. Unlike OrcMCJITReplacement (which will continue to be
supported) it is not API or bug-for-bug compatible, but targets the same
use cases: Simple, non-lazy compilation and execution of LLVM IR.

LLLazyJIT extends LLJIT with support for function-at-a-time lazy compilation,
similar to what was provided by LLVM's original (now long deprecated) JIT APIs.

This commit also contains some simple utility classes (CtorDtorRunner2,
LocalCXXRuntimeOverrides2, JITTargetMachineBuilder) to support LLJIT and
LLLazyJIT.

Both of these classes are works in progress. Feedback from JIT clients is very
welcome!

llvm-svn: 335670
---
 include/llvm/ExecutionEngine/Orc/Core.h       |  21 +-
 .../llvm/ExecutionEngine/Orc/ExecutionUtils.h | 115 ++++++--
 include/llvm/ExecutionEngine/Orc/LLJIT.h      | 153 +++++++++++
 lib/ExecutionEngine/Orc/CMakeLists.txt        |   1 +
 .../Orc/CompileOnDemandLayer.cpp              |  56 +++-
 lib/ExecutionEngine/Orc/Core.cpp              |  35 ++-
 lib/ExecutionEngine/Orc/ExecutionUtils.cpp    | 150 ++++++++++-
 lib/ExecutionEngine/Orc/IndirectionUtils.cpp  |   2 -
 lib/ExecutionEngine/Orc/LLJIT.cpp             | 171 ++++++++++++
 lib/ExecutionEngine/Orc/Layer.cpp             |   6 +-
 test/ExecutionEngine/OrcLazy/hello.ll         |   2 +-
 tools/lli/CMakeLists.txt                      |   1 -
 tools/lli/OrcLazyJIT.cpp                      | 155 -----------
 tools/lli/OrcLazyJIT.h                        | 252 ------------------
 tools/lli/lli.cpp                             | 143 +++++++++-
 15 files changed, 809 insertions(+), 454 deletions(-)
 create mode 100644 include/llvm/ExecutionEngine/Orc/LLJIT.h
 create mode 100644 lib/ExecutionEngine/Orc/LLJIT.cpp
 delete mode 100644 tools/lli/OrcLazyJIT.cpp
 delete mode 100644 tools/lli/OrcLazyJIT.h

diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h
index 3ba210d55de..9d08b2ad584 100644
--- a/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/include/llvm/ExecutionEngine/Orc/Core.h
@@ -622,13 +622,6 @@ private:
   VSO(ExecutionSessionBase &ES, std::string Name)
       : ES(ES), VSOName(std::move(Name)) {}
 
-  ExecutionSessionBase &ES;
-  std::string VSOName;
-  SymbolMap Symbols;
-  UnmaterializedInfosMap UnmaterializedInfos;
-  MaterializingInfosMap MaterializingInfos;
-  FallbackDefinitionGeneratorFunction FallbackDefinitionGenerator;
-
   Error defineImpl(MaterializationUnit &MU);
 
   SymbolNameSet lookupFlagsImpl(SymbolFlagsMap &Flags,
@@ -660,6 +653,20 @@ private:
   void finalize(const SymbolFlagsMap &Finalized);
 
   void notifyFailed(const SymbolNameSet &FailedSymbols);
+
+  void runOutstandingMUs();
+
+  ExecutionSessionBase &ES;
+  std::string VSOName;
+  SymbolMap Symbols;
+  UnmaterializedInfosMap UnmaterializedInfos;
+  MaterializingInfosMap MaterializingInfos;
+  FallbackDefinitionGeneratorFunction FallbackDefinitionGenerator;
+
+  // FIXME: Remove this (and runOutstandingMUs) once the linking layer works
+  //        with callbacks from asynchronous queries.
+  mutable std::recursive_mutex OutstandingMUsMutex;
+  std::vector<std::unique_ptr<MaterializationUnit>> OutstandingMUs;
 };
 
 /// An ExecutionSession represents a running JIT program.
diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index 0268177f38d..e27f6e1e2cd 100644
--- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -20,6 +20,8 @@
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/OrcError.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Target/TargetOptions.h"
 #include <algorithm>
 #include <cstdint>
 #include <string>
@@ -32,10 +34,50 @@ class ConstantArray;
 class GlobalVariable;
 class Function;
 class Module;
+class TargetMachine;
 class Value;
 
 namespace orc {
 
+/// A utility class for building TargetMachines for JITs.
+class JITTargetMachineBuilder {
+public:
+  JITTargetMachineBuilder(Triple TT);
+  static Expected<JITTargetMachineBuilder> detectHost();
+  Expected<std::unique_ptr<TargetMachine>> createTargetMachine();
+
+  JITTargetMachineBuilder &setArch(std::string Arch) {
+    this->Arch = std::move(Arch);
+    return *this;
+  }
+  JITTargetMachineBuilder &setCPU(std::string CPU) {
+    this->CPU = std::move(CPU);
+    return *this;
+  }
+  JITTargetMachineBuilder &setRelocationModel(Optional<Reloc::Model> RM) {
+    this->RM = std::move(RM);
+    return *this;
+  }
+  JITTargetMachineBuilder &setCodeModel(Optional<CodeModel::Model> CM) {
+    this->CM = std::move(CM);
+    return *this;
+  }
+  JITTargetMachineBuilder &
+  addFeatures(const std::vector<std::string> &FeatureVec);
+  SubtargetFeatures &getFeatures() { return Features; }
+  TargetOptions &getOptions() { return Options; }
+
+private:
+  Triple TT;
+  std::string Arch;
+  std::string CPU;
+  SubtargetFeatures Features;
+  TargetOptions Options;
+  Optional<Reloc::Model> RM;
+  Optional<CodeModel::Model> CM;
+  CodeGenOpt::Level OptLevel = CodeGenOpt::Default;
+};
+
 /// This iterator provides a convenient way to iterate over the elements
 ///        of an llvm.global_ctors/llvm.global_dtors instance.
 ///
@@ -127,6 +169,20 @@ private:
   orc::VModuleKey K;
 };
 
+class CtorDtorRunner2 {
+public:
+  CtorDtorRunner2(VSO &V) : V(V) {}
+  void add(iterator_range<CtorDtorIterator> CtorDtors);
+  Error run();
+
+private:
+  using CtorDtorList = std::vector<SymbolStringPtr>;
+  using CtorDtorPriorityMap = std::map<unsigned, CtorDtorList>;
+
+  VSO &V;
+  CtorDtorPriorityMap CtorDtorsByPriority;
+};
+
 /// Support class for static dtor execution. For hosted (in-process) JITs
 ///        only!
 ///
@@ -142,7 +198,26 @@ private:
 /// the client determines that destructors should be run (generally at JIT
 /// teardown or after a return from main), the runDestructors method should be
 /// called.
-class LocalCXXRuntimeOverrides {
+class LocalCXXRuntimeOverridesBase {
+public:
+  /// Run any destructors recorded by the overriden __cxa_atexit function
+  /// (CXAAtExitOverride).
+  void runDestructors();
+
+protected:
+  template <typename PtrTy> JITTargetAddress toTargetAddress(PtrTy *P) {
+    return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(P));
+  }
+
+  using DestructorPtr = void (*)(void *);
+  using CXXDestructorDataPair = std::pair<DestructorPtr, void *>;
+  using CXXDestructorDataPairList = std::vector<CXXDestructorDataPair>;
+  CXXDestructorDataPairList DSOHandleOverride;
+  static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg,
+                               void *DSOHandle);
+};
+
+class LocalCXXRuntimeOverrides : public LocalCXXRuntimeOverridesBase {
 public:
   /// Create a runtime-overrides class.
   template <typename MangleFtorT>
@@ -159,32 +234,38 @@ public:
     return nullptr;
   }
 
-  /// Run any destructors recorded by the overriden __cxa_atexit function
-  /// (CXAAtExitOverride).
-  void runDestructors();
-
 private:
-  template <typename PtrTy>
-  JITTargetAddress toTargetAddress(PtrTy* P) {
-    return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(P));
-  }
-
   void addOverride(const std::string &Name, JITTargetAddress Addr) {
     CXXRuntimeOverrides.insert(std::make_pair(Name, Addr));
   }
 
   StringMap<JITTargetAddress> CXXRuntimeOverrides;
+};
 
-  using DestructorPtr = void (*)(void *);
-  using CXXDestructorDataPair = std::pair<DestructorPtr, void *>;
-  using CXXDestructorDataPairList = std::vector<CXXDestructorDataPair>;
-  CXXDestructorDataPairList DSOHandleOverride;
-  static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg,
-                               void *DSOHandle);
+class LocalCXXRuntimeOverrides2 : public LocalCXXRuntimeOverridesBase {
+public:
+  Error enable(VSO &V, MangleAndInterner &Mangler);
+};
+
+/// A utility class to expose symbols found via dlsym to the JIT.
+///
+/// If an instance of this class is attached to a VSO as a fallback definition
+/// generator, then any symbol found in the given DynamicLibrary that passes
+/// the 'Allow' predicate will be added to the VSO.
+class DynamicLibraryFallbackGenerator {
+public:
+  using SymbolPredicate = std::function<bool(SymbolStringPtr)>;
+  DynamicLibraryFallbackGenerator(sys::DynamicLibrary Dylib,
+                                  const DataLayout &DL, SymbolPredicate Allow);
+  SymbolNameSet operator()(VSO &V, const SymbolNameSet &Names);
+
+private:
+  sys::DynamicLibrary Dylib;
+  SymbolPredicate Allow;
+  char GlobalPrefix;
 };
 
 } // end namespace orc
-
 } // end namespace llvm
 
 #endif // LLVM_EXECUTIONENGINE_ORC_EXECUTIONUTILS_H
diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h
new file mode 100644
index 00000000000..8d764c3eab1
--- /dev/null
+++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -0,0 +1,153 @@
+//===----- LLJIT.h -- An ORC-based JIT for compiling LLVM IR ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for 3Bdetails.
+//
+//===----------------------------------------------------------------------===//
+//
+// An ORC-based JIT for compiling LLVM IR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_LLJIT_H
+#define LLVM_EXECUTIONENGINE_ORC_LLJIT_H
+
+#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
+#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
+#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
+#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
+#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+namespace orc {
+
+/// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT.
+class LLJIT {
+public:
+  /// Create an LLJIT instance.
+  static Expected<std::unique_ptr<LLJIT>>
+  Create(std::unique_ptr<ExecutionSession> ES,
+         std::unique_ptr<TargetMachine> TM, DataLayout DL);
+
+  /// Returns a reference to the ExecutionSession for this JIT instance.
+  ExecutionSession &getExecutionSession() { return *ES; }
+
+  /// Returns a reference to the VSO representing the JIT'd main program.
+  VSO &getMainVSO() { return Main; }
+
+  /// Convenience method for defining an absolute symbol.
+  Error defineAbsolute(StringRef Name, JITEvaluatedSymbol Address);
+
+  /// Adds an IR module to the given VSO.
+  Error addIRModule(VSO &V, std::unique_ptr<Module> M);
+
+  /// Adds an IR module to the Main VSO.
+  Error addIRModule(std::unique_ptr<Module> M) {
+    return addIRModule(Main, std::move(M));
+  }
+
+  /// Look up a symbol in VSO V by the symbol's linker-mangled name (to look up
+  /// symbols based on their IR name use the lookup function instead).
+  Expected<JITEvaluatedSymbol> lookupLinkerMangled(VSO &V, StringRef Name);
+
+  /// Look up a symbol in the main VSO by the symbol's linker-mangled name (to
+  /// look up symbols based on their IR name use the lookup function instead).
+  Expected<JITEvaluatedSymbol> lookupLinkerMangled(StringRef Name) {
+    return lookupLinkerMangled(Main, Name);
+  }
+
+  /// Look up a symbol in VSO V based on its IR symbol name.
+  Expected<JITEvaluatedSymbol> lookup(VSO &V, StringRef UnmangledName) {
+    return lookupLinkerMangled(V, mangle(UnmangledName));
+  }
+
+  /// Look up a symbol in the main VSO based on its IR symbol name.
+  Expected<JITEvaluatedSymbol> lookup(StringRef UnmangledName) {
+    return lookup(Main, UnmangledName);
+  }
+
+  /// Runs all not-yet-run static constructors.
+  Error runConstructors() { return CtorRunner.run(); }
+
+  /// Runs all not-yet-run static destructors.
+  Error runDestructors() { return DtorRunner.run(); }
+
+protected:
+  LLJIT(std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM,
+        DataLayout DL);
+
+  std::shared_ptr<SymbolResolver> takeSymbolResolver(VModuleKey K);
+  RTDyldObjectLinkingLayer2::Resources getRTDyldResources(VModuleKey K);
+
+  std::string mangle(StringRef UnmangledName);
+
+  std::unique_ptr<SymbolResolver> createResolverFor(VSO &V);
+
+  Error applyDataLayout(Module &M);
+
+  void recordCtorDtors(Module &M);
+
+  std::unique_ptr<ExecutionSession> ES;
+  VSO &Main;
+
+  std::unique_ptr<TargetMachine> TM;
+  DataLayout DL;
+
+  std::map<VSO *, VSOList> VSOLookupOrder;
+
+  RTDyldObjectLinkingLayer2 ObjLinkingLayer;
+  IRCompileLayer2 CompileLayer;
+
+  std::map<VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers;
+  CtorDtorRunner2 CtorRunner, DtorRunner;
+};
+
+/// An extended version of LLJIT that supports lazy function-at-a-time
+/// compilation of LLVM IR.
+class LLLazyJIT : public LLJIT {
+public:
+  /// Create an LLLazyJIT instance.
+  static Expected<std::unique_ptr<LLLazyJIT>>
+  Create(std::unique_ptr<ExecutionSession> ES,
+         std::unique_ptr<TargetMachine> TM, DataLayout DL, LLVMContext &Ctx);
+
+  /// Set an IR transform (e.g. pass manager pipeline) to run on each function
+  /// when it is compiled.
+  void setLazyCompileTransform(IRTransformLayer2::TransformFunction Transform) {
+    TransformLayer.setTransform(std::move(Transform));
+  }
+
+  /// Add a module to be lazily compiled to VSO V.
+  Error addLazyIRModule(VSO &V, std::unique_ptr<Module> M);
+
+  /// Add a module to be lazily compiled to the main VSO.
+  Error addLazyIRModule(std::unique_ptr<Module> M) {
+    return addLazyIRModule(Main, std::move(M));
+  }
+
+private:
+  LLLazyJIT(std::unique_ptr<ExecutionSession> ES,
+            std::unique_ptr<TargetMachine> TM, DataLayout DL, LLVMContext &Ctx,
+            std::unique_ptr<JITCompileCallbackManager> CCMgr,
+            std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder);
+
+  std::shared_ptr<SymbolResolver> getSymbolResolver(VModuleKey K);
+
+  void setSymbolResolver(VModuleKey K, std::shared_ptr<SymbolResolver> R);
+
+  std::unique_ptr<JITCompileCallbackManager> CCMgr;
+  std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder;
+
+  IRTransformLayer2 TransformLayer;
+  CompileOnDemandLayer2 CODLayer;
+};
+
+} // End namespace orc
+} // End namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_LLJIT_H
diff --git a/lib/ExecutionEngine/Orc/CMakeLists.txt b/lib/ExecutionEngine/Orc/CMakeLists.txt
index 9af5f211e90..2725fdfd14a 100644
--- a/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -7,6 +7,7 @@ add_llvm_library(LLVMOrcJIT
   IRTransformLayer.cpp
   Legacy.cpp
   Layer.cpp
+  LLJIT.cpp
   NullResolver.cpp
   ObjectTransformLayer.cpp
   OrcABISupport.cpp
diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
index 584f990976a..27a4ad68f59 100644
--- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
+++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
@@ -35,12 +35,42 @@ createLambdaValueMaterializer(MaterializerFtor M) {
 }
 } // namespace
 
+static void extractAliases(MaterializationResponsibility &R, Module &M,
+                           MangleAndInterner &Mangle) {
+  SymbolAliasMap Aliases;
+
+  std::vector<GlobalAlias *> ModAliases;
+  for (auto &A : M.aliases())
+    ModAliases.push_back(&A);
+
+  for (auto *A : ModAliases) {
+    Constant *Aliasee = A->getAliasee();
+    assert(A->hasName() && "Anonymous alias?");
+    assert(Aliasee->hasName() && "Anonymous aliasee");
+    std::string AliasName = A->getName();
+
+    Aliases[Mangle(AliasName)] = SymbolAliasMapEntry(
+        {Mangle(Aliasee->getName()), JITSymbolFlags::fromGlobalValue(*A)});
+
+    if (isa<Function>(Aliasee)) {
+      auto *F = cloneFunctionDecl(M, *cast<Function>(Aliasee));
+      A->replaceAllUsesWith(F);
+      A->eraseFromParent();
+      F->setName(AliasName);
+    } else if (isa<GlobalValue>(Aliasee)) {
+      auto *G = cloneGlobalVariableDecl(M, *cast<GlobalVariable>(Aliasee));
+      A->replaceAllUsesWith(G);
+      A->eraseFromParent();
+      G->setName(AliasName);
+    }
+  }
+
+  R.delegate(symbolAliases(std::move(Aliases)));
+}
+
 static std::unique_ptr<Module> extractGlobals(Module &M) {
   // FIXME: Add alias support.
 
-  if (M.global_empty() && M.alias_empty() && !M.getModuleFlagsMetadata())
-    return nullptr;
-
   auto GlobalsModule = llvm::make_unique<Module>(
       (M.getName() + ".globals").str(), M.getContext());
   GlobalsModule->setDataLayout(M.getDataLayout());
@@ -161,7 +191,6 @@ CompileOnDemandLayer2::CompileOnDemandLayer2(
 
 Error CompileOnDemandLayer2::add(VSO &V, VModuleKey K,
                                  std::unique_ptr<Module> M) {
-  makeAllSymbolsExternallyAccessible(*M);
   return IRLayer::add(V, K, std::move(M));
 }
 
@@ -174,10 +203,12 @@ void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K,
     if (GV.hasWeakLinkage())
       GV.setLinkage(GlobalValue::ExternalLinkage);
 
-  auto GlobalsModule = extractGlobals(*M);
-
   MangleAndInterner Mangle(ES, M->getDataLayout());
 
+  extractAliases(R, *M, Mangle);
+
+  auto GlobalsModule = extractGlobals(*M);
+
   // Delete the bodies of any available externally functions, rename the
   // rest, and build the compile callbacks.
   std::map<SymbolStringPtr, std::pair<JITTargetAddress, JITSymbolFlags>>
@@ -194,8 +225,12 @@ void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K,
     }
 
     assert(F.hasName() && "Function should have a name");
-    auto StubName = Mangle(F.getName());
+    std::string StubUnmangledName = F.getName();
     F.setName(F.getName() + "$body");
+    auto StubDecl = cloneFunctionDecl(*M, F);
+    StubDecl->setName(StubUnmangledName);
+    F.replaceAllUsesWith(StubDecl);
+    auto StubName = Mangle(StubUnmangledName);
     auto BodyName = Mangle(F.getName());
     if (auto CallbackAddr = CCMgr.getCompileCallback(
             [BodyName, &TargetVSO, &ES]() -> JITTargetAddress {
@@ -223,14 +258,19 @@ void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K,
     StubInits[*KV.first] = KV.second;
 
   // Build the function-body-extracting materialization unit.
+  auto SR = GetSymbolResolver(K);
   if (auto Err = R.getTargetVSO().define(
           llvm::make_unique<ExtractingIRMaterializationUnit>(
-              ES, *this, std::move(M), GetSymbolResolver(K)))) {
+              ES, *this, std::move(M), SR))) {
     ES.reportError(std::move(Err));
     R.failMaterialization();
     return;
   }
 
+  // Replace the fallback symbol resolver: We will re-use M's VModuleKey for
+  // the GlobalsModule.
+  SetSymbolResolver(K, SR);
+
   // Build the stubs.
   // FIXME: Remove function bodies materialization unit if stub creation fails.
   auto &StubsMgr = getStubsManager(TargetVSO);
diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp
index 53e4a853b35..717f2f085fe 100644
--- a/lib/ExecutionEngine/Orc/Core.cpp
+++ b/lib/ExecutionEngine/Orc/Core.cpp
@@ -725,6 +725,25 @@ void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) {
     Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols));
 }
 
+void VSO::runOutstandingMUs() {
+  while (1) {
+    std::unique_ptr<MaterializationUnit> MU;
+
+    {
+      std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
+      if (!OutstandingMUs.empty()) {
+        MU = std::move(OutstandingMUs.back());
+        OutstandingMUs.pop_back();
+      }
+    }
+
+    if (MU)
+      ES.dispatchMaterialization(*this, std::move(MU));
+    else
+      break;
+  }
+}
+
 SymbolNameSet VSO::lookupFlags(SymbolFlagsMap &Flags,
                                const SymbolNameSet &Names) {
   return ES.runSessionLocked([&, this]() {
@@ -767,6 +786,8 @@ SymbolNameSet VSO::lookup(std::shared_ptr<AsynchronousSymbolQuery> Q,
                           SymbolNameSet Names) {
   assert(Q && "Query can not be null");
 
+  runOutstandingMUs();
+
   LookupImplActionFlags ActionFlags = None;
   std::vector<std::unique_ptr<MaterializationUnit>> MUs;
 
@@ -796,9 +817,19 @@ SymbolNameSet VSO::lookup(std::shared_ptr<AsynchronousSymbolQuery> Q,
   if (ActionFlags & NotifyFullyReady)
     Q->handleFullyReady();
 
+  // FIXME: Swap back to the old code below once RuntimeDyld works with
+  //        callbacks from asynchronous queries.
+  // Add MUs to the OutstandingMUs list.
+  {
+    std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
+    for (auto &MU : MUs)
+      OutstandingMUs.push_back(std::move(MU));
+  }
+  runOutstandingMUs();
+
   // Dispatch any required MaterializationUnits for materialization.
-  for (auto &MU : MUs)
-    ES.dispatchMaterialization(*this, std::move(MU));
+  // for (auto &MU : MUs)
+  //  ES.dispatchMaterialization(*this, std::move(MU));
 
   return Unresolved;
 }
diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index f6e6ed66ef9..1389c2c723e 100644
--- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -13,10 +13,51 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Target/TargetMachine.h"
 
 namespace llvm {
 namespace orc {
 
+JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT)
+    : TT(std::move(TT)) {}
+
+Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() {
+  return JITTargetMachineBuilder(Triple(sys::getProcessTriple()));
+}
+
+Expected<std::unique_ptr<TargetMachine>>
+JITTargetMachineBuilder::createTargetMachine() {
+  if (!Arch.empty()) {
+    Triple::ArchType Type = Triple::getArchTypeForLLVMName(Arch);
+
+    if (Type == Triple::UnknownArch)
+      return make_error<StringError>(std::string("Unknown arch: ") + Arch,
+                                     inconvertibleErrorCode());
+  }
+
+  std::string ErrMsg;
+  auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg);
+  if (!TheTarget)
+    return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
+
+  auto *TM =
+      TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(),
+                                     Options, RM, CM, OptLevel, /*JIT*/ true);
+  if (!TM)
+    return make_error<StringError>("Could not allocate target machine",
+                                   inconvertibleErrorCode());
+
+  return std::unique_ptr<TargetMachine>(TM);
+}
+
+JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures(
+    const std::vector<std::string> &FeatureVec) {
+  for (const auto &F : FeatureVec)
+    Features.AddFeature(F);
+  return *this;
+}
+
 CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End)
   : InitList(
       GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr),
@@ -68,6 +109,8 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const {
 
   ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
   Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr;
+  if (!isa<GlobalValue>(Data))
+    Data = nullptr;
   return Element(Priority->getZExtValue(), Func, Data);
 }
 
@@ -83,20 +126,121 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) {
                     CtorDtorIterator(DtorsList, true));
 }
 
-void LocalCXXRuntimeOverrides::runDestructors() {
+void CtorDtorRunner2::add(iterator_range<CtorDtorIterator> CtorDtors) {
+  if (CtorDtors.begin() == CtorDtors.end())
+    return;
+
+  MangleAndInterner Mangle(
+      V.getExecutionSession(),
+      (*CtorDtors.begin()).Func->getParent()->getDataLayout());
+
+  for (const auto &CtorDtor : CtorDtors) {
+    assert(CtorDtor.Func && CtorDtor.Func->hasName() &&
+           "Ctor/Dtor function must be named to be runnable under the JIT");
+
+    if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) {
+      dbgs() << "  Skipping because why now?\n";
+      continue;
+    }
+
+    CtorDtorsByPriority[CtorDtor.Priority].push_back(
+        Mangle(CtorDtor.Func->getName()));
+  }
+}
+
+Error CtorDtorRunner2::run() {
+  using CtorDtorTy = void (*)();
+
+  SymbolNameSet Names;
+
+  for (auto &KV : CtorDtorsByPriority) {
+    for (auto &Name : KV.second) {
+      auto Added = Names.insert(Name).second;
+      (void)Added;
+      assert(Added && "Ctor/Dtor names clashed");
+    }
+  }
+
+  if (auto CtorDtorMap = lookup({&V}, std::move(Names))) {
+    for (auto &KV : CtorDtorsByPriority) {
+      for (auto &Name : KV.second) {
+        assert(CtorDtorMap->count(Name) && "No entry for Name");
+        auto CtorDtor = reinterpret_cast<CtorDtorTy>(
+            static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress()));
+        CtorDtor();
+      }
+    }
+    return Error::success();
+  } else
+    return CtorDtorMap.takeError();
+
+  CtorDtorsByPriority.clear();
+}
+
+void LocalCXXRuntimeOverridesBase::runDestructors() {
   auto& CXXDestructorDataPairs = DSOHandleOverride;
   for (auto &P : CXXDestructorDataPairs)
     P.first(P.second);
   CXXDestructorDataPairs.clear();
 }
 
-int LocalCXXRuntimeOverrides::CXAAtExitOverride(DestructorPtr Destructor,
-                                                void *Arg, void *DSOHandle) {
+int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor,
+                                                    void *Arg,
+                                                    void *DSOHandle) {
   auto& CXXDestructorDataPairs =
     *reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle);
   CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg));
   return 0;
 }
 
+Error LocalCXXRuntimeOverrides2::enable(VSO &V, MangleAndInterner &Mangle) {
+  SymbolMap RuntimeInterposes(
+      {{Mangle("__dso_handle"),
+        JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride),
+                           JITSymbolFlags::Exported)},
+       {Mangle("__cxa_atexit"),
+        JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride),
+                           JITSymbolFlags::Exported)}});
+
+  return V.define(absoluteSymbols(std::move(RuntimeInterposes)));
+}
+
+DynamicLibraryFallbackGenerator::DynamicLibraryFallbackGenerator(
+    sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow)
+    : Dylib(std::move(Dylib)), Allow(std::move(Allow)),
+      GlobalPrefix(DL.getGlobalPrefix()) {}
+
+SymbolNameSet DynamicLibraryFallbackGenerator::
+operator()(VSO &V, const SymbolNameSet &Names) {
+  orc::SymbolNameSet Added;
+  orc::SymbolMap NewSymbols;
+
+  bool HasGlobalPrefix = (GlobalPrefix != '\0');
+
+  for (auto &Name : Names) {
+    if (!Allow(Name) || (*Name).empty())
+      continue;
+
+    if (HasGlobalPrefix && (*Name).front() != GlobalPrefix)
+      continue;
+
+    std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size());
+    if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) {
+      Added.insert(Name);
+      NewSymbols[Name] = JITEvaluatedSymbol(
+          static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)),
+          JITSymbolFlags::Exported);
+    }
+  }
+
+  // Add any new symbols to V. Since the fallback generator is only called for
+  // symbols that are not already defined, this will never trigger a duplicate
+  // definition error, so we can wrap this call in a 'cantFail'.
+  if (!NewSymbols.empty())
+    cantFail(V.define(absoluteSymbols(std::move(NewSymbols))));
+
+  return Added;
+}
+
 } // End namespace orc.
 } // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index c2dfaa1a5f6..dc9628f183e 100644
--- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -265,7 +265,6 @@ void makeAllSymbolsExternallyAccessible(Module &M) {
 
 Function* cloneFunctionDecl(Module &Dst, const Function &F,
                             ValueToValueMapTy *VMap) {
-  assert(F.getParent() != &Dst && "Can't copy decl over existing function.");
   Function *NewF =
     Function::Create(cast<FunctionType>(F.getValueType()),
                      F.getLinkage(), F.getName(), &Dst);
@@ -303,7 +302,6 @@ void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
 
 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
                                         ValueToValueMapTy *VMap) {
-  assert(GV.getParent() != &Dst && "Can't copy decl over existing global var.");
   GlobalVariable *NewGV = new GlobalVariable(
       Dst, GV.getValueType(), GV.isConstant(),
       GV.getLinkage(), nullptr, GV.getName(), nullptr,
diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp
new file mode 100644
index 00000000000..059fba23d57
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -0,0 +1,171 @@
+//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/OrcError.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/IR/Mangler.h"
+
+namespace llvm {
+namespace orc {
+
+Expected<std::unique_ptr<LLJIT>>
+LLJIT::Create(std::unique_ptr<ExecutionSession> ES,
+              std::unique_ptr<TargetMachine> TM, DataLayout DL) {
+  return std::unique_ptr<LLJIT>(
+      new LLJIT(std::move(ES), std::move(TM), std::move(DL)));
+}
+
+Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) {
+  auto InternedName = ES->getSymbolStringPool().intern(Name);
+  SymbolMap Symbols({{InternedName, Sym}});
+  return Main.define(absoluteSymbols(std::move(Symbols)));
+}
+
+Error LLJIT::addIRModule(VSO &V, std::unique_ptr<Module> M) {
+  assert(M && "Can not add null module");
+
+  if (auto Err = applyDataLayout(*M))
+    return Err;
+
+  auto K = ES->allocateVModule();
+  Resolvers[K] = createResolverFor(V);
+  return CompileLayer.add(V, K, std::move(M));
+}
+
+Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(VSO &V,
+                                                        StringRef Name) {
+  return llvm::orc::lookup({&V}, ES->getSymbolStringPool().intern(Name));
+}
+
+LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES,
+             std::unique_ptr<TargetMachine> TM, DataLayout DL)
+    : ES(std::move(ES)), Main(this->ES->createVSO("main")), TM(std::move(TM)),
+      DL(std::move(DL)),
+      ObjLinkingLayer(*this->ES,
+                      [this](VModuleKey K) { return getRTDyldResources(K); }),
+      CompileLayer(*this->ES, ObjLinkingLayer, SimpleCompiler(*this->TM)),
+      CtorRunner(Main), DtorRunner(Main) {
+  VSOLookupOrder[&Main] = VSOList({&Main});
+}
+
+std::shared_ptr<SymbolResolver> LLJIT::takeSymbolResolver(VModuleKey K) {
+  auto ResolverI = Resolvers.find(K);
+  assert(ResolverI != Resolvers.end() && "Missing resolver");
+  auto Resolver = std::move(ResolverI->second);
+  Resolvers.erase(ResolverI);
+  return Resolver;
+}
+
+RTDyldObjectLinkingLayer2::Resources LLJIT::getRTDyldResources(VModuleKey K) {
+  return orc::RTDyldObjectLinkingLayer2::Resources(
+      {llvm::make_unique<SectionMemoryManager>(), takeSymbolResolver(K)});
+}
+
+std::string LLJIT::mangle(StringRef UnmangledName) {
+  std::string MangledName;
+  {
+    raw_string_ostream MangledNameStream(MangledName);
+    Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
+  }
+  return MangledName;
+}
+
+std::unique_ptr<SymbolResolver> LLJIT::createResolverFor(VSO &V) {
+  return createSymbolResolver(
+      [&](SymbolFlagsMap &Flags, const SymbolNameSet &Symbols) {
+        return V.lookupFlags(Flags, Symbols);
+      },
+      [&, this](std::shared_ptr<AsynchronousSymbolQuery> Q,
+                SymbolNameSet Symbols) {
+        assert(VSOLookupOrder.count(&V) && "No VSO lookup order for V");
+        SymbolNameSet Unresolved = std::move(Symbols);
+        for (auto *LV : VSOLookupOrder[&V])
+          Unresolved = LV->lookup(Q, std::move(Unresolved));
+        return Unresolved;
+      });
+}
+
+Error LLJIT::applyDataLayout(Module &M) {
+  if (M.getDataLayout().isDefault())
+    M.setDataLayout(DL);
+
+  if (M.getDataLayout() != DL)
+    return make_error<StringError>(
+        "Added modules have incompatible data layouts",
+        inconvertibleErrorCode());
+
+  return Error::success();
+}
+
+void LLJIT::recordCtorDtors(Module &M) {
+  CtorRunner.add(getConstructors(M));
+  DtorRunner.add(getDestructors(M));
+}
+
+Expected<std::unique_ptr<LLLazyJIT>>
+LLLazyJIT::Create(std::unique_ptr<ExecutionSession> ES,
+                  std::unique_ptr<TargetMachine> TM, DataLayout DL,
+                  LLVMContext &Ctx) {
+  const Triple &TT = TM->getTargetTriple();
+
+  auto CCMgr = createLocalCompileCallbackManager(TT, *ES, 0);
+  if (!CCMgr)
+    return make_error<StringError>(
+        std::string("No callback manager available for ") + TT.str(),
+        inconvertibleErrorCode());
+
+  auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT);
+  if (!ISMBuilder)
+    return make_error<StringError>(
+        std::string("No indirect stubs manager builder for ") + TT.str(),
+        inconvertibleErrorCode());
+
+  return std::unique_ptr<LLLazyJIT>(
+      new LLLazyJIT(std::move(ES), std::move(TM), std::move(DL), Ctx,
+                    std::move(CCMgr), std::move(ISMBuilder)));
+}
+
+Error LLLazyJIT::addLazyIRModule(VSO &V, std::unique_ptr<Module> M) {
+  assert(M && "Can not add null module");
+
+  if (auto Err = applyDataLayout(*M))
+    return Err;
+
+  makeAllSymbolsExternallyAccessible(*M);
+
+  recordCtorDtors(*M);
+
+  auto K = ES->allocateVModule();
+  setSymbolResolver(K, createResolverFor(V));
+  return CODLayer.add(V, K, std::move(M));
+}
+
+LLLazyJIT::LLLazyJIT(
+    std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM,
+    DataLayout DL, LLVMContext &Ctx,
+    std::unique_ptr<JITCompileCallbackManager> CCMgr,
+    std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder)
+    : LLJIT(std::move(ES), std::move(TM), std::move(DL)),
+      CCMgr(std::move(CCMgr)), TransformLayer(*this->ES, CompileLayer),
+      CODLayer(*this->ES, TransformLayer, *this->CCMgr, std::move(ISMBuilder),
+               [this](VModuleKey K) { return takeSymbolResolver(K); },
+               [this](VModuleKey K, std::shared_ptr<SymbolResolver> R) {
+                 setSymbolResolver(K, std::move(R));
+               },
+               [&]() -> LLVMContext & { return Ctx; }) {}
+
+void LLLazyJIT::setSymbolResolver(VModuleKey K,
+                                  std::shared_ptr<SymbolResolver> R) {
+  assert(!Resolvers.count(K) && "Resolver already present for VModule K");
+  Resolvers[K] = std::move(R);
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp
index 76255f56fd4..b9da3b7fb8d 100644
--- a/lib/ExecutionEngine/Orc/Layer.cpp
+++ b/lib/ExecutionEngine/Orc/Layer.cpp
@@ -8,7 +8,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/Layer.h"
-#include "llvm/IR/Mangler.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/MemoryBuffer.h"
 
@@ -29,9 +28,8 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES,
 
   MangleAndInterner Mangle(ES, this->M->getDataLayout());
   for (auto &G : this->M->global_values()) {
-    if (G.hasName() && !G.isDeclaration() &&
-        !G.hasLocalLinkage() &&
-        !G.hasAvailableExternallyLinkage()) {
+    if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() &&
+        !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) {
       auto MangledName = Mangle(G.getName());
       SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G);
       SymbolToDefinition[MangledName] = &G;
diff --git a/test/ExecutionEngine/OrcLazy/hello.ll b/test/ExecutionEngine/OrcLazy/hello.ll
index c3cf0f346a1..86d9a9a4b31 100644
--- a/test/ExecutionEngine/OrcLazy/hello.ll
+++ b/test/ExecutionEngine/OrcLazy/hello.ll
@@ -1,7 +1,7 @@
 ; RUN: lli -jit-kind=orc-lazy -orc-lazy-debug=funcs-to-stdout %s | FileCheck %s
 ;
 ; CHECK: Hello
-; CHECK: [ {{.*}}main ]
+; CHECK: [ {{.*}}main{{.*}} ]
 ; CHECK: Goodbye
 
 %class.Foo = type { i8 }
diff --git a/tools/lli/CMakeLists.txt b/tools/lli/CMakeLists.txt
index f02e19313b7..778107b79b0 100644
--- a/tools/lli/CMakeLists.txt
+++ b/tools/lli/CMakeLists.txt
@@ -38,7 +38,6 @@ endif( LLVM_USE_INTEL_JITEVENTS )
 
 add_llvm_tool(lli
   lli.cpp
-  OrcLazyJIT.cpp
 
   DEPENDS
   intrinsics_gen
diff --git a/tools/lli/OrcLazyJIT.cpp b/tools/lli/OrcLazyJIT.cpp
deleted file mode 100644
index 31edabd6a29..00000000000
--- a/tools/lli/OrcLazyJIT.cpp
+++ /dev/null
@@ -1,155 +0,0 @@
-//===- OrcLazyJIT.cpp - Basic Orc-based JIT for lazy execution ------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "OrcLazyJIT.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/Support/CodeGen.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/FileSystem.h"
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <system_error>
-
-using namespace llvm;
-
-namespace {
-
-enum class DumpKind {
-  NoDump,
-  DumpFuncsToStdOut,
-  DumpModsToStdOut,
-  DumpModsToDisk
-};
-
-} // end anonymous namespace
-
-static cl::opt<DumpKind> OrcDumpKind(
-    "orc-lazy-debug", cl::desc("Debug dumping for the orc-lazy JIT."),
-    cl::init(DumpKind::NoDump),
-    cl::values(clEnumValN(DumpKind::NoDump, "no-dump", "Don't dump anything."),
-               clEnumValN(DumpKind::DumpFuncsToStdOut, "funcs-to-stdout",
-                          "Dump function names to stdout."),
-               clEnumValN(DumpKind::DumpModsToStdOut, "mods-to-stdout",
-                          "Dump modules to stdout."),
-               clEnumValN(DumpKind::DumpModsToDisk, "mods-to-disk",
-                          "Dump modules to the current "
-                          "working directory. (WARNING: "
-                          "will overwrite existing files).")),
-    cl::Hidden);
-
-static cl::opt<bool> OrcInlineStubs("orc-lazy-inline-stubs",
-                                    cl::desc("Try to inline stubs"),
-                                    cl::init(true), cl::Hidden);
-
-OrcLazyJIT::TransformFtor OrcLazyJIT::createDebugDumper() {
-  switch (OrcDumpKind) {
-  case DumpKind::NoDump:
-    return [](std::unique_ptr<Module> M) { return M; };
-
-  case DumpKind::DumpFuncsToStdOut:
-    return [](std::unique_ptr<Module> M) {
-      printf("[ ");
-
-      for (const auto &F : *M) {
-        if (F.isDeclaration())
-          continue;
-
-        if (F.hasName()) {
-          std::string Name(F.getName());
-          printf("%s ", Name.c_str());
-        } else
-          printf("<anon> ");
-      }
-
-      printf("]\n");
-      return M;
-    };
-
-  case DumpKind::DumpModsToStdOut:
-    return [](std::unique_ptr<Module> M) {
-      outs() << "----- Module Start -----\n"
-             << *M << "----- Module End -----\n";
-
-      return M;
-    };
-
-  case DumpKind::DumpModsToDisk:
-    return [](std::unique_ptr<Module> M) {
-      std::error_code EC;
-      raw_fd_ostream Out(M->getModuleIdentifier() + ".ll", EC, sys::fs::F_Text);
-      if (EC) {
-        errs() << "Couldn't open " << M->getModuleIdentifier()
-               << " for dumping.\nError:" << EC.message() << "\n";
-        exit(1);
-      }
-      Out << *M;
-      return M;
-    };
-  }
-  llvm_unreachable("Unknown DumpKind");
-}
-
-// Defined in lli.cpp.
-CodeGenOpt::Level getOptLevel();
-
-template <typename PtrTy>
-static PtrTy fromTargetAddress(JITTargetAddress Addr) {
-  return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
-}
-
-int llvm::runOrcLazyJIT(std::vector<std::unique_ptr<Module>> Ms,
-                        const std::vector<std::string> &Args) {
-  // Add the program's symbols into the JIT's search space.
-  if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr)) {
-    errs() << "Error loading program symbols.\n";
-    return 1;
-  }
-
-  // Grab a target machine and try to build a factory function for the
-  // target-specific Orc callback manager.
-  EngineBuilder EB;
-  EB.setOptLevel(getOptLevel());
-  auto TM = std::unique_ptr<TargetMachine>(EB.selectTarget());
-  Triple T(TM->getTargetTriple());
-
-  auto IndirectStubsMgrBuilder = orc::createLocalIndirectStubsManagerBuilder(T);
-
-  // If we couldn't build a stubs-manager-builder for this target then bail out.
-  if (!IndirectStubsMgrBuilder) {
-    errs() << "No indirect stubs manager available for target '"
-           << TM->getTargetTriple().str() << "'.\n";
-    return 1;
-  }
-
-  // Everything looks good. Build the JIT.
-  OrcLazyJIT J(std::move(TM), std::move(IndirectStubsMgrBuilder),
-               OrcInlineStubs);
-
-  // Add the module, look up main and run it.
-  for (auto &M : Ms)
-    cantFail(J.addModule(std::move(M)));
-
-  if (auto MainSym = J.findSymbol("main")) {
-    typedef int (*MainFnPtr)(int, const char*[]);
-    std::vector<const char *> ArgV;
-    for (auto &Arg : Args)
-      ArgV.push_back(Arg.c_str());
-    auto Main = fromTargetAddress<MainFnPtr>(cantFail(MainSym.getAddress()));
-    return Main(ArgV.size(), (const char**)ArgV.data());
-  } else if (auto Err = MainSym.takeError())
-    logAllUnhandledErrors(std::move(Err), llvm::errs(), "");
-  else
-    errs() << "Could not find main function.\n";
-
-  return 1;
-}
diff --git a/tools/lli/OrcLazyJIT.h b/tools/lli/OrcLazyJIT.h
deleted file mode 100644
index 33778cef74a..00000000000
--- a/tools/lli/OrcLazyJIT.h
+++ /dev/null
@@ -1,252 +0,0 @@
-//===- OrcLazyJIT.h - Basic Orc-based JIT for lazy execution ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Simple Orc-based JIT. Uses the compile-on-demand layer to break up and
-// lazily compile modules.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TOOLS_LLI_ORCLAZYJIT_H
-#define LLVM_TOOLS_LLI_ORCLAZYJIT_H
-
-#include "llvm/ADT/Optional.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/ExecutionEngine/JITSymbol.h"
-#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
-#include "llvm/ExecutionEngine/Orc/CompileUtils.h"
-#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
-#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
-#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
-#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
-#include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
-#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
-#include "llvm/ExecutionEngine/RTDyldMemoryManager.h"
-#include "llvm/ExecutionEngine/SectionMemoryManager.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/Mangler.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetMachine.h"
-#include <algorithm>
-#include <functional>
-#include <memory>
-#include <set>
-#include <string>
-#include <vector>
-
-namespace llvm {
-
-class OrcLazyJIT {
-public:
-
-  using CompileCallbackMgr = orc::JITCompileCallbackManager;
-  using ObjLayerT = orc::RTDyldObjectLinkingLayer;
-  using CompileLayerT = orc::IRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
-  using TransformFtor =
-      std::function<std::unique_ptr<Module>(std::unique_ptr<Module>)>;
-  using IRDumpLayerT = orc::IRTransformLayer<CompileLayerT, TransformFtor>;
-  using CODLayerT = orc::CompileOnDemandLayer<IRDumpLayerT, CompileCallbackMgr>;
-  using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
-
-  OrcLazyJIT(std::unique_ptr<TargetMachine> TM,
-             IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
-             bool InlineStubs)
-      : TM(std::move(TM)), DL(this->TM->createDataLayout()),
-        CCMgr(orc::createLocalCompileCallbackManager(
-            this->TM->getTargetTriple(), ES, 0)),
-        ObjectLayer(ES,
-                    [this](orc::VModuleKey K) {
-                      auto ResolverI = Resolvers.find(K);
-                      assert(ResolverI != Resolvers.end() &&
-                             "Missing resolver for module K");
-                      auto Resolver = std::move(ResolverI->second);
-                      Resolvers.erase(ResolverI);
-                      return ObjLayerT::Resources{
-                          std::make_shared<SectionMemoryManager>(),
-                          std::move(Resolver)};
-                    }),
-        CompileLayer(ObjectLayer, orc::SimpleCompiler(*this->TM)),
-        IRDumpLayer(CompileLayer, createDebugDumper()),
-        CODLayer(
-            ES, IRDumpLayer,
-            [&](orc::VModuleKey K) {
-              auto ResolverI = Resolvers.find(K);
-              assert(ResolverI != Resolvers.end() &&
-                     "Missing resolver for module K");
-              auto Resolver = std::move(ResolverI->second);
-              Resolvers.erase(ResolverI);
-              return Resolver;
-            },
-            [&](orc::VModuleKey K, std::shared_ptr<orc::SymbolResolver> R) {
-              assert(!Resolvers.count(K) && "Resolver already present");
-              Resolvers[K] = std::move(R);
-            },
-            extractSingleFunction, *this->CCMgr,
-            std::move(IndirectStubsMgrBuilder), InlineStubs),
-        CXXRuntimeOverrides(
-            [this](const std::string &S) { return mangle(S); }) {}
-
-  ~OrcLazyJIT() {
-    // Run any destructors registered with __cxa_atexit.
-    CXXRuntimeOverrides.runDestructors();
-    // Run any IR destructors.
-    for (auto &DtorRunner : IRStaticDestructorRunners)
-      if (auto Err = DtorRunner.runViaLayer(CODLayer)) {
-        // FIXME: OrcLazyJIT should probably take a "shutdownError" callback to
-        //        report these errors on.
-        report_fatal_error(std::move(Err));
-      }
-  }
-
-  Error addModule(std::unique_ptr<Module> M) {
-    if (M->getDataLayout().isDefault())
-      M->setDataLayout(DL);
-
-    // Rename, bump linkage and record static constructors and destructors.
-    // We have to do this before we hand over ownership of the module to the
-    // JIT.
-    std::vector<std::string> CtorNames, DtorNames;
-    {
-      unsigned CtorId = 0, DtorId = 0;
-      for (auto Ctor : orc::getConstructors(*M)) {
-        std::string NewCtorName = ("$static_ctor." + Twine(CtorId++)).str();
-        Ctor.Func->setName(NewCtorName);
-        Ctor.Func->setLinkage(GlobalValue::ExternalLinkage);
-        Ctor.Func->setVisibility(GlobalValue::HiddenVisibility);
-        CtorNames.push_back(mangle(NewCtorName));
-      }
-      for (auto Dtor : orc::getDestructors(*M)) {
-        std::string NewDtorName = ("$static_dtor." + Twine(DtorId++)).str();
-        Dtor.Func->setLinkage(GlobalValue::ExternalLinkage);
-        Dtor.Func->setVisibility(GlobalValue::HiddenVisibility);
-        DtorNames.push_back(mangle(Dtor.Func->getName()));
-        Dtor.Func->setName(NewDtorName);
-      }
-    }
-
-    // Symbol resolution order:
-    //   1) Search the JIT symbols.
-    //   2) Check for C++ runtime overrides.
-    //   3) Search the host process (LLI)'s symbol table.
-    if (!ModulesKey) {
-      auto LegacyLookupInDylib = [this](const std::string &Name) -> JITSymbol {
-        if (auto Sym = CODLayer.findSymbol(Name, true))
-          return Sym;
-        else if (auto Err = Sym.takeError())
-          return std::move(Err);
-        return CXXRuntimeOverrides.searchOverrides(Name);
-      };
-
-      auto LegacyLookup =
-          [LegacyLookupInDylib](const std::string &Name) -> JITSymbol {
-        if (auto Sym = LegacyLookupInDylib(Name))
-          return Sym;
-        else if (auto Err = Sym.takeError())
-          return std::move(Err);
-
-        if (auto Addr = RTDyldMemoryManager::getSymbolAddressInProcess(Name))
-          return JITSymbol(Addr, JITSymbolFlags::Exported);
-
-        return nullptr;
-      };
-
-      ModulesKey = ES.allocateVModule();
-      assert(!Resolvers.count(*ModulesKey) && "Resolver already present");
-      Resolvers[*ModulesKey] = orc::createSymbolResolver(
-          [LegacyLookupInDylib](orc::SymbolFlagsMap &SymbolFlags,
-                                const orc::SymbolNameSet &Symbols) {
-            auto NotFoundViaLegacyLookup = lookupFlagsWithLegacyFn(
-                SymbolFlags, Symbols, LegacyLookupInDylib);
-            if (!NotFoundViaLegacyLookup) {
-              logAllUnhandledErrors(NotFoundViaLegacyLookup.takeError(), errs(),
-                                    "OrcLazyJIT lookupFlags error: ");
-              SymbolFlags.clear();
-              return orc::SymbolNameSet();
-            }
-            return std::move(*NotFoundViaLegacyLookup);
-          },
-          [this,
-           LegacyLookup](std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
-                         orc::SymbolNameSet Symbols) {
-            return lookupWithLegacyFn(ES, *Query, Symbols, LegacyLookup);
-          });
-
-      // Add the module to the JIT.
-      if (auto Err = CODLayer.addModule(*ModulesKey, std::move(M)))
-        return Err;
-
-    } else if (auto Err = CODLayer.addExtraModule(*ModulesKey, std::move(M)))
-      return Err;
-
-    // Run the static constructors, and save the static destructor runner for
-    // execution when the JIT is torn down.
-    orc::CtorDtorRunner<CODLayerT> CtorRunner(std::move(CtorNames),
-                                              *ModulesKey);
-    if (auto Err = CtorRunner.runViaLayer(CODLayer))
-      return Err;
-
-    IRStaticDestructorRunners.emplace_back(std::move(DtorNames), *ModulesKey);
-
-    return Error::success();
-  }
-
-  JITSymbol findSymbol(const std::string &Name) {
-    return CODLayer.findSymbol(mangle(Name), true);
-  }
-
-  JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name) {
-    return CODLayer.findSymbolIn(K, mangle(Name), true);
-  }
-
-private:
-  std::string mangle(const std::string &Name) {
-    std::string MangledName;
-    {
-      raw_string_ostream MangledNameStream(MangledName);
-      Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
-    }
-    return MangledName;
-  }
-
-  static std::set<Function*> extractSingleFunction(Function &F) {
-    std::set<Function*> Partition;
-    Partition.insert(&F);
-    return Partition;
-  }
-
-  static TransformFtor createDebugDumper();
-
-  orc::SymbolStringPool SSP;
-  orc::ExecutionSession ES;
-
-  std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers;
-
-  std::unique_ptr<TargetMachine> TM;
-  DataLayout DL;
-  SectionMemoryManager CCMgrMemMgr;
-
-  std::unique_ptr<CompileCallbackMgr> CCMgr;
-  ObjLayerT ObjectLayer;
-  CompileLayerT CompileLayer;
-  IRDumpLayerT IRDumpLayer;
-  CODLayerT CODLayer;
-
-  orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides;
-  std::vector<orc::CtorDtorRunner<CODLayerT>> IRStaticDestructorRunners;
-  llvm::Optional<orc::VModuleKey> ModulesKey;
-};
-
-int runOrcLazyJIT(std::vector<std::unique_ptr<Module>> Ms,
-                  const std::vector<std::string> &Args);
-
-} // end namespace llvm
-
-#endif // LLVM_TOOLS_LLI_ORCLAZYJIT_H
diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp
index 07075c2496d..b7c821bb585 100644
--- a/tools/lli/lli.cpp
+++ b/tools/lli/lli.cpp
@@ -13,7 +13,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "OrcLazyJIT.h"
 #include "RemoteJITUtils.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Triple.h"
@@ -26,6 +25,8 @@
 #include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/MCJIT.h"
 #include "llvm/ExecutionEngine/ObjectCache.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h"
 #include "llvm/ExecutionEngine/OrcMCJITReplacement.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
@@ -177,6 +178,28 @@ namespace {
     cl::desc("Generate software floating point library calls"),
     cl::init(false));
 
+  enum class DumpKind {
+    NoDump,
+    DumpFuncsToStdOut,
+    DumpModsToStdOut,
+    DumpModsToDisk
+  };
+
+  cl::opt<DumpKind> OrcDumpKind(
+      "orc-lazy-debug", cl::desc("Debug dumping for the orc-lazy JIT."),
+      cl::init(DumpKind::NoDump),
+      cl::values(clEnumValN(DumpKind::NoDump, "no-dump",
+                            "Don't dump anything."),
+                 clEnumValN(DumpKind::DumpFuncsToStdOut, "funcs-to-stdout",
+                            "Dump function names to stdout."),
+                 clEnumValN(DumpKind::DumpModsToStdOut, "mods-to-stdout",
+                            "Dump modules to stdout."),
+                 clEnumValN(DumpKind::DumpModsToDisk, "mods-to-disk",
+                            "Dump modules to the current "
+                            "working directory. (WARNING: "
+                            "will overwrite existing files).")),
+      cl::Hidden);
+
   ExitOnError ExitOnErr;
 }
 
@@ -313,6 +336,9 @@ static void reportError(SMDiagnostic Err, const char *ProgName) {
   exit(1);
 }
 
+int runOrcLazyJIT(LLVMContext &Ctx, std::vector<std::unique_ptr<Module>> Ms,
+                  const std::vector<std::string> &Args);
+
 //===----------------------------------------------------------------------===//
 // main Driver function
 //
@@ -356,7 +382,7 @@ int main(int argc, char **argv, char * const *envp) {
     Args.push_back(InputFile);
     for (auto &Arg : InputArgv)
       Args.push_back(Arg);
-    return runOrcLazyJIT(std::move(Ms), Args);
+    return runOrcLazyJIT(Context, std::move(Ms), Args);
   }
 
   if (EnableCacheManager) {
@@ -658,6 +684,119 @@ int main(int argc, char **argv, char * const *envp) {
   return Result;
 }
 
+static orc::IRTransformLayer2::TransformFunction createDebugDumper() {
+  switch (OrcDumpKind) {
+  case DumpKind::NoDump:
+    return [](std::unique_ptr<Module> M) { return M; };
+
+  case DumpKind::DumpFuncsToStdOut:
+    return [](std::unique_ptr<Module> M) {
+      printf("[ ");
+
+      for (const auto &F : *M) {
+        if (F.isDeclaration())
+          continue;
+
+        if (F.hasName()) {
+          std::string Name(F.getName());
+          printf("%s ", Name.c_str());
+        } else
+          printf("<anon> ");
+      }
+
+      printf("]\n");
+      return M;
+    };
+
+  case DumpKind::DumpModsToStdOut:
+    return [](std::unique_ptr<Module> M) {
+      outs() << "----- Module Start -----\n"
+             << *M << "----- Module End -----\n";
+
+      return M;
+    };
+
+  case DumpKind::DumpModsToDisk:
+    return [](std::unique_ptr<Module> M) {
+      std::error_code EC;
+      raw_fd_ostream Out(M->getModuleIdentifier() + ".ll", EC, sys::fs::F_Text);
+      if (EC) {
+        errs() << "Couldn't open " << M->getModuleIdentifier()
+               << " for dumping.\nError:" << EC.message() << "\n";
+        exit(1);
+      }
+      Out << *M;
+      return M;
+    };
+  }
+  llvm_unreachable("Unknown DumpKind");
+}
+
+int runOrcLazyJIT(LLVMContext &Ctx, std::vector<std::unique_ptr<Module>> Ms,
+                  const std::vector<std::string> &Args) {
+  // Bail out early if no modules loaded.
+  if (Ms.empty())
+    return 0;
+
+  // Add lli's symbols into the JIT's search space.
+  std::string ErrMsg;
+  sys::DynamicLibrary LibLLI =
+      sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg);
+  if (!LibLLI.isValid()) {
+    errs() << "Error loading lli symbols: " << ErrMsg << ".\n";
+    return 1;
+  }
+
+  const auto &TT = Ms.front()->getTargetTriple();
+  orc::JITTargetMachineBuilder TMD =
+      TT.empty() ? ExitOnErr(orc::JITTargetMachineBuilder::detectHost())
+                 : orc::JITTargetMachineBuilder(Triple(TT));
+
+  TMD.setArch(MArch)
+      .setCPU(getCPUStr())
+      .addFeatures(getFeatureList())
+      .setRelocationModel(RelocModel.getNumOccurrences()
+                              ? Optional<Reloc::Model>(RelocModel)
+                              : None)
+      .setCodeModel(CMModel.getNumOccurrences()
+                        ? Optional<CodeModel::Model>(CMModel)
+                        : None);
+  auto TM = ExitOnErr(TMD.createTargetMachine());
+  auto DL = TM->createDataLayout();
+  auto ES = llvm::make_unique<orc::ExecutionSession>();
+  auto J =
+      ExitOnErr(orc::LLLazyJIT::Create(std::move(ES), std::move(TM), DL, Ctx));
+
+  J->setLazyCompileTransform(createDebugDumper());
+  J->getMainVSO().setFallbackDefinitionGenerator(
+      orc::DynamicLibraryFallbackGenerator(
+          std::move(LibLLI), DL, [](orc::SymbolStringPtr) { return true; }));
+
+  orc::MangleAndInterner Mangle(J->getExecutionSession(), DL);
+  orc::LocalCXXRuntimeOverrides2 CXXRuntimeOverrides;
+  ExitOnErr(CXXRuntimeOverrides.enable(J->getMainVSO(), Mangle));
+
+  for (auto &M : Ms)
+    ExitOnErr(J->addLazyIRModule(std::move(M)));
+
+  ExitOnErr(J->runConstructors());
+
+  auto MainSym = ExitOnErr(J->lookup("main"));
+  typedef int (*MainFnPtr)(int, const char *[]);
+  std::vector<const char *> ArgV;
+  for (auto &Arg : Args)
+    ArgV.push_back(Arg.c_str());
+  auto Main =
+      reinterpret_cast<MainFnPtr>(static_cast<uintptr_t>(MainSym.getAddress()));
+  auto Result = Main(ArgV.size(), (const char **)ArgV.data());
+
+  ExitOnErr(J->runDestructors());
+
+  CXXRuntimeOverrides.runDestructors();
+
+  return Result;
+}
+
 std::unique_ptr<FDRawChannel> launchRemote() {
 #ifndef LLVM_ON_UNIX
   llvm_unreachable("launchRemote not supported on non-Unix platforms");