[ORC] Add a utility to support dumping JIT'd objects to disk for debugging.

Adds a DumpObjects utility that can be used to dump JIT'd objects to disk.
Instances of DebugObjects may be used by ObjectTransformLayer as no-op
transforms.

This patch also adds an ObjectTransformLayer to LLJIT and an example of how
to use this utility to dump JIT'd objects in LLJIT.

examples/LLJITExamples/CMakeLists.txt

@@ -1,2 +1,3 @@
+add_subdirectory(LLJITDumpObjects)
 add_subdirectory(LLJITWithObjectCache)
 add_subdirectory(LLJITWithJITLink)

examples/LLJITExamples/LLJITDumpObjects/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(LLVM_LINK_COMPONENTS
+  Core
+  ExecutionEngine
+  IRReader
+  OrcJIT
+  Support
+  nativecodegen
+  )
+
+add_llvm_example(LLJITDumpObjects
+  LLJITDumpObjects.cpp
+  )

examples/LLJITExamples/LLJITDumpObjects/LLJITDumpObjects.cpp

@@ -0,0 +1,70 @@
+//===----- LLJITDumpObjects.cpp - How to dump JIT'd objects with LLJIT ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "../ExampleModules.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+
+ExitOnError ExitOnErr;
+
+cl::opt<bool> DumpJITdObjects("dump-jitted-objects",
+                              cl::desc("dump jitted objects"), cl::Optional,
+                              cl::init(true));
+
+cl::opt<std::string> DumpDir("dump-dir",
+                             cl::desc("directory to dump objects to"),
+                             cl::Optional, cl::init(""));
+
+cl::opt<std::string> DumpFileStem("dump-file-stem",
+                                  cl::desc("Override default dump names"),
+                                  cl::Optional, cl::init(""));
+
+int main(int argc, char *argv[]) {
+  // Initialize LLVM.
+  InitLLVM X(argc, argv);
+
+  InitializeNativeTarget();
+  InitializeNativeTargetAsmPrinter();
+
+  cl::ParseCommandLineOptions(argc, argv, "HowToUseLLJIT");
+  ExitOnErr.setBanner(std::string(argv[0]) + ": ");
+
+  outs()
+      << "Usage notes:\n"
+         "  Use -debug-only=orc on debug builds to see log messages of objects "
+         "being dumped\n"
+         "  Specify -dump-dir to specify a dump directory\n"
+         "  Specify -dump-file-stem to override the dump file stem\n"
+         "  Specify -dump-jitted-objects=false to disable dumping\n";
+
+  auto J = ExitOnErr(LLJITBuilder().create());
+
+  if (DumpJITdObjects)
+    J->getObjTransformLayer().setTransform(DumpObjects(DumpDir, DumpFileStem));
+
+  auto M = ExitOnErr(parseExampleModule(Add1Example, "add1"));
+
+  ExitOnErr(J->addIRModule(std::move(M)));
+
+  // Look up the JIT'd function, cast it to a function pointer, then call it.
+  auto Add1Sym = ExitOnErr(J->lookup("add1"));
+  int (*Add1)(int) = (int (*)(int))Add1Sym.getAddress();
+
+  int Result = Add1(42);
+  outs() << "add1(42) = " << Result << "\n";
+
+  return 0;
+}

include/llvm/ExecutionEngine/Orc/DebugUtils.h

@@ -0,0 +1,58 @@
+//===----- DebugUtils.h - Utilities for debugging ORC JITs ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for debugging ORC-based JITs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H
+#define LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H
+
+#include "llvm/Support/Error.h"
+#include <memory>
+#include <string>
+
+namespace llvm {
+
+class MemoryBuffer;
+
+namespace orc {
+
+/// A function object that can be used as an ObjectTransformLayer transform
+/// to dump object files to disk at a specified path.
+class DumpObjects {
+public:
+  /// Construct a DumpObjects transform that will dump objects to disk.
+  ///
+  /// @param DumpDir specifies the path to write dumped objects to. DumpDir may
+  /// be empty, in which case files will be dumped to the working directory. If
+  /// DumpDir is non-empty then any trailing separators will be discarded.
+  ///
+  /// @param IdentifierOverride specifies a file name stem to use when dumping
+  /// objects. If empty, each MemoryBuffer's identifier will be used (with a .o
+  /// suffix added if not already present). If an identifier override is
+  /// supplied it will be used instead (since all buffers will use the same
+  /// identifier, the resulting files will be named <ident>.o, <ident>.2.o,
+  /// <ident>.3.o, and so on). IdentifierOverride should not contain an
+  /// extension, as a .o suffix will be added by DumpObjects.
+  DumpObjects(std::string DumpDir = "", std::string IdentifierOverride = "");
+
+  /// Dumps the given buffer to disk.
+  Expected<std::unique_ptr<MemoryBuffer>>
+  operator()(std::unique_ptr<MemoryBuffer> Obj);
+
+private:
+  StringRef getBufferIdentifier(MemoryBuffer &B);
+  std::string DumpDir;
+  std::string IdentifierOverride;
+};
+
+} // End namespace orc
+} // End namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H

include/llvm/ExecutionEngine/Orc/LLJIT.h

@@ -117,6 +117,9 @@ public:
   /// Returns a reference to the ObjLinkingLayer
   ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; }
 
+  /// Returns a reference to the object transform layer.
+  ObjectTransformLayer &getObjTransformLayer() { return ObjTransformLayer; }
+
 protected:
   static std::unique_ptr<ObjectLayer>
   createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES);
@@ -140,6 +143,7 @@ protected:
   std::unique_ptr<ThreadPool> CompileThreads;
 
   std::unique_ptr<ObjectLayer> ObjLinkingLayer;
+  ObjectTransformLayer ObjTransformLayer;
   std::unique_ptr<IRCompileLayer> CompileLayer;
 
   CtorDtorRunner CtorRunner, DtorRunner;

include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h

@@ -29,11 +29,15 @@ public:
           std::unique_ptr<MemoryBuffer>)>;
 
   ObjectTransformLayer(ExecutionSession &ES, ObjectLayer &BaseLayer,
-                       TransformFunction Transform);
+                       TransformFunction Transform = TransformFunction());
 
   void emit(MaterializationResponsibility R,
             std::unique_ptr<MemoryBuffer> O) override;
 
+  void setTransform(TransformFunction Transform) {
+    this->Transform = std::move(Transform);
+  }
+
 private:
   ObjectLayer &BaseLayer;
   TransformFunction Transform;

lib/ExecutionEngine/Orc/CMakeLists.txt

@@ -2,6 +2,7 @@ add_llvm_library(LLVMOrcJIT
   CompileOnDemandLayer.cpp
   CompileUtils.cpp
   Core.cpp
+  DebugUtils.cpp
   ExecutionUtils.cpp
   IndirectionUtils.cpp
   IRCompileLayer.cpp

lib/ExecutionEngine/Orc/CompileUtils.cpp

@@ -43,8 +43,7 @@ SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) {
   }
 
   auto ObjBuffer = std::make_unique<SmallVectorMemoryBuffer>(
-      std::move(ObjBufferSV),
+      std::move(ObjBufferSV), M.getModuleIdentifier() + "-jitted-objectbuffer");
-      "<in memory object compiled from " + M.getModuleIdentifier() + ">");
 
   auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
 

lib/ExecutionEngine/Orc/DebugUtils.cpp

@@ -0,0 +1,68 @@
+//===---------- DebugUtils.cpp - Utilities for debugging ORC JITs ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm {
+namespace orc {
+
+DumpObjects::DumpObjects(std::string DumpDir, std::string IdentifierOverride)
+    : DumpDir(std::move(DumpDir)),
+      IdentifierOverride(std::move(IdentifierOverride)) {
+
+  /// Discard any trailing separators.
+  while (!this->DumpDir.empty() &&
+         sys::path::is_separator(this->DumpDir.back()))
+    this->DumpDir.pop_back();
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+DumpObjects::operator()(std::unique_ptr<MemoryBuffer> Obj) {
+  size_t Idx = 1;
+
+  std::string DumpPathStem;
+  raw_string_ostream(DumpPathStem)
+      << DumpDir << (DumpDir.empty() ? "" : "/") << getBufferIdentifier(*Obj);
+
+  std::string DumpPath = DumpPathStem + ".o";
+  while (sys::fs::exists(DumpPath)) {
+    DumpPath.clear();
+    raw_string_ostream(DumpPath) << DumpPathStem << "." << (++Idx) << ".o";
+  }
+
+  LLVM_DEBUG({
+    dbgs() << "Dumping object buffer [ " << (void *)Obj->getBufferStart()
+           << " -- " << (void *)(Obj->getBufferEnd() - 1) << " ] to "
+           << DumpPath << "\n";
+  });
+
+  std::error_code EC;
+  raw_fd_ostream DumpStream(DumpPath, EC);
+  if (EC)
+    return errorCodeToError(EC);
+  DumpStream.write(Obj->getBufferStart(), Obj->getBufferSize());
+
+  return Obj;
+}
+
+StringRef DumpObjects::getBufferIdentifier(MemoryBuffer &B) {
+  if (!IdentifierOverride.empty())
+    return IdentifierOverride;
+  StringRef Identifier = B.getBufferIdentifier();
+  Identifier.consume_back(".o");
+  return Identifier;
+}
+
+} // End namespace orc.
+} // End namespace llvm.

lib/ExecutionEngine/Orc/LLJIT.cpp

@@ -51,7 +51,7 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
   assert(Obj && "Can not add null object");
 
-  return ObjLinkingLayer->add(JD, std::move(Obj), ES->allocateVModule());
+  return ObjTransformLayer.add(JD, std::move(Obj), ES->allocateVModule());
 }
 
 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
@@ -103,13 +103,13 @@ LLJIT::createCompileFunction(LLJITBuilderState &S,
 
 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
     : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()),
-      Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main),
+      Main(this->ES->getMainJITDylib()), DL(""),
+      ObjLinkingLayer(createObjectLinkingLayer(S, *ES)),
+      ObjTransformLayer(*this->ES, *ObjLinkingLayer), CtorRunner(Main),
       DtorRunner(Main) {
 
   ErrorAsOutParameter _(&Err);
 
-  ObjLinkingLayer = createObjectLinkingLayer(S, *ES);
-
   if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
     DL = std::move(*DLOrErr);
   else {
@@ -124,7 +124,7 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
       return;
     }
     CompileLayer = std::make_unique<IRCompileLayer>(
-        *ES, *ObjLinkingLayer, std::move(*CompileFunction));
+        *ES, ObjTransformLayer, std::move(*CompileFunction));
   }
 
   if (S.NumCompileThreads > 0) {

lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp

@@ -21,12 +21,18 @@ void ObjectTransformLayer::emit(MaterializationResponsibility R,
                                 std::unique_ptr<MemoryBuffer> O) {
   assert(O && "Module must not be null");
 
-  if (auto TransformedObj = Transform(std::move(O)))
+  // If there is a transform set then apply it.
-    BaseLayer.emit(std::move(R), std::move(*TransformedObj));
+  if (Transform) {
-  else {
+    if (auto TransformedObj = Transform(std::move(O)))
-    R.failMaterialization();
+      O = std::move(*TransformedObj);
-    getExecutionSession().reportError(TransformedObj.takeError());
+    else {
+      R.failMaterialization();
+      getExecutionSession().reportError(TransformedObj.takeError());
+      return;
+    }
   }
+
+  BaseLayer.emit(std::move(R), std::move(O));
 }
 
 } // End namespace orc.