[dsymutil] Add reproducers to dsymutil

Add support for generating a dsymutil reproducer. The result is a folder
containing all the object files for linking.

When --gen-reproducer is passed, dsymutil uses a FileCollectorFileSystem
which keeps track of all the files used by dsymutil. These files are
copied into a temporary directory when dsymutil exists.

When this path is passed to --use-reproducer, dsymutil uses a
RedirectingFileSystem that will use the files from the reproducer
directory instead of the actual paths. This means you don't need to mess
with the OSO path prefix.

Differential revision: https://reviews.llvm.org/D79398

docs/CommandGuide/dsymutil.rst

@@ -42,6 +42,10 @@ OPTIONS
  Produce a flat dSYM file. A ``.dwarf`` extension will be appended to the
  executable name unless the output file is specified using the ``-o`` option.
 
+.. option:: --gen-reproducer
+
+ Generate a reproducer consisting of the input object files.
+
 .. option:: --help, -h
 
  Print this help output.
@@ -131,6 +135,10 @@ OPTIONS
  other DWARF optimizations. This option will rebuild the '.apple_names' and
  '.apple_types' hashed accelerator tables.
 
+.. option:: --use-reproducer <path>
+
+ Use the object files from the given reproducer path.
+
 .. option:: --verbose
 
  Display verbose information when linking.

test/tools/dsymutil/X86/reproducer.test

@@ -0,0 +1,76 @@
+# Recreate the folder structure in a temp directory we can remove later.
+RUN: rm -rf %t
+RUN: mkdir -p %t/Inputs
+RUN: cp %p/../Inputs/basic.macho.x86_64 %t/Inputs
+RUN: cp %p/../Inputs/basic1.macho.x86_64.o %t/Inputs
+RUN: cp %p/../Inputs/basic2.macho.x86_64.o %t/Inputs
+RUN: cp %p/../Inputs/basic3.macho.x86_64.o %t/Inputs
+
+# Sanity check all the files are present.
+RUN: dsymutil -f -o - -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | llvm-dwarfdump -a - | FileCheck %s
+
+# Create a reproducer.
+RUN: env DSYMUTIL_REPRODUCER_PATH=%t.repro dsymutil -gen-reproducer -f -o %t.generate -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | FileCheck %s --check-prefixes=REPRODUCER
+RUN: llvm-dwarfdump -a %t.generate | FileCheck %s
+
+# Remove the input files and sanity check that was successful.
+RUN: rm -rf %t
+RUN: not dsymutil -f -o %t.error -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# Use the reproducer.
+RUN: dsymutil -use-reproducer %t.repro -f -o - -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | llvm-dwarfdump -a - | FileCheck %s
+
+# Conflicting options.
+RUN: not dsymutil -gen-reproducer -use-reproducer %t.repro -f -o %t.error -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 2>&1 |  FileCheck %s --check-prefix=CONFLICT
+
+CHECK: .debug_info
+CHECK:  DW_TAG_compile_unit
+CHECK-NEXT:    DW_AT_producer ("Apple LLVM version 6.0 (clang-600.0.39) (based on LLVM 3.5svn)")
+CHECK-NEXT:    DW_AT_language (DW_LANG_C99)
+CHECK-NEXT:    DW_AT_name ("basic1.c")
+CHECK-NEXT:    DW_AT_stmt_list (0x00000000)
+CHECK-NEXT:    DW_AT_comp_dir ("/Inputs")
+CHECK-NEXT:    DW_AT_low_pc (0x0000000100000ea0)
+CHECK:    DW_TAG_subprogram
+CHECK-NEXT:    DW_AT_name ("main")
+CHECK-NEXT:      DW_AT_decl_file ("/Inputs{{[/\\]}}basic1.c")
+CHECK-NEXT:      DW_AT_decl_line (23)
+CHECK-NEXT:      DW_AT_prototyped (0x01)
+CHECK-NEXT:      DW_AT_type (0x00000063
+CHECK-NEXT:      DW_AT_external (0x01)
+CHECK-NEXT:      DW_AT_accessibility (DW_ACCESS_public)
+CHECK-NEXT:      DW_AT_low_pc (0x0000000100000ea0)
+CHECK-NEXT:      DW_AT_high_pc (0x0000000100000ec4)
+CHECK-NEXT:      DW_AT_frame_base (DW_OP_reg6 RBP)
+CHECK:      DW_TAG_formal_parameter
+CHECK-NEXT:        DW_AT_name ("argc")
+CHECK-NEXT:        DW_AT_decl_file ("/Inputs{{[/\\]}}basic1.c")
+CHECK-NEXT:        DW_AT_decl_line (23)
+CHECK-NEXT:        DW_AT_type (0x00000063
+CHECK-NEXT:        DW_AT_location (DW_OP_fbreg -8)
+CHECK:      DW_TAG_formal_parameter
+CHECK-NEXT:        DW_AT_name ("argv")
+CHECK-NEXT:        DW_AT_decl_file ("/Inputs{{[/\\]}}basic1.c")
+CHECK-NEXT:        DW_AT_decl_line (23)
+CHECK-NEXT:        DW_AT_type (0x0000006a
+CHECK-NEXT:        DW_AT_location (DW_OP_fbreg -16)
+CHECK:      NULL
+CHECK:    DW_TAG_base_type
+CHECK-NEXT:      DW_AT_name ("int")
+CHECK-NEXT:      DW_AT_encoding (DW_ATE_signed)
+CHECK-NEXT:      DW_AT_byte_size (0x04)
+CHECK:    DW_TAG_pointer_type
+CHECK-NEXT:      DW_AT_type (0x0000006f
+CHECK:    DW_TAG_pointer_type
+CHECK-NEXT:      DW_AT_type (0x00000074
+CHECK:    DW_TAG_const_type
+CHECK-NEXT:      DW_AT_type (0x00000079
+CHECK:    DW_TAG_base_type
+CHECK-NEXT:      DW_AT_name ("char")
+CHECK-NEXT:      DW_AT_encoding (DW_ATE_signed_char)
+CHECK-NEXT:      DW_AT_byte_size (0x01)
+CHECK:      NULL
+
+REPRODUCER: reproducer written
+ERROR: error: cannot parse the debug map
+CONFLICT: cannot combine --gen-reproducer and --use-reproducer

test/tools/dsymutil/cmdline.test

@@ -9,6 +9,7 @@ CHECK: -accelerator
 CHECK: -arch <arch>
 CHECK: -dump-debug-map
 CHECK: -flat
+CHECK: -gen-reproducer
 CHECK: -help
 CHECK: -minimize
 CHECK: -no-odr
@@ -28,6 +29,7 @@ CHECK: -symtab
 CHECK: {{-S}}
 CHECK: -toolchain
 CHECK: -update
+CHECK: -use-reproducer <path>
 CHECK: -verbose
 CHECK: -verify
 CHECK: {{-y}}

tools/dsymutil/CMakeLists.txt

@@ -26,6 +26,7 @@ add_llvm_tool(dsymutil
   DwarfLinkerForBinary.cpp
   MachODebugMapParser.cpp
   MachOUtils.cpp
+  Reproducer.cpp
   SymbolMap.cpp
 
   DEPENDS

tools/dsymutil/Options.td

@@ -164,6 +164,16 @@ def: Separate<["-"], "j">,
   HelpText<"Alias for --num-threads">,
   Group<grp_general>;
 
+def gen_reproducer: F<"gen-reproducer">,
+  HelpText<"Generate a reproducer consisting of the input object files.">,
+  Group<grp_general>;
+
+def use_reproducer: Separate<["--", "-"], "use-reproducer">,
+  MetaVarName<"<path>">,
+  HelpText<"Use the object files from the given reproducer path.">,
+  Group<grp_general>;
+def: Joined<["--", "-"], "use-reproducer=">, Alias<use_reproducer>;
+
 def remarks_prepend_path: Separate<["--", "-"], "remarks-prepend-path">,
   MetaVarName<"<path>">,
   HelpText<"Specify a directory to prepend to the paths of the external remark files.">,

tools/dsymutil/Reproducer.cpp

@@ -0,0 +1,82 @@
+//===- Reproducer.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Path.h"
+#include <Reproducer.h>
+
+using namespace llvm;
+using namespace llvm::dsymutil;
+
+static std::string createReproducerDir(std::error_code &EC) {
+  SmallString<128> Root;
+  if (const char *Path = getenv("DSYMUTIL_REPRODUCER_PATH")) {
+    Root.assign(Path);
+    EC = sys::fs::create_directory(Root);
+  } else {
+    EC = sys::fs::createUniqueDirectory("dsymutil", Root);
+  }
+  return EC ? "" : std::string(Root);
+}
+
+Reproducer::Reproducer() : VFS(vfs::getRealFileSystem()) {}
+Reproducer::~Reproducer() = default;
+
+ReproducerGenerate::ReproducerGenerate(std::error_code &EC)
+    : Root(createReproducerDir(EC)), FC() {
+  if (!Root.empty())
+    FC = std::make_shared<FileCollector>(Root, Root);
+  VFS = FileCollector::createCollectorVFS(vfs::getRealFileSystem(), FC);
+}
+
+ReproducerGenerate::~ReproducerGenerate() {
+  if (!FC)
+    return;
+  FC->copyFiles(false);
+  SmallString<128> Mapping(Root);
+  sys::path::append(Mapping, "mapping.yaml");
+  FC->writeMapping(Mapping.str());
+  outs() << "reproducer written to " << Root << '\n';
+}
+
+ReproducerUse::~ReproducerUse() = default;
+
+ReproducerUse::ReproducerUse(StringRef Root, std::error_code &EC) {
+  SmallString<128> Mapping(Root);
+  sys::path::append(Mapping, "mapping.yaml");
+  ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
+      vfs::getRealFileSystem()->getBufferForFile(Mapping.str());
+
+  if (!Buffer) {
+    EC = Buffer.getError();
+    return;
+  }
+
+  VFS = llvm::vfs::getVFSFromYAML(std::move(Buffer.get()), nullptr, Mapping);
+}
+
+llvm::Expected<std::unique_ptr<Reproducer>>
+Reproducer::createReproducer(ReproducerMode Mode, StringRef Root) {
+  switch (Mode) {
+  case ReproducerMode::Generate: {
+    std::error_code EC;
+    auto Repro = std::make_unique<ReproducerGenerate>(EC);
+    if (EC)
+      return errorCodeToError(EC);
+    return Repro;
+  }
+  case ReproducerMode::Use: {
+    std::error_code EC;
+    auto Repro = std::make_unique<ReproducerUse>(Root, EC);
+    if (EC)
+      return errorCodeToError(EC);
+    return Repro;
+  }
+  case ReproducerMode::Off:
+    return std::make_unique<Reproducer>();
+  }
+}

tools/dsymutil/Reproducer.h

@@ -0,0 +1,77 @@
+//===- tools/dsymutil/Reproducer.h ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_DSYMUTIL_REPRODUCER_H
+#define LLVM_TOOLS_DSYMUTIL_REPRODUCER_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileCollector.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace llvm {
+namespace dsymutil {
+
+/// The reproducer mode.
+enum class ReproducerMode {
+  Generate,
+  Use,
+  Off,
+};
+
+/// The reproducer class manages the sate related to reproducers in dsymutil.
+/// Instances should be created with Reproducer::createReproducer. An instance
+/// of this class is returned when reproducers are off. The VFS returned by
+/// this instance is the real file system.
+class Reproducer {
+public:
+  Reproducer();
+  virtual ~Reproducer();
+
+  IntrusiveRefCntPtr<vfs::FileSystem> getVFS() const { return VFS; }
+
+  /// Create a Reproducer instance based on the given mode.
+  static llvm::Expected<std::unique_ptr<Reproducer>>
+  createReproducer(ReproducerMode Mode, StringRef Root);
+
+protected:
+  IntrusiveRefCntPtr<vfs::FileSystem> VFS;
+};
+
+/// Reproducer instance used to generate a new reproducer. The VFS returned by
+/// this instance is a FileCollectorFileSystem that tracks every file used by
+/// dsymutil.
+class ReproducerGenerate : public Reproducer {
+public:
+  ReproducerGenerate(std::error_code &EC);
+  ~ReproducerGenerate() override;
+
+private:
+  /// The path to the reproducer.
+  std::string Root;
+
+  /// The FileCollector used by the FileCollectorFileSystem.
+  std::shared_ptr<FileCollector> FC;
+};
+
+/// Reproducer instance used to use an existing reproducer. The VFS returned by
+/// this instance is a RedirectingFileSystem that remaps paths to their
+/// counterpart in the reproducer.
+class ReproducerUse : public Reproducer {
+public:
+  ReproducerUse(StringRef Root, std::error_code &EC);
+  ~ReproducerUse() override;
+
+private:
+  /// The path to the reproducer.
+  std::string Root;
+};
+
+} // end namespace dsymutil
+} // end namespace llvm
+
+#endif // LLVM_TOOLS_DSYMUTIL_REPRODUCER_H

tools/dsymutil/dsymutil.cpp

@@ -16,6 +16,7 @@
 #include "DebugMap.h"
 #include "LinkUtils.h"
 #include "MachOUtils.h"
+#include "Reproducer.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -31,6 +32,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileCollector.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/ManagedStatic.h"
@@ -92,9 +94,11 @@ struct DsymutilOptions {
   std::string SymbolMap;
   std::string OutputFile;
   std::string Toolchain;
+  std::string ReproducerPath;
   std::vector<std::string> Archs;
   std::vector<std::string> InputFiles;
   unsigned NumThreads;
+  ReproducerMode ReproMode = ReproducerMode::Off;
   dsymutil::LinkOptions LinkOpts;
 };
 
@@ -182,6 +186,12 @@ static Error verifyOptions(const DsymutilOptions &Options) {
         "paper trail warnings are not supported for YAML input.",
         errc::invalid_argument);
 
+  if (!Options.ReproducerPath.empty() &&
+      Options.ReproMode != ReproducerMode::Use)
+    return make_error<StringError>(
+        "cannot combine --gen-reproducer and --use-reproducer.",
+        errc::invalid_argument);
+
   return Error::success();
 }
 
@@ -222,6 +232,14 @@ static Expected<DsymutilOptions> getOptions(opt::InputArgList &Args) {
   Options.LinkOpts.Verbose = Args.hasArg(OPT_verbose);
   Options.LinkOpts.Statistics = Args.hasArg(OPT_statistics);
 
+  if (opt::Arg *ReproducerPath = Args.getLastArg(OPT_use_reproducer)) {
+    Options.ReproMode = ReproducerMode::Use;
+    Options.ReproducerPath = ReproducerPath->getValue();
+  }
+
+  if (Args.hasArg(OPT_gen_reproducer))
+    Options.ReproMode = ReproducerMode::Generate;
+
   if (Expected<AccelTableKind> AccelKind = getAccelTableKind(Args)) {
     Options.LinkOpts.TheAccelTableKind = *AccelKind;
   } else {
@@ -499,6 +517,15 @@ int main(int argc, char **argv) {
   InitializeAllTargets();
   InitializeAllAsmPrinters();
 
+  auto Repro =
+      Reproducer::createReproducer(Options.ReproMode, Options.ReproducerPath);
+  if (!Repro) {
+    WithColor::error() << toString(Repro.takeError());
+    return 1;
+  }
+
+  Options.LinkOpts.VFS = (*Repro)->getVFS();
+
   for (const auto &Arch : Options.Archs)
     if (Arch != "*" && Arch != "all" &&
         !object::MachOObjectFile::isValidArch(Arch)) {