From b03521b98d9048799bb3bb376ef581cd9ccb73b1 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 21 May 2020 10:57:53 -0700 Subject: [PATCH] [dsymutil] Add reproducers to dsymutil Add support for generating a dsymutil reproducer. The result is a folder containing all the object files for linking. When --gen-reproducer is passed, dsymutil uses a FileCollectorFileSystem which keeps track of all the files used by dsymutil. These files are copied into a temporary directory when dsymutil exists. When this path is passed to --use-reproducer, dsymutil uses a RedirectingFileSystem that will use the files from the reproducer directory instead of the actual paths. This means you don't need to mess with the OSO path prefix. Differential revision: https://reviews.llvm.org/D79398 --- docs/CommandGuide/dsymutil.rst | 8 +++ test/tools/dsymutil/X86/reproducer.test | 76 +++++++++++++++++++++++ test/tools/dsymutil/cmdline.test | 2 + tools/dsymutil/CMakeLists.txt | 1 + tools/dsymutil/Options.td | 10 +++ tools/dsymutil/Reproducer.cpp | 82 +++++++++++++++++++++++++ tools/dsymutil/Reproducer.h | 77 +++++++++++++++++++++++ tools/dsymutil/dsymutil.cpp | 27 ++++++++ 8 files changed, 283 insertions(+) create mode 100644 test/tools/dsymutil/X86/reproducer.test create mode 100644 tools/dsymutil/Reproducer.cpp create mode 100644 tools/dsymutil/Reproducer.h diff --git a/docs/CommandGuide/dsymutil.rst b/docs/CommandGuide/dsymutil.rst index 5b7016ced2f..78954fcc8d8 100644 --- a/docs/CommandGuide/dsymutil.rst +++ b/docs/CommandGuide/dsymutil.rst @@ -42,6 +42,10 @@ OPTIONS Produce a flat dSYM file. A ``.dwarf`` extension will be appended to the executable name unless the output file is specified using the ``-o`` option. +.. option:: --gen-reproducer + + Generate a reproducer consisting of the input object files. + .. option:: --help, -h Print this help output. @@ -131,6 +135,10 @@ OPTIONS other DWARF optimizations. This option will rebuild the '.apple_names' and '.apple_types' hashed accelerator tables. +.. option:: --use-reproducer + + Use the object files from the given reproducer path. + .. option:: --verbose Display verbose information when linking. diff --git a/test/tools/dsymutil/X86/reproducer.test b/test/tools/dsymutil/X86/reproducer.test new file mode 100644 index 00000000000..02d73b91e39 --- /dev/null +++ b/test/tools/dsymutil/X86/reproducer.test @@ -0,0 +1,76 @@ +# Recreate the folder structure in a temp directory we can remove later. +RUN: rm -rf %t +RUN: mkdir -p %t/Inputs +RUN: cp %p/../Inputs/basic.macho.x86_64 %t/Inputs +RUN: cp %p/../Inputs/basic1.macho.x86_64.o %t/Inputs +RUN: cp %p/../Inputs/basic2.macho.x86_64.o %t/Inputs +RUN: cp %p/../Inputs/basic3.macho.x86_64.o %t/Inputs + +# Sanity check all the files are present. +RUN: dsymutil -f -o - -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | llvm-dwarfdump -a - | FileCheck %s + +# Create a reproducer. +RUN: env DSYMUTIL_REPRODUCER_PATH=%t.repro dsymutil -gen-reproducer -f -o %t.generate -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | FileCheck %s --check-prefixes=REPRODUCER +RUN: llvm-dwarfdump -a %t.generate | FileCheck %s + +# Remove the input files and sanity check that was successful. +RUN: rm -rf %t +RUN: not dsymutil -f -o %t.error -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=ERROR + +# Use the reproducer. +RUN: dsymutil -use-reproducer %t.repro -f -o - -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 | llvm-dwarfdump -a - | FileCheck %s + +# Conflicting options. +RUN: not dsymutil -gen-reproducer -use-reproducer %t.repro -f -o %t.error -oso-prepend-path=%t %t/Inputs/basic.macho.x86_64 2>&1 | FileCheck %s --check-prefix=CONFLICT + +CHECK: .debug_info +CHECK: DW_TAG_compile_unit +CHECK-NEXT: DW_AT_producer ("Apple LLVM version 6.0 (clang-600.0.39) (based on LLVM 3.5svn)") +CHECK-NEXT: DW_AT_language (DW_LANG_C99) +CHECK-NEXT: DW_AT_name ("basic1.c") +CHECK-NEXT: DW_AT_stmt_list (0x00000000) +CHECK-NEXT: DW_AT_comp_dir ("/Inputs") +CHECK-NEXT: DW_AT_low_pc (0x0000000100000ea0) +CHECK: DW_TAG_subprogram +CHECK-NEXT: DW_AT_name ("main") +CHECK-NEXT: DW_AT_decl_file ("/Inputs{{[/\\]}}basic1.c") +CHECK-NEXT: DW_AT_decl_line (23) +CHECK-NEXT: DW_AT_prototyped (0x01) +CHECK-NEXT: DW_AT_type (0x00000063 +CHECK-NEXT: DW_AT_external (0x01) +CHECK-NEXT: DW_AT_accessibility (DW_ACCESS_public) +CHECK-NEXT: DW_AT_low_pc (0x0000000100000ea0) +CHECK-NEXT: DW_AT_high_pc (0x0000000100000ec4) +CHECK-NEXT: DW_AT_frame_base (DW_OP_reg6 RBP) +CHECK: DW_TAG_formal_parameter +CHECK-NEXT: DW_AT_name ("argc") +CHECK-NEXT: DW_AT_decl_file ("/Inputs{{[/\\]}}basic1.c") +CHECK-NEXT: DW_AT_decl_line (23) +CHECK-NEXT: DW_AT_type (0x00000063 +CHECK-NEXT: DW_AT_location (DW_OP_fbreg -8) +CHECK: DW_TAG_formal_parameter +CHECK-NEXT: DW_AT_name ("argv") +CHECK-NEXT: DW_AT_decl_file ("/Inputs{{[/\\]}}basic1.c") +CHECK-NEXT: DW_AT_decl_line (23) +CHECK-NEXT: DW_AT_type (0x0000006a +CHECK-NEXT: DW_AT_location (DW_OP_fbreg -16) +CHECK: NULL +CHECK: DW_TAG_base_type +CHECK-NEXT: DW_AT_name ("int") +CHECK-NEXT: DW_AT_encoding (DW_ATE_signed) +CHECK-NEXT: DW_AT_byte_size (0x04) +CHECK: DW_TAG_pointer_type +CHECK-NEXT: DW_AT_type (0x0000006f +CHECK: DW_TAG_pointer_type +CHECK-NEXT: DW_AT_type (0x00000074 +CHECK: DW_TAG_const_type +CHECK-NEXT: DW_AT_type (0x00000079 +CHECK: DW_TAG_base_type +CHECK-NEXT: DW_AT_name ("char") +CHECK-NEXT: DW_AT_encoding (DW_ATE_signed_char) +CHECK-NEXT: DW_AT_byte_size (0x01) +CHECK: NULL + +REPRODUCER: reproducer written +ERROR: error: cannot parse the debug map +CONFLICT: cannot combine --gen-reproducer and --use-reproducer diff --git a/test/tools/dsymutil/cmdline.test b/test/tools/dsymutil/cmdline.test index 29f6e70f013..e3f8bbdf750 100644 --- a/test/tools/dsymutil/cmdline.test +++ b/test/tools/dsymutil/cmdline.test @@ -9,6 +9,7 @@ CHECK: -accelerator CHECK: -arch CHECK: -dump-debug-map CHECK: -flat +CHECK: -gen-reproducer CHECK: -help CHECK: -minimize CHECK: -no-odr @@ -28,6 +29,7 @@ CHECK: -symtab CHECK: {{-S}} CHECK: -toolchain CHECK: -update +CHECK: -use-reproducer CHECK: -verbose CHECK: -verify CHECK: {{-y}} diff --git a/tools/dsymutil/CMakeLists.txt b/tools/dsymutil/CMakeLists.txt index a42e1a98080..1efc58d19e9 100644 --- a/tools/dsymutil/CMakeLists.txt +++ b/tools/dsymutil/CMakeLists.txt @@ -26,6 +26,7 @@ add_llvm_tool(dsymutil DwarfLinkerForBinary.cpp MachODebugMapParser.cpp MachOUtils.cpp + Reproducer.cpp SymbolMap.cpp DEPENDS diff --git a/tools/dsymutil/Options.td b/tools/dsymutil/Options.td index bdd11b5c4c1..7bd21b0d434 100644 --- a/tools/dsymutil/Options.td +++ b/tools/dsymutil/Options.td @@ -164,6 +164,16 @@ def: Separate<["-"], "j">, HelpText<"Alias for --num-threads">, Group; +def gen_reproducer: F<"gen-reproducer">, + HelpText<"Generate a reproducer consisting of the input object files.">, + Group; + +def use_reproducer: Separate<["--", "-"], "use-reproducer">, + MetaVarName<"">, + HelpText<"Use the object files from the given reproducer path.">, + Group; +def: Joined<["--", "-"], "use-reproducer=">, Alias; + def remarks_prepend_path: Separate<["--", "-"], "remarks-prepend-path">, MetaVarName<"">, HelpText<"Specify a directory to prepend to the paths of the external remark files.">, diff --git a/tools/dsymutil/Reproducer.cpp b/tools/dsymutil/Reproducer.cpp new file mode 100644 index 00000000000..baa6bc6f11e --- /dev/null +++ b/tools/dsymutil/Reproducer.cpp @@ -0,0 +1,82 @@ +//===- Reproducer.cpp -----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/Path.h" +#include + +using namespace llvm; +using namespace llvm::dsymutil; + +static std::string createReproducerDir(std::error_code &EC) { + SmallString<128> Root; + if (const char *Path = getenv("DSYMUTIL_REPRODUCER_PATH")) { + Root.assign(Path); + EC = sys::fs::create_directory(Root); + } else { + EC = sys::fs::createUniqueDirectory("dsymutil", Root); + } + return EC ? "" : std::string(Root); +} + +Reproducer::Reproducer() : VFS(vfs::getRealFileSystem()) {} +Reproducer::~Reproducer() = default; + +ReproducerGenerate::ReproducerGenerate(std::error_code &EC) + : Root(createReproducerDir(EC)), FC() { + if (!Root.empty()) + FC = std::make_shared(Root, Root); + VFS = FileCollector::createCollectorVFS(vfs::getRealFileSystem(), FC); +} + +ReproducerGenerate::~ReproducerGenerate() { + if (!FC) + return; + FC->copyFiles(false); + SmallString<128> Mapping(Root); + sys::path::append(Mapping, "mapping.yaml"); + FC->writeMapping(Mapping.str()); + outs() << "reproducer written to " << Root << '\n'; +} + +ReproducerUse::~ReproducerUse() = default; + +ReproducerUse::ReproducerUse(StringRef Root, std::error_code &EC) { + SmallString<128> Mapping(Root); + sys::path::append(Mapping, "mapping.yaml"); + ErrorOr> Buffer = + vfs::getRealFileSystem()->getBufferForFile(Mapping.str()); + + if (!Buffer) { + EC = Buffer.getError(); + return; + } + + VFS = llvm::vfs::getVFSFromYAML(std::move(Buffer.get()), nullptr, Mapping); +} + +llvm::Expected> +Reproducer::createReproducer(ReproducerMode Mode, StringRef Root) { + switch (Mode) { + case ReproducerMode::Generate: { + std::error_code EC; + auto Repro = std::make_unique(EC); + if (EC) + return errorCodeToError(EC); + return Repro; + } + case ReproducerMode::Use: { + std::error_code EC; + auto Repro = std::make_unique(Root, EC); + if (EC) + return errorCodeToError(EC); + return Repro; + } + case ReproducerMode::Off: + return std::make_unique(); + } +} diff --git a/tools/dsymutil/Reproducer.h b/tools/dsymutil/Reproducer.h new file mode 100644 index 00000000000..e965e1ceda2 --- /dev/null +++ b/tools/dsymutil/Reproducer.h @@ -0,0 +1,77 @@ +//===- tools/dsymutil/Reproducer.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLS_DSYMUTIL_REPRODUCER_H +#define LLVM_TOOLS_DSYMUTIL_REPRODUCER_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/FileCollector.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace llvm { +namespace dsymutil { + +/// The reproducer mode. +enum class ReproducerMode { + Generate, + Use, + Off, +}; + +/// The reproducer class manages the sate related to reproducers in dsymutil. +/// Instances should be created with Reproducer::createReproducer. An instance +/// of this class is returned when reproducers are off. The VFS returned by +/// this instance is the real file system. +class Reproducer { +public: + Reproducer(); + virtual ~Reproducer(); + + IntrusiveRefCntPtr getVFS() const { return VFS; } + + /// Create a Reproducer instance based on the given mode. + static llvm::Expected> + createReproducer(ReproducerMode Mode, StringRef Root); + +protected: + IntrusiveRefCntPtr VFS; +}; + +/// Reproducer instance used to generate a new reproducer. The VFS returned by +/// this instance is a FileCollectorFileSystem that tracks every file used by +/// dsymutil. +class ReproducerGenerate : public Reproducer { +public: + ReproducerGenerate(std::error_code &EC); + ~ReproducerGenerate() override; + +private: + /// The path to the reproducer. + std::string Root; + + /// The FileCollector used by the FileCollectorFileSystem. + std::shared_ptr FC; +}; + +/// Reproducer instance used to use an existing reproducer. The VFS returned by +/// this instance is a RedirectingFileSystem that remaps paths to their +/// counterpart in the reproducer. +class ReproducerUse : public Reproducer { +public: + ReproducerUse(StringRef Root, std::error_code &EC); + ~ReproducerUse() override; + +private: + /// The path to the reproducer. + std::string Root; +}; + +} // end namespace dsymutil +} // end namespace llvm + +#endif // LLVM_TOOLS_DSYMUTIL_REPRODUCER_H diff --git a/tools/dsymutil/dsymutil.cpp b/tools/dsymutil/dsymutil.cpp index 8ea89933e36..32df55611f0 100644 --- a/tools/dsymutil/dsymutil.cpp +++ b/tools/dsymutil/dsymutil.cpp @@ -16,6 +16,7 @@ #include "DebugMap.h" #include "LinkUtils.h" #include "MachOUtils.h" +#include "Reproducer.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -31,6 +32,7 @@ #include "llvm/Option/ArgList.h" #include "llvm/Option/Option.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileCollector.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/InitLLVM.h" #include "llvm/Support/ManagedStatic.h" @@ -92,9 +94,11 @@ struct DsymutilOptions { std::string SymbolMap; std::string OutputFile; std::string Toolchain; + std::string ReproducerPath; std::vector Archs; std::vector InputFiles; unsigned NumThreads; + ReproducerMode ReproMode = ReproducerMode::Off; dsymutil::LinkOptions LinkOpts; }; @@ -182,6 +186,12 @@ static Error verifyOptions(const DsymutilOptions &Options) { "paper trail warnings are not supported for YAML input.", errc::invalid_argument); + if (!Options.ReproducerPath.empty() && + Options.ReproMode != ReproducerMode::Use) + return make_error( + "cannot combine --gen-reproducer and --use-reproducer.", + errc::invalid_argument); + return Error::success(); } @@ -222,6 +232,14 @@ static Expected getOptions(opt::InputArgList &Args) { Options.LinkOpts.Verbose = Args.hasArg(OPT_verbose); Options.LinkOpts.Statistics = Args.hasArg(OPT_statistics); + if (opt::Arg *ReproducerPath = Args.getLastArg(OPT_use_reproducer)) { + Options.ReproMode = ReproducerMode::Use; + Options.ReproducerPath = ReproducerPath->getValue(); + } + + if (Args.hasArg(OPT_gen_reproducer)) + Options.ReproMode = ReproducerMode::Generate; + if (Expected AccelKind = getAccelTableKind(Args)) { Options.LinkOpts.TheAccelTableKind = *AccelKind; } else { @@ -499,6 +517,15 @@ int main(int argc, char **argv) { InitializeAllTargets(); InitializeAllAsmPrinters(); + auto Repro = + Reproducer::createReproducer(Options.ReproMode, Options.ReproducerPath); + if (!Repro) { + WithColor::error() << toString(Repro.takeError()); + return 1; + } + + Options.LinkOpts.VFS = (*Repro)->getVFS(); + for (const auto &Arch : Options.Archs) if (Arch != "*" && Arch != "all" && !object::MachOObjectFile::isValidArch(Arch)) {