[ThinLTO] Add option to emit imports files for distributed backends

Summary:
Add support for emission of plaintext lists of the imported files for
each distributed backend compilation. Used for distributed build file
staging.

Invoked with new gold-plugin thinlto-emit-imports-files option, which is
only valid with thinlto-index-only (i.e. for distributed builds), or
from llvm-lto with new -thinlto-action=emitimports value.

Depends on D19556.

Reviewers: joker.eph

Subscribers: llvm-commits, joker.eph

Differential Revision: http://reviews.llvm.org/D19636

llvm-svn: 269067

include/llvm/LTO/ThinLTOCodeGenerator.h

@@ -201,6 +201,12 @@ public:
    */
   void promote(Module &Module, ModuleSummaryIndex &Index);
 
+  /**
+   * Compute and emit the imported files for module at \p ModulePath.
+   */
+  static void emitImports(StringRef ModulePath, StringRef OutputName,
+                          ModuleSummaryIndex &Index);
+
   /**
    * Perform cross-module importing for the module identified by
    * ModuleIdentifier.

include/llvm/Transforms/IPO/FunctionImport.h

@@ -103,6 +103,10 @@ void gatherImportedSummariesForModule(
     const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
     const StringMap<FunctionImporter::ImportMapTy> &ImportLists,
     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex);
+
+std::error_code
+EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename,
+                 const StringMap<FunctionImporter::ImportMapTy> &ImportLists);
 }
 
 #endif // LLVM_FUNCTIONIMPORT_H

lib/LTO/ThinLTOCodeGenerator.cpp

@@ -742,6 +742,30 @@ void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
                                          ModuleToSummariesForIndex);
 }
 
+/**
+ * Emit the list of files needed for importing into module.
+ */
+void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
+                                       StringRef OutputName,
+                                       ModuleSummaryIndex &Index) {
+  auto ModuleCount = Index.modulePaths().size();
+
+  // Collect for each module the list of function it defines (GUID -> Summary).
+  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
+  Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
+
+  // Generate import/export list
+  StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
+  StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
+  ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
+                           ExportLists);
+
+  std::error_code EC;
+  if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists)))
+    report_fatal_error(Twine("Failed to open ") + OutputName +
+                       " to save imports lists\n");
+}
+
 /**
  * Perform internalization.
  */

lib/Transforms/IPO/FunctionImport.cpp

@@ -445,6 +445,21 @@ void llvm::gatherImportedSummariesForModule(
   }
 }
 
+/// Emit the files \p ModulePath will import from into \p OutputFilename.
+std::error_code llvm::EmitImportsFiles(
+    StringRef ModulePath, StringRef OutputFilename,
+    const StringMap<FunctionImporter::ImportMapTy> &ImportLists) {
+  auto ModuleImports = ImportLists.find(ModulePath);
+  std::error_code EC;
+  raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None);
+  if (EC)
+    return EC;
+  if (ModuleImports != ImportLists.end())
+    for (auto &ILI : ModuleImports->second)
+      ImportsOS << ILI.first() << "\n";
+  return std::error_code();
+}
+
 // Automatically import functions in Module \p DestModule based on the summaries
 // index.
 //

test/ThinLTO/X86/Inputs/emit_imports.ll

@@ -0,0 +1,4 @@
+define void @g() {
+entry:
+  ret void
+}

test/ThinLTO/X86/emit_imports.ll

@@ -0,0 +1,21 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %p/Inputs/emit_imports.ll -o %t2.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc
+; RUN: llvm-lto -thinlto-action=emitimports -thinlto-index %t.index.bc %t1.bc %t2.bc
+
+; The imports file for this module contains the bitcode file for
+; Inputs/emit_imports.ll
+; RUN: cat %t1.bc.imports | count 1
+; RUN: cat %t1.bc.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: emit_imports.ll.tmp2.bc
+
+; The imports file for Input/emit_imports.ll is empty as it does not import anything.
+; RUN: cat %t2.bc.imports | count 0
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}

test/tools/gold/X86/thinlto_emit_imports.ll

@@ -0,0 +1,27 @@
+; Generate summary sections and test gold handling.
+; RUN: opt -module-summary %s -o %t.o
+; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o
+
+; Ensure gold generates imports files if requested for distributed backends.
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=thinlto-index-only \
+; RUN:    --plugin-opt=thinlto-emit-imports-files \
+; RUN:    -shared %t.o %t2.o -o %t3
+
+; The imports file for this module contains the bitcode file for
+; Inputs/thinlto.ll
+; RUN: cat %t.o.imports | count 1
+; RUN: cat %t.o.imports | FileCheck %s --check-prefix=IMPORTS1
+; IMPORTS1: test/tools/gold/X86/Output/thinlto_emit_imports.ll.tmp2.o
+
+; The imports file for Input/thinlto.ll is empty as it does not import anything.
+; RUN: cat %t2.o.imports | count 0
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}

tools/gold/gold-plugin.cpp

@@ -184,6 +184,11 @@ namespace options {
   // the import decisions, and exit afterwards. The assumption is
   // that the build system will launch the backend processes.
   static bool thinlto_index_only = false;
+  // If true, when generating individual index files for distributed backends,
+  // also generate a "${bitcodefile}.imports" file at the same location for each
+  // bitcode file, listing the files it imports from in plain text. This is to
+  // support distributed build file staging.
+  static bool thinlto_emit_imports_files = false;
   // Additional options to pass into the code generator.
   // Note: This array will contain all plugin options which are not claimed
   // as plugin exclusive to pass to the code generator.
@@ -217,6 +222,8 @@ namespace options {
       thinlto = true;
     } else if (opt == "thinlto-index-only") {
       thinlto_index_only = true;
+    } else if (opt == "thinlto-emit-imports-files") {
+      thinlto_emit_imports_files = true;
     } else if (opt.size() == 2 && opt[0] == 'O') {
       if (opt[1] < '0' || opt[1] > '3')
         message(LDPL_FATAL, "Optimization level must be between 0 and 3");
@@ -1209,6 +1216,10 @@ static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) {
       CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
   }
 
+  if (options::thinlto_emit_imports_files && !options::thinlto_index_only)
+    message(LDPL_WARNING,
+            "thinlto-emit-imports-files ignored unless thinlto-index-only");
+
   if (options::thinlto_index_only) {
     // Collect for each module the list of function it defines (GUID ->
     // Summary).
@@ -1244,6 +1255,15 @@ static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) {
                                        ModuleToDefinedGVSummaries, ImportLists,
                                        ModuleToSummariesForIndex);
       WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
+
+      if (options::thinlto_emit_imports_files) {
+        if ((EC = EmitImportsFiles(
+                 InputFile.file().name,
+                 (Twine(InputFile.file().name) + ".imports").str(),
+                 ImportLists)))
+          message(LDPL_FATAL, "Unable to open %s.imports",
+                  InputFile.file().name, EC.message().c_str());
+      }
     }
 
     cleanup_hook();

tools/llvm-lto/llvm-lto.cpp

@@ -67,6 +67,7 @@ static cl::opt<bool>
 enum ThinLTOModes {
   THINLINK,
   THINDISTRIBUTE,
+  THINEMITIMPORTS,
   THINPROMOTE,
   THINIMPORT,
   THININTERNALIZE,
@@ -83,6 +84,8 @@ cl::opt<ThinLTOModes> ThinLTOMode(
             "ThinLink: produces the index by linking only the summaries."),
         clEnumValN(THINDISTRIBUTE, "distributedindexes",
                    "Produces individual indexes for distributed backends."),
+        clEnumValN(THINEMITIMPORTS, "emitimports",
+                   "Emit imports files for distributed backends."),
         clEnumValN(THINPROMOTE, "promote",
                    "Perform pre-import promotion (requires -thinlto-index)."),
         clEnumValN(THINIMPORT, "import", "Perform both promotion and "
@@ -359,6 +362,8 @@ public:
       return thinLink();
     case THINDISTRIBUTE:
       return distributedIndexes();
+    case THINEMITIMPORTS:
+      return emitImports();
     case THINPROMOTE:
       return promote();
     case THINIMPORT:
@@ -431,6 +436,25 @@ private:
     }
   }
 
+  /// Load the combined index from disk, compute the imports, and emit
+  /// the import file lists for each module to disk.
+  void emitImports() {
+    if (InputFilenames.size() != 1 && !OutputFilename.empty())
+      report_fatal_error("Can't handle a single output filename and multiple "
+                         "input files, do not provide an output filename and "
+                         "the output files will be suffixed from the input "
+                         "ones.");
+
+    auto Index = loadCombinedIndex();
+    for (auto &Filename : InputFilenames) {
+      std::string OutputName = OutputFilename;
+      if (OutputName.empty()) {
+        OutputName = Filename + ".imports";
+      }
+      ThinLTOCodeGenerator::emitImports(Filename, OutputName, *Index);
+    }
+  }
+
   /// Load the combined index from disk, then load every file referenced by
   /// the index and add them to the generator, finally perform the promotion
   /// on the files mentioned on the command line (these must match the index