Restore "[ThinLTO] Emit individual index files for distributed backends"

This restores commit r268627:
    Summary:
    When launching ThinLTO backends in a distributed build (currently
    supported in gold via the thinlto-index-only plugin option), emit
    an individual index file for each backend process as described here:
    http://lists.llvm.org/pipermail/llvm-dev/2016-April/098272.html

    ...

    Differential Revision: http://reviews.llvm.org/D19556

Address msan failures by avoiding std::prev on map.end(), the
theory is that this is causing issues due to some known UB problems
in __tree.

llvm-svn: 269059

include/llvm/Bitcode/ReaderWriter.h

@@ -97,8 +97,12 @@ namespace llvm {
 
   /// Write the specified module summary index to the given raw output stream,
   /// where it will be written in a new bitcode block. This is used when
-  /// writing the combined index file for ThinLTO.
-  void WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out);
+  /// writing the combined index file for ThinLTO. When writing a subset of the
+  /// index for a distributed backend, provide the \p ModuleToSummariesForIndex
+  /// map.
+  void WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out,
+                        std::map<std::string, GVSummaryMapTy>
+                            *ModuleToSummariesForIndex = nullptr);
 
   /// isBitcodeWrapper - Return true if the given bytes are the magic bytes
   /// for an LLVM IR bitcode wrapper.

include/llvm/LTO/ThinLTOCodeGenerator.h

@@ -19,6 +19,7 @@
 #include "llvm-c/lto.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Target/TargetOptions.h"
@@ -27,7 +28,6 @@
 
 namespace llvm {
 class StringRef;
-class ModuleSummaryIndex;
 class LLVMContext;
 class TargetMachine;
 
@@ -207,6 +207,13 @@ public:
    */
   void crossModuleImport(Module &Module, ModuleSummaryIndex &Index);
 
+  /**
+   * Compute the list of summaries needed for importing into module.
+   */
+  static void gatherImportedSummariesForModule(
+      StringRef ModulePath, ModuleSummaryIndex &Index,
+      std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex);
+
   /**
    * Perform internalization.
    */

include/llvm/Transforms/IPO/FunctionImport.h

@@ -87,6 +87,22 @@ void ComputeCrossModuleImport(
 void ComputeCrossModuleImportForModule(
     StringRef ModulePath, const ModuleSummaryIndex &Index,
     FunctionImporter::ImportMapTy &ImportList);
+
+/// Compute the set of summaries needed for a ThinLTO backend compilation of
+/// \p ModulePath.
+//
+/// This includes summaries from that module (in case any global summary based
+/// optimizations were recorded) and from any definitions in other modules that
+/// should be imported.
+//
+/// \p ModuleToSummariesForIndex will be populated with the needed summaries
+/// from each required module path. Use a std::map instead of StringMap to get
+/// stable order for bitcode emission.
+void gatherImportedSummariesForModule(
+    StringRef ModulePath,
+    const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
+    const StringMap<FunctionImporter::ImportMapTy> &ImportLists,
+    std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex);
 }
 
 #endif // LLVM_FUNCTIONIMPORT_H

lib/Bitcode/Writer/BitcodeWriter.cpp

@@ -263,6 +263,10 @@ class IndexBitcodeWriter : public BitcodeWriter {
   /// The combined index to write to bitcode.
   const ModuleSummaryIndex &Index;
 
+  /// When writing a subset of the index for distributed backends, client
+  /// provides a map of modules to the corresponding GUIDs/summaries to write.
+  std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex;
+
   /// Map that holds the correspondence between the GUID used in the combined
   /// index and a value id generated by this class to use in references.
   std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
@@ -272,18 +276,148 @@ class IndexBitcodeWriter : public BitcodeWriter {
 
 public:
   /// Constructs a IndexBitcodeWriter object for the given combined index,
-  /// writing to the provided \p Buffer.
+  /// writing to the provided \p Buffer. When writing a subset of the index
+  /// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
   IndexBitcodeWriter(SmallVectorImpl<char> &Buffer,
-                     const ModuleSummaryIndex &Index)
-      : BitcodeWriter(Buffer), Index(Index) {
-    // Assign unique value ids to all functions in the index for use
+                     const ModuleSummaryIndex &Index,
+                     std::map<std::string, GVSummaryMapTy>
+                         *ModuleToSummariesForIndex = nullptr)
+      : BitcodeWriter(Buffer), Index(Index),
+        ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
+    // Assign unique value ids to all summaries to be written, for use
     // in writing out the call graph edges. Save the mapping from GUID
     // to the new global value id to use when writing those edges, which
     // are currently saved in the index in terms of GUID.
-    for (auto &II : Index)
-      GUIDToValueIdMap[II.first] = ++GlobalValueId;
+    for (const auto &I : *this)
+      GUIDToValueIdMap[I.first] = ++GlobalValueId;
   }
 
+  /// The below iterator returns the GUID and associated summary.
+  typedef std::pair<GlobalValue::GUID, GlobalValueSummary *> GVInfo;
+
+  /// Iterator over the value GUID and summaries to be written to bitcode,
+  /// hides the details of whether they are being pulled from the entire
+  /// index or just those in a provided ModuleToSummariesForIndex map.
+  class iterator
+      : public llvm::iterator_facade_base<iterator, std::forward_iterator_tag,
+                                          GVInfo> {
+    /// Enables access to parent class.
+    const IndexBitcodeWriter &Writer;
+
+    // Iterators used when writing only those summaries in a provided
+    // ModuleToSummariesForIndex map:
+
+    /// Points to the last element in outer ModuleToSummariesForIndex map.
+    std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesBack;
+    /// Iterator on outer ModuleToSummariesForIndex map.
+    std::map<std::string, GVSummaryMapTy>::iterator ModuleSummariesIter;
+    /// Iterator on an inner global variable summary map.
+    GVSummaryMapTy::iterator ModuleGVSummariesIter;
+
+    // Iterators used when writing all summaries in the index:
+
+    /// Points to the last element in the Index outer GlobalValueMap.
+    const_gvsummary_iterator IndexSummariesBack;
+    /// Iterator on outer GlobalValueMap.
+    const_gvsummary_iterator IndexSummariesIter;
+    /// Iterator on an inner GlobalValueSummaryList.
+    GlobalValueSummaryList::const_iterator IndexGVSummariesIter;
+
+  public:
+    /// Construct iterator from parent \p Writer and indicate if we are
+    /// constructing the end iterator.
+    iterator(const IndexBitcodeWriter &Writer, bool IsAtEnd) : Writer(Writer) {
+      // Set up the appropriate set of iterators given whether we are writing
+      // the full index or just a subset.
+      // Can't setup the Back or inner iterators if the corresponding map
+      // is empty. This will be handled specially in operator== as well.
+      if (Writer.ModuleToSummariesForIndex &&
+          !Writer.ModuleToSummariesForIndex->empty()) {
+        ModuleSummariesIter = Writer.ModuleToSummariesForIndex->begin();
+        for (ModuleSummariesBack = Writer.ModuleToSummariesForIndex->begin();
+             std::next(ModuleSummariesBack) !=
+             Writer.ModuleToSummariesForIndex->end();
+             ModuleSummariesBack++)
+          ;
+        ModuleGVSummariesIter = !IsAtEnd ? ModuleSummariesIter->second.begin()
+                                         : ModuleSummariesBack->second.end();
+      } else if (!Writer.ModuleToSummariesForIndex &&
+                 Writer.Index.begin() != Writer.Index.end()) {
+        IndexSummariesIter = Writer.Index.begin();
+        for (IndexSummariesBack = Writer.Index.begin();
+             std::next(IndexSummariesBack) != Writer.Index.end();
+             IndexSummariesBack++)
+          ;
+        IndexGVSummariesIter = !IsAtEnd ? IndexSummariesIter->second.begin()
+                                        : IndexSummariesBack->second.end();
+      }
+    }
+
+    /// Increment the appropriate set of iterators.
+    iterator &operator++() {
+      // First the inner iterator is incremented, then if it is at the end
+      // and there are more outer iterations to go, the inner is reset to
+      // the start of the next inner list.
+      if (Writer.ModuleToSummariesForIndex) {
+        ++ModuleGVSummariesIter;
+        if (ModuleGVSummariesIter == ModuleSummariesIter->second.end() &&
+            ModuleSummariesIter != ModuleSummariesBack) {
+          ++ModuleSummariesIter;
+          ModuleGVSummariesIter = ModuleSummariesIter->second.begin();
+        }
+      } else {
+        ++IndexGVSummariesIter;
+        if (IndexGVSummariesIter == IndexSummariesIter->second.end() &&
+            IndexSummariesIter != IndexSummariesBack) {
+          ++IndexSummariesIter;
+          IndexGVSummariesIter = IndexSummariesIter->second.begin();
+        }
+      }
+      return *this;
+    }
+
+    /// Access the <GUID,GlobalValueSummary*> pair corresponding to the current
+    /// outer and inner iterator positions.
+    GVInfo operator*() {
+      if (Writer.ModuleToSummariesForIndex)
+        return std::make_pair(ModuleGVSummariesIter->first,
+                              ModuleGVSummariesIter->second);
+      return std::make_pair(IndexSummariesIter->first,
+                            IndexGVSummariesIter->get());
+    }
+
+    /// Checks if the iterators are equal, with special handling for empty
+    /// indexes.
+    bool operator==(const iterator &RHS) const {
+      if (Writer.ModuleToSummariesForIndex) {
+        // First ensure that both are writing the same subset.
+        if (Writer.ModuleToSummariesForIndex !=
+            RHS.Writer.ModuleToSummariesForIndex)
+          return false;
+        // Already determined above that maps are the same, so if one is
+        // empty, they both are.
+        if (Writer.ModuleToSummariesForIndex->empty())
+          return true;
+        return ModuleGVSummariesIter == RHS.ModuleGVSummariesIter;
+      }
+      // First ensure RHS also writing the full index, and that both are
+      // writing the same full index.
+      if (RHS.Writer.ModuleToSummariesForIndex ||
+          &Writer.Index != &RHS.Writer.Index)
+        return false;
+      // Already determined above that maps are the same, so if one is
+      // empty, they both are.
+      if (Writer.Index.begin() == Writer.Index.end())
+        return true;
+      return IndexGVSummariesIter == RHS.IndexGVSummariesIter;
+    }
+  };
+
+  /// Obtain the start iterator over the summaries to be written.
+  iterator begin() { return iterator(*this, /*IsAtEnd=*/false); }
+  /// Obtain the end iterator over the summaries to be written.
+  iterator end() { return iterator(*this, /*IsAtEnd=*/true); }
+
 private:
   /// Main entry point for writing a combined index to bitcode, invoked by
   /// BitcodeWriter::write() after it writes the header.
@@ -294,6 +428,14 @@ private:
   void writeCombinedValueSymbolTable();
   void writeCombinedGlobalValueSummary();
 
+  /// Indicates whether the provided \p ModulePath should be written into
+  /// the module string table, e.g. if full index written or if it is in
+  /// the provided subset.
+  bool doIncludeModule(StringRef ModulePath) {
+    return !ModuleToSummariesForIndex ||
+           ModuleToSummariesForIndex->count(ModulePath);
+  }
+
   bool hasValueId(GlobalValue::GUID ValGUID) {
     const auto &VMI = GUIDToValueIdMap.find(ValGUID);
     return VMI != GUIDToValueIdMap.end();
@@ -2964,6 +3106,8 @@ void IndexBitcodeWriter::writeModStrings() {
 
   SmallVector<unsigned, 64> Vals;
   for (const auto &MPSE : Index.modulePaths()) {
+    if (!doIncludeModule(MPSE.getKey()))
+      continue;
     StringEncoding Bits =
         getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size());
     unsigned AbbrevToUse = Abbrev8Bit;
@@ -3217,78 +3361,75 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
     NameVals.clear();
   };
 
-  for (const auto &GSI : Index) {
-    for (auto &SI : GSI.second) {
-      GlobalValueSummary *S = SI.get();
-      assert(S);
+  for (const auto &I : *this) {
+    GlobalValueSummary *S = I.second;
+    assert(S);
 
-      assert(hasValueId(GSI.first));
-      unsigned ValueId = getValueId(GSI.first);
-      SummaryToValueIdMap[S] = ValueId;
+    assert(hasValueId(I.first));
+    unsigned ValueId = getValueId(I.first);
+    SummaryToValueIdMap[S] = ValueId;
 
-      if (auto *AS = dyn_cast<AliasSummary>(S)) {
-        // Will process aliases as a post-pass because the reader wants all
-        // global to be loaded first.
-        Aliases.push_back(AS);
-        continue;
-      }
+    if (auto *AS = dyn_cast<AliasSummary>(S)) {
+      // Will process aliases as a post-pass because the reader wants all
+      // global to be loaded first.
+      Aliases.push_back(AS);
+      continue;
+    }
 
-      if (auto *VS = dyn_cast<GlobalVarSummary>(S)) {
-        NameVals.push_back(ValueId);
-        NameVals.push_back(Index.getModuleId(VS->modulePath()));
-        NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
-        for (auto &RI : VS->refs()) {
-          NameVals.push_back(getValueId(RI.getGUID()));
-        }
-
-        // Emit the finished record.
-        Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals,
-                          FSModRefsAbbrev);
-        NameVals.clear();
-        MaybeEmitOriginalName(*S);
-        continue;
-      }
-
-      auto *FS = cast<FunctionSummary>(S);
+    if (auto *VS = dyn_cast<GlobalVarSummary>(S)) {
       NameVals.push_back(ValueId);
-      NameVals.push_back(Index.getModuleId(FS->modulePath()));
-      NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
-      NameVals.push_back(FS->instCount());
-      NameVals.push_back(FS->refs().size());
-
-      for (auto &RI : FS->refs()) {
+      NameVals.push_back(Index.getModuleId(VS->modulePath()));
+      NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
+      for (auto &RI : VS->refs()) {
         NameVals.push_back(getValueId(RI.getGUID()));
       }
 
-      bool HasProfileData = false;
-      for (auto &EI : FS->calls()) {
-        HasProfileData |= EI.second.ProfileCount != 0;
-        if (HasProfileData)
-          break;
-      }
-
-      for (auto &EI : FS->calls()) {
-        // If this GUID doesn't have a value id, it doesn't have a function
-        // summary and we don't need to record any calls to it.
-        if (!hasValueId(EI.first.getGUID()))
-          continue;
-        NameVals.push_back(getValueId(EI.first.getGUID()));
-        assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite");
-        NameVals.push_back(EI.second.CallsiteCount);
-        if (HasProfileData)
-          NameVals.push_back(EI.second.ProfileCount);
-      }
-
-      unsigned FSAbbrev =
-          (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
-      unsigned Code =
-          (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED);
-
       // Emit the finished record.
-      Stream.EmitRecord(Code, NameVals, FSAbbrev);
+      Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals,
+                        FSModRefsAbbrev);
       NameVals.clear();
       MaybeEmitOriginalName(*S);
+      continue;
     }
+
+    auto *FS = cast<FunctionSummary>(S);
+    NameVals.push_back(ValueId);
+    NameVals.push_back(Index.getModuleId(FS->modulePath()));
+    NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
+    NameVals.push_back(FS->instCount());
+    NameVals.push_back(FS->refs().size());
+
+    for (auto &RI : FS->refs()) {
+      NameVals.push_back(getValueId(RI.getGUID()));
+    }
+
+    bool HasProfileData = false;
+    for (auto &EI : FS->calls()) {
+      HasProfileData |= EI.second.ProfileCount != 0;
+      if (HasProfileData)
+        break;
+    }
+
+    for (auto &EI : FS->calls()) {
+      // If this GUID doesn't have a value id, it doesn't have a function
+      // summary and we don't need to record any calls to it.
+      if (!hasValueId(EI.first.getGUID()))
+        continue;
+      NameVals.push_back(getValueId(EI.first.getGUID()));
+      assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite");
+      NameVals.push_back(EI.second.CallsiteCount);
+      if (HasProfileData)
+        NameVals.push_back(EI.second.ProfileCount);
+    }
+
+    unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
+    unsigned Code =
+        (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED);
+
+    // Emit the finished record.
+    Stream.EmitRecord(Code, NameVals, FSAbbrev);
+    NameVals.clear();
+    MaybeEmitOriginalName(*S);
   }
 
   for (auto *AS : Aliases) {
@@ -3562,12 +3703,15 @@ void IndexBitcodeWriter::writeIndex() {
 
 // Write the specified module summary index to the given raw output stream,
 // where it will be written in a new bitcode block. This is used when
-// writing the combined index file for ThinLTO.
-void llvm::WriteIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out) {
+// writing the combined index file for ThinLTO. When writing a subset of the
+// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
+void llvm::WriteIndexToFile(
+    const ModuleSummaryIndex &Index, raw_ostream &Out,
+    std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) {
   SmallVector<char, 0> Buffer;
   Buffer.reserve(256 * 1024);
 
-  IndexBitcodeWriter IndexWriter(Buffer, Index);
+  IndexBitcodeWriter IndexWriter(Buffer, Index, ModuleToSummariesForIndex);
   IndexWriter.write();
 
   Out.write((char *)&Buffer.front(), Buffer.size());

lib/LTO/ThinLTOCodeGenerator.cpp

@@ -719,6 +719,29 @@ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 }
 
+/**
+ * Compute the list of summaries needed for importing into module.
+ */
+void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
+    StringRef ModulePath, ModuleSummaryIndex &Index,
+    std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
+  auto ModuleCount = Index.modulePaths().size();
+
+  // Collect for each module the list of function it defines (GUID -> Summary).
+  StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
+  Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
+
+  // Generate import/export list
+  StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
+  StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
+  ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
+                           ExportLists);
+
+  llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
+                                         ImportLists,
+                                         ModuleToSummariesForIndex);
+}
+
 /**
  * Perform internalization.
  */

lib/Transforms/IPO/FunctionImport.cpp

@@ -418,6 +418,33 @@ void llvm::ComputeCrossModuleImportForModule(
 #endif
 }
 
+/// Compute the set of summaries needed for a ThinLTO backend compilation of
+/// \p ModulePath.
+void llvm::gatherImportedSummariesForModule(
+    StringRef ModulePath,
+    const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
+    const StringMap<FunctionImporter::ImportMapTy> &ImportLists,
+    std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
+  // Include all summaries from the importing module.
+  ModuleToSummariesForIndex[ModulePath] =
+      ModuleToDefinedGVSummaries.lookup(ModulePath);
+  auto ModuleImports = ImportLists.find(ModulePath);
+  if (ModuleImports != ImportLists.end()) {
+    // Include summaries for imports.
+    for (auto &ILI : ModuleImports->second) {
+      auto &SummariesForIndex = ModuleToSummariesForIndex[ILI.first()];
+      const auto &DefinedGVSummaries =
+          ModuleToDefinedGVSummaries.lookup(ILI.first());
+      for (auto &GI : ILI.second) {
+        const auto &DS = DefinedGVSummaries.find(GI.first);
+        assert(DS != DefinedGVSummaries.end() &&
+               "Expected a defined summary for imported global value");
+        SummariesForIndex[GI.first] = DS->second;
+      }
+    }
+  }
+}
+
 // Automatically import functions in Module \p DestModule based on the summaries
 // index.
 //

test/ThinLTO/X86/Inputs/distributed_indexes.ll

@@ -0,0 +1,4 @@
+define void @g() {
+entry:
+  ret void
+}

test/ThinLTO/X86/distributed_indexes.ll

@@ -0,0 +1,47 @@
+; RUN: opt -module-summary %s -o %t1.bc
+; RUN: opt -module-summary %p/Inputs/distributed_indexes.ll -o %t2.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc %t2.bc
+; RUN: llvm-lto -thinlto-action=distributedindexes -thinlto-index %t.index.bc %t1.bc %t2.bc
+; RUN: llvm-bcanalyzer -dump %t1.bc.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump %t2.bc.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
+
+; The backend index for this module contains summaries from itself and
+; Inputs/distributed_indexes.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}distributed_indexes.ll.tmp{{.*}}.bc'
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}distributed_indexes.ll.tmp{{.*}}.bc'
+; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1-NEXT: <VERSION
+; BACKEND1-NEXT: <COMBINED
+; BACKEND1-NEXT: <COMBINED
+; BACKEND1-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1-NEXT: <VALUE_SYMTAB
+; Check that the format is: op0=valueid, op1=offset, op2=funcguid,
+; where funcguid is the lower 64 bits of the function name MD5.
+; BACKEND1-NEXT: <COMBINED_ENTRY abbrevid={{[0-9]+}} op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
+; BACKEND1-NEXT: <COMBINED_ENTRY abbrevid={{[0-9]+}} op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
+; BACKEND1-NEXT: </VALUE_SYMTAB
+
+; The backend index for Input/distributed_indexes.ll contains summaries from
+; itself only, as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = '{{.*}}distributed_indexes.ll.tmp2.bc'
+; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VALUE_SYMTAB
+; Check that the format is: op0=valueid, op1=offset, op2=funcguid,
+; where funcguid is the lower 64 bits of the function name MD5.
+; BACKEND2-NEXT: <COMBINED_ENTRY abbrevid={{[0-9]+}} op0=1 op1=-5300342847281564238
+; BACKEND2-NEXT: </VALUE_SYMTAB
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}

test/tools/gold/X86/thinlto.ll

@@ -21,7 +21,8 @@
 ; RUN:    --plugin-opt=thinlto \
 ; RUN:    --plugin-opt=thinlto-index-only \
 ; RUN:    -shared %t.o %t2.o -o %t3
-; RUN: llvm-bcanalyzer -dump %t3.thinlto.bc | FileCheck %s --check-prefix=COMBINED
+; RUN: llvm-bcanalyzer -dump %t.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND1
+; RUN: llvm-bcanalyzer -dump %t2.o.thinlto.bc | FileCheck %s --check-prefix=BACKEND2
 ; RUN: not test -e %t3
 
 ; Ensure gold generates an index as well as a binary by default in ThinLTO mode.
@@ -53,6 +54,39 @@
 ; NM: T f
 ; NM2: T {{f|g}}
 
+; The backend index for this module contains summaries from itself and
+; Inputs/thinlto.ll, as it imports from the latter.
+; BACKEND1: <MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'
+; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'
+; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND1-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1-NEXT: <VERSION
+; BACKEND1-NEXT: <COMBINED
+; BACKEND1-NEXT: <COMBINED
+; BACKEND1-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+; BACKEND1-NEXT: <VALUE_SYMTAB
+; Check that the format is: op0=valueid, op1=offset, op2=funcguid,
+; where funcguid is the lower 64 bits of the function name MD5.
+; BACKEND1-NEXT: <COMBINED_ENTRY abbrevid={{[0-9]+}} op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
+; BACKEND1-NEXT: <COMBINED_ENTRY abbrevid={{[0-9]+}} op0={{1|2}} op1={{-3706093650706652785|-5300342847281564238}}
+; BACKEND1-NEXT: </VALUE_SYMTAB
+
+; The backend index for Input/thinlto.ll contains summaries from itself only,
+; as it does not import anything.
+; BACKEND2: <MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp2.o'
+; BACKEND2-NEXT: </MODULE_STRTAB_BLOCK
+; BACKEND2-NEXT: <GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VERSION
+; BACKEND2-NEXT: <COMBINED
+; BACKEND2-NEXT: </GLOBALVAL_SUMMARY_BLOCK
+; BACKEND2-NEXT: <VALUE_SYMTAB
+; Check that the format is: op0=valueid, op1=offset, op2=funcguid,
+; where funcguid is the lower 64 bits of the function name MD5.
+; BACKEND2-NEXT: <COMBINED_ENTRY abbrevid={{[0-9]+}} op0=1 op1=-5300342847281564238
+; BACKEND2-NEXT: </VALUE_SYMTAB
+
 ; COMBINED: <MODULE_STRTAB_BLOCK
 ; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'
 ; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'

tools/gold/gold-plugin.cpp

@@ -41,6 +41,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/thread.h"
 #include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/IPO/FunctionImport.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
 #include "llvm/Transforms/Utils/GlobalStatus.h"
@@ -179,7 +180,8 @@ namespace options {
   static bool thinlto = false;
   // If false, all ThinLTO backend compilations through code gen are performed
   // using multiple threads in the gold-plugin, before handing control back to
-  // gold. If true, exit after creating the combined index, the assuming is
+  // gold. If true, write individual backend index files which reflect
+  // the import decisions, and exit afterwards. The assumption is
   // that the build system will launch the backend processes.
   static bool thinlto_index_only = false;
   // Additional options to pass into the code generator.
@@ -1190,6 +1192,79 @@ static void thinLTOBackends(raw_fd_ostream *ApiFile,
     Task.cleanup();
 }
 
+/// Perform ThinLTO link, which creates the combined index file.
+/// Also, either launch backend threads or (under thinlto-index-only)
+/// emit individual index files for distributed backends and exit.
+static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) {
+  ModuleSummaryIndex CombinedIndex;
+  uint64_t NextModuleId = 0;
+  for (claimed_file &F : Modules) {
+    PluginInputFile InputFile(F.handle);
+
+    std::unique_ptr<ModuleSummaryIndex> Index =
+        getModuleSummaryIndexForFile(F, InputFile.file());
+
+    // Skip files without a module summary.
+    if (Index)
+      CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
+  }
+
+  if (options::thinlto_index_only) {
+    // Collect for each module the list of function it defines (GUID ->
+    // Summary).
+    StringMap<std::map<GlobalValue::GUID, GlobalValueSummary *>>
+        ModuleToDefinedGVSummaries(NextModuleId);
+    CombinedIndex.collectDefinedGVSummariesPerModule(
+        ModuleToDefinedGVSummaries);
+
+    // FIXME: We want to do this for the case where the threads are launched
+    // from gold as well, in which case this will be moved out of the
+    // thinlto_index_only handling, and the function importer will be invoked
+    // directly using the Lists.
+    StringMap<FunctionImporter::ImportMapTy> ImportLists(NextModuleId);
+    StringMap<FunctionImporter::ExportSetTy> ExportLists(NextModuleId);
+    ComputeCrossModuleImport(CombinedIndex, ModuleToDefinedGVSummaries,
+                             ImportLists, ExportLists);
+
+    // For each input bitcode file, generate an individual index that
+    // contains summaries only for its own global values, and for any that
+    // should be imported.
+    for (claimed_file &F : Modules) {
+      PluginInputFile InputFile(F.handle);
+      std::error_code EC;
+      raw_fd_ostream OS((Twine(InputFile.file().name) + ".thinlto.bc").str(),
+                        EC, sys::fs::OpenFlags::F_None);
+      if (EC)
+        message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
+                InputFile.file().name, EC.message().c_str());
+      // Build a map of module to the GUIDs and summary objects that should
+      // be written to its index.
+      std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
+      gatherImportedSummariesForModule(InputFile.file().name,
+                                       ModuleToDefinedGVSummaries, ImportLists,
+                                       ModuleToSummariesForIndex);
+      WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
+    }
+
+    cleanup_hook();
+    exit(0);
+  }
+
+  // Create OS in nested scope so that it will be closed on destruction.
+  {
+    std::error_code EC;
+    raw_fd_ostream OS(output_name + ".thinlto.bc", EC,
+                      sys::fs::OpenFlags::F_None);
+    if (EC)
+      message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
+              output_name.data(), EC.message().c_str());
+    WriteIndexToFile(CombinedIndex, OS);
+  }
+
+  thinLTOBackends(ApiFile, CombinedIndex);
+  return LDPS_OK;
+}
+
 /// gold informs us that all symbols have been read. At this point, we use
 /// get_symbols to see if any of our definitions have been overridden by a
 /// native object file. Then, perform optimization and codegen.
@@ -1200,40 +1275,8 @@ static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
   if (unsigned NumOpts = options::extra.size())
     cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
 
-  // If we are doing ThinLTO compilation, simply build the combined
-  // module index/summary and emit it. We don't need to parse the modules
-  // and link them in this case.
-  if (options::thinlto) {
-    ModuleSummaryIndex CombinedIndex;
-    uint64_t NextModuleId = 0;
-    for (claimed_file &F : Modules) {
-      PluginInputFile InputFile(F.handle);
-
-      std::unique_ptr<ModuleSummaryIndex> Index =
-          getModuleSummaryIndexForFile(F, InputFile.file());
-
-      // Skip files without a module summary.
-      if (Index)
-        CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
-    }
-
-    std::error_code EC;
-    raw_fd_ostream OS(output_name + ".thinlto.bc", EC,
-                      sys::fs::OpenFlags::F_None);
-    if (EC)
-      message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
-              output_name.data(), EC.message().c_str());
-    WriteIndexToFile(CombinedIndex, OS);
-    OS.close();
-
-    if (options::thinlto_index_only) {
-      cleanup_hook();
-      exit(0);
-    }
-
-    thinLTOBackends(ApiFile, CombinedIndex);
-    return LDPS_OK;
-  }
+  if (options::thinlto)
+    return thinLTOLink(ApiFile);
 
   LLVMContext Context;
   Context.setDiscardValueNames(options::TheOutputType !=

tools/llvm-lto/llvm-lto.cpp

@@ -66,6 +66,7 @@ static cl::opt<bool>
 
 enum ThinLTOModes {
   THINLINK,
+  THINDISTRIBUTE,
   THINPROMOTE,
   THINIMPORT,
   THININTERNALIZE,
@@ -80,6 +81,8 @@ cl::opt<ThinLTOModes> ThinLTOMode(
         clEnumValN(
             THINLINK, "thinlink",
             "ThinLink: produces the index by linking only the summaries."),
+        clEnumValN(THINDISTRIBUTE, "distributedindexes",
+                   "Produces individual indexes for distributed backends."),
         clEnumValN(THINPROMOTE, "promote",
                    "Perform pre-import promotion (requires -thinlto-index)."),
         clEnumValN(THINIMPORT, "import", "Perform both promotion and "
@@ -354,6 +357,8 @@ public:
     switch (ThinLTOMode) {
     case THINLINK:
       return thinLink();
+    case THINDISTRIBUTE:
+      return distributedIndexes();
     case THINPROMOTE:
       return promote();
     case THINIMPORT:
@@ -396,6 +401,36 @@ private:
     return;
   }
 
+  /// Load the combined index from disk, then compute and generate
+  /// individual index files suitable for ThinLTO distributed backend builds
+  /// on the files mentioned on the command line (these must match the index
+  /// content).
+  void distributedIndexes() {
+    if (InputFilenames.size() != 1 && !OutputFilename.empty())
+      report_fatal_error("Can't handle a single output filename and multiple "
+                         "input files, do not provide an output filename and "
+                         "the output files will be suffixed from the input "
+                         "ones.");
+
+    auto Index = loadCombinedIndex();
+    for (auto &Filename : InputFilenames) {
+      // Build a map of module to the GUIDs and summary objects that should
+      // be written to its index.
+      std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
+      ThinLTOCodeGenerator::gatherImportedSummariesForModule(
+          Filename, *Index, ModuleToSummariesForIndex);
+
+      std::string OutputName = OutputFilename;
+      if (OutputName.empty()) {
+        OutputName = Filename + ".thinlto.bc";
+      }
+      std::error_code EC;
+      raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None);
+      error(EC, "error opening the file '" + OutputName + "'");
+      WriteIndexToFile(*Index, OS, &ModuleToSummariesForIndex);
+    }
+  }
+
   /// Load the combined index from disk, then load every file referenced by
   /// the index and add them to the generator, finally perform the promotion
   /// on the files mentioned on the command line (these must match the index