[ThinLTO] Allow indexing to request backend to ignore the module

Summary:
Gold plugin does not add pass to ThinLTO modules without useful symbols.
In this case ThinLTO can't create corresponding index file and some features, like CFI,
cannot be processes by backed correctly without index.
Given that we don't need the backed output we can request it to avoid
processing the module. This is implemented by this patch using new
"SkipModuleByDistributedBackend" flag.

Reviewers: pcc, tejohnson

Subscribers: mehdi_amini, inglorion, eraman, cfe-commits

Differential Revision: https://reviews.llvm.org/D42995

llvm-svn: 325411

include/llvm/IR/ModuleSummaryIndex.h

@@ -682,6 +682,13 @@ private:
   /// considered live.
   bool WithGlobalValueDeadStripping = false;
 
+  /// Indicates that distributed backend should skip compilation of the
+  /// module. Flag is suppose to be set by distributed ThinLTO indexing
+  /// when it detected that the module is not needed during the final
+  /// linking. As result distributed backend should just output a minimal
+  /// valid object file.
+  bool SkipModuleByDistributedBackend = false;
+
   /// If true then we're performing analysis of IR module, filling summary
   /// accordingly. The value of 'false' means we're reading summary from
   /// BC or YAML source. Affects the type of value stored in NameOrGV union
@@ -718,6 +725,13 @@ public:
     WithGlobalValueDeadStripping = true;
   }
 
+  bool skipModuleByDistributedBackend() const {
+    return SkipModuleByDistributedBackend;
+  }
+  void setSkipModuleByDistributedBackend() {
+    SkipModuleByDistributedBackend = true;
+  }
+
   bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
     return !WithGlobalValueDeadStripping || GVS->isLive();
   }

lib/Bitcode/Reader/BitcodeReader.cpp

@@ -5186,11 +5186,14 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
     case bitc::FS_FLAGS: {  // [flags]
       uint64_t Flags = Record[0];
       // Scan flags (set only on the combined index).
-      assert(Flags <= 1 && "Unexpected bits in flag");
+      assert(Flags <= 0x3 && "Unexpected bits in flag");
 
       // 1 bit: WithGlobalValueDeadStripping flag.
       if (Flags & 0x1)
         TheIndex.setWithGlobalValueDeadStripping();
+      // 1 bit: SkipModuleByDistributedBackend flag.
+      if (Flags & 0x2)
+        TheIndex.setSkipModuleByDistributedBackend();
       break;
     }
     case bitc::FS_VALUE_GUID: { // [valueid, refguid]

lib/Bitcode/Writer/BitcodeWriter.cpp

@@ -3604,10 +3604,13 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
   Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
   Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION});
 
-  // Write the index flags. Currently we only write a single flag, the value of
-  // withGlobalValueDeadStripping, which only applies to the combined index.
-  Stream.EmitRecord(bitc::FS_FLAGS,
-                    ArrayRef<uint64_t>{Index.withGlobalValueDeadStripping()});
+  // Write the index flags.
+  uint64_t Flags = 0;
+  if (Index.withGlobalValueDeadStripping())
+    Flags |= 0x1;
+  if (Index.skipModuleByDistributedBackend())
+    Flags |= 0x2;
+  Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
 
   for (const auto &GVI : valueIds()) {
     Stream.EmitRecord(bitc::FS_VALUE_GUID,

test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll

@@ -31,6 +31,24 @@
 ; RUN: ls %t2.o.imports
 ; RUN: ls %t3.o.imports
 
+; Regular *thinlto.bc file. "SkipModuleByDistributedBackend" flag (0x2)
+; should not be set.
+; RUN: llvm-bcanalyzer --dump %t1.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC1
+; CHECK-BC1: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-BC1: <FLAGS op0=1/>
+; CHECK-BC1: </GLOBALVAL_SUMMARY_BLOCK
+
+; Nothing interesting in the corresponding object file, so
+; "SkipModuleByDistributedBackend" flag (0x2) should be set.
+; RUN: llvm-bcanalyzer --dump %t2.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC2
+; CHECK-BC2: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-BC2: <FLAGS op0=2/>
+; CHECK-BC2: </GLOBALVAL_SUMMARY_BLOCK
+
+; Empty as the corresponding object file is not ThinTLO.
+; RUN: not llvm-bcanalyzer --dump %t3.o.thinlto.bc 2>&1 | FileCheck %s -check-prefixes=CHECK-BC3
+; CHECK-BC3: LLVM ERROR: Unexpected end of file
+
 ; RUN: cat %t.index | FileCheck %s
 ; CHECK: thinlto_emit_linked_objects.ll.tmp1.o
 ; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o

tools/gold/gold-plugin.cpp

@@ -811,9 +811,14 @@ static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite) {
 // final link. Frequently the distributed build system will want to
 // confirm that all expected outputs are created based on all of the
 // modules provided to the linker.
+// If SkipModule is true then .thinlto.bc should contain just
+// SkipModuleByDistributedBackend flag which requests distributed backend
+// to skip the compilation of the corresponding module and produce an empty
+// object file.
 static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
                                               const std::string &OldPrefix,
-                                              const std::string &NewPrefix) {
+                                              const std::string &NewPrefix,
+                                              bool SkipModule) {
   std::string NewModulePath =
       getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
   std::error_code EC;
@@ -823,6 +828,12 @@ static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
     if (EC)
       message(LDPL_FATAL, "Failed to write '%s': %s",
               (NewModulePath + ".thinlto.bc").c_str(), EC.message().c_str());
+
+    if (SkipModule) {
+      ModuleSummaryIndex Index(false);
+      Index.setSkipModuleByDistributedBackend();
+      WriteIndexToFile(Index, OS, nullptr);
+    }
   }
   if (options::thinlto_emit_imports_files) {
     raw_fd_ostream OS(NewModulePath + ".imports", EC,
@@ -878,6 +889,11 @@ static ld_plugin_status allSymbolsReadHook() {
     assert(ObjFilename.second);
     if (const void *View = getSymbolsAndView(F))
       addModule(*Lto, F, View, ObjFilename.first->first());
+    else if (options::thinlto_index_only) {
+      ObjFilename.first->second = true;
+      writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix,
+                                        /* SkipModule */ true);
+    }
   }
 
   SmallString<128> Filename;
@@ -895,7 +911,7 @@ static ld_plugin_status allSymbolsReadHook() {
   auto AddStream =
       [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
     IsTemporary[Task] = !SaveTemps;
-    int FD = getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps,
+    int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
                                Filenames[Task], Task);
     return llvm::make_unique<lto::NativeObjectStream>(
         llvm::make_unique<llvm::raw_fd_ostream>(FD, true));
@@ -920,7 +936,7 @@ static ld_plugin_status allSymbolsReadHook() {
     for (auto &Identifier : ObjectToIndexFileState)
       if (!Identifier.getValue())
         writeEmptyDistributedBuildOutputs(Identifier.getKey(), OldPrefix,
-                                          NewPrefix);
+                                          NewPrefix, /* SkipModule */ false);
 
   if (options::TheOutputType == options::OT_DISABLE ||
       options::TheOutputType == options::OT_BC_ONLY)