From d7c94852436821726ce48f9b2cf0c29d59a32ceb Mon Sep 17 00:00:00 2001 From: Teresa Johnson Date: Mon, 11 Apr 2016 13:58:45 +0000 Subject: [PATCH] [ThinLTO] Move summary computation from BitcodeWriter to new pass Summary: This is the first step in also serializing the index out to LLVM assembly. The per-module summary written to bitcode is moved out of the bitcode writer and to a new analysis pass (ModuleSummaryIndexWrapperPass). The pass itself uses a new builder class to compute index, and the builder class is used directly in places where we don't have a pass manager (e.g. llvm-as). Because we are computing summaries outside of the bitcode writer, we no longer can use value ids created by the bitcode writer's ValueEnumerator. This required changing the reference graph edge type to use a new ValueInfo class holding a union between a GUID (combined index) and Value* (permodule index). The Value* are converted to the appropriate value ID during bitcode writing. Also, this enables removal of the BitWriter library's dependence on the Analysis library that was previously required for the summary computation. Reviewers: joker.eph Subscribers: joker.eph, llvm-commits Differential Revision: http://reviews.llvm.org/D18763 llvm-svn: 265941 --- include/llvm/Analysis/ModuleSummaryAnalysis.h | 86 ++++++++ include/llvm/Bitcode/ReaderWriter.h | 2 +- include/llvm/IR/ModuleSummaryIndex.h | 71 ++++++- include/llvm/InitializePasses.h | 2 + lib/Analysis/Analysis.cpp | 1 + lib/Analysis/CMakeLists.txt | 1 + lib/Analysis/ModuleSummaryAnalysis.cpp | 186 +++++++++++++++++ lib/Bitcode/Writer/BitcodeWriter.cpp | 196 ++++-------------- lib/Bitcode/Writer/BitcodeWriterPass.cpp | 32 ++- lib/Bitcode/Writer/LLVMBuild.txt | 2 +- lib/LTO/ThinLTOCodeGenerator.cpp | 4 +- lib/Transforms/IPO/FunctionImport.cpp | 7 +- test/Bitcode/thinlto-function-summary.ll | 4 +- test/Transforms/FunctionImport/funcimport.ll | 8 +- tools/llvm-as/CMakeLists.txt | 1 + tools/llvm-as/LLVMBuild.txt | 2 +- tools/llvm-as/llvm-as.cpp | 12 +- 17 files changed, 434 insertions(+), 183 deletions(-) create mode 100644 include/llvm/Analysis/ModuleSummaryAnalysis.h create mode 100644 lib/Analysis/ModuleSummaryAnalysis.cpp diff --git a/include/llvm/Analysis/ModuleSummaryAnalysis.h b/include/llvm/Analysis/ModuleSummaryAnalysis.h new file mode 100644 index 00000000000..89178e117cf --- /dev/null +++ b/include/llvm/Analysis/ModuleSummaryAnalysis.h @@ -0,0 +1,86 @@ +//===- ModuleSummaryAnalysis.h - Module summary index builder ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This is the interface to build a ModuleSummaryIndex for a module. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_MODULESUMMARYANALYSIS_H +#define LLVM_ANALYSIS_MODULESUMMARYANALYSIS_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/Pass.h" + +namespace llvm { + +class BlockFrequencyInfo; + +/// Class to build a module summary index for the given Module, possibly from +/// a Pass. +class ModuleSummaryIndexBuilder { + /// The index being built + std::unique_ptr Index; + /// The module for which we are building an index + const Module *M; + +public: + /// Default constructor + ModuleSummaryIndexBuilder() = default; + + /// Constructor that builds an index for the given Module. An optional + /// callback can be supplied to obtain the frequency info for a function. + ModuleSummaryIndexBuilder( + const Module *M, + std::function Ftor = nullptr); + + /// Get a reference to the index owned by builder + ModuleSummaryIndex &getIndex() const { return *Index; } + + /// Take ownership of the built index + std::unique_ptr takeIndex() { return std::move(Index); } + +private: + /// Compute info for given function with optional frequency information + void computeFunctionInfo(const Function &F, + BlockFrequencyInfo *BFI = nullptr); + + /// Compute info for given variable with optional frequency information + void computeVariableInfo(const GlobalVariable &V); +}; + +/// Legacy wrapper pass to provide the ModuleSummaryIndex object. +class ModuleSummaryIndexWrapperPass : public ModulePass { + std::unique_ptr IndexBuilder; + +public: + static char ID; + + ModuleSummaryIndexWrapperPass(); + + /// Get the index built by pass + ModuleSummaryIndex &getIndex() { return IndexBuilder->getIndex(); } + const ModuleSummaryIndex &getIndex() const { + return IndexBuilder->getIndex(); + } + + bool runOnModule(Module &M) override; + bool doFinalization(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override; +}; + +//===--------------------------------------------------------------------===// +// +// createModuleSummaryIndexWrapperPass - This pass builds a ModuleSummaryIndex +// object for the module, to be written to bitcode or LLVM assembly. +// +ModulePass *createModuleSummaryIndexWrapperPass(); +} + +#endif diff --git a/include/llvm/Bitcode/ReaderWriter.h b/include/llvm/Bitcode/ReaderWriter.h index 1afffa05527..4121b4c9a3b 100644 --- a/include/llvm/Bitcode/ReaderWriter.h +++ b/include/llvm/Bitcode/ReaderWriter.h @@ -107,7 +107,7 @@ namespace llvm { /// for use in ThinLTO optimization). void WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder = false, - bool EmitSummaryIndex = false, + const ModuleSummaryIndex *Index = nullptr, bool GenerateHash = false); /// Write the specified module summary index to the given raw output stream, diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h index 681ebd4dc60..484375d9241 100644 --- a/include/llvm/IR/ModuleSummaryIndex.h +++ b/include/llvm/IR/ModuleSummaryIndex.h @@ -18,6 +18,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/Function.h" @@ -46,6 +47,43 @@ struct CalleeInfo { } }; +/// Struct to hold value either by GUID or Value*, depending on whether this +/// is a combined or per-module index, respectively. +struct ValueInfo { + /// The value representation used in this instance. + enum ValueInfoKind { + VI_GUID, + VI_Value, + }; + + /// Union of the two possible value types. + union ValueUnion { + GlobalValue::GUID Id; + const Value *V; + ValueUnion(GlobalValue::GUID Id) : Id(Id) {} + ValueUnion(const Value *V) : V(V) {} + }; + + /// The value being represented. + ValueUnion TheValue; + /// The value representation. + ValueInfoKind Kind; + /// Constructor for a GUID value + ValueInfo(GlobalValue::GUID Id = 0) : TheValue(Id), Kind(VI_GUID) {} + /// Constructor for a Value* value + ValueInfo(const Value *V) : TheValue(V), Kind(VI_Value) {} + /// Accessor for GUID value + GlobalValue::GUID getGUID() const { + assert(Kind == VI_GUID && "Not a GUID type"); + return TheValue.Id; + } + /// Accessor for Value* value + const Value *getValue() const { + assert(Kind == VI_Value && "Not a Value type"); + return TheValue.V; + } +}; + /// \brief Function and variable summary information to aid decisions and /// implementation of importing. /// @@ -78,11 +116,11 @@ private: /// types based on global summary-based analysis. GlobalValue::LinkageTypes Linkage; - /// List of GUIDs of values referenced by this global value's definition + /// List of values referenced by this global value's definition /// (either by the initializer of a global variable, or referenced /// from within a function). This does not include functions called, which /// are listed in the derived FunctionSummary object. - std::vector RefEdgeList; + std::vector RefEdgeList; protected: /// GlobalValueSummary constructor. @@ -109,31 +147,35 @@ public: /// by \p RefGUID. void addRefEdge(GlobalValue::GUID RefGUID) { RefEdgeList.push_back(RefGUID); } + /// Record a reference from this global value to the global value identified + /// by \p RefV. + void addRefEdge(const Value *RefV) { RefEdgeList.push_back(RefV); } + /// Record a reference from this global value to each global value identified /// in \p RefEdges. - void addRefEdges(DenseSet &RefEdges) { + void addRefEdges(DenseSet &RefEdges) { for (auto &RI : RefEdges) addRefEdge(RI); } - /// Return the list of GUIDs referenced by this global value definition. - std::vector &refs() { return RefEdgeList; } - const std::vector &refs() const { return RefEdgeList; } + /// Return the list of values referenced by this global value definition. + std::vector &refs() { return RefEdgeList; } + const std::vector &refs() const { return RefEdgeList; } }; /// \brief Function summary information to aid decisions and implementation of /// importing. class FunctionSummary : public GlobalValueSummary { public: - /// call edge pair. - typedef std::pair EdgeTy; + /// call edge pair. + typedef std::pair EdgeTy; private: /// Number of instructions (ignoring debug instructions, e.g.) computed /// during the initial compile step when the summary index is first built. unsigned InstCount; - /// List of call edge pairs from this function. + /// List of call edge pairs from this function. std::vector CallGraphEdgeList; public: @@ -156,14 +198,21 @@ public: CallGraphEdgeList.push_back(std::make_pair(CalleeGUID, Info)); } + /// Record a call graph edge from this function to the function identified + /// by \p CalleeV, with \p CalleeInfo including the cumulative profile + /// count (across all calls from this function) or 0 if no PGO. + void addCallGraphEdge(const Value *CalleeV, CalleeInfo Info) { + CallGraphEdgeList.push_back(std::make_pair(CalleeV, Info)); + } + /// Record a call graph edge from this function to each function recorded /// in \p CallGraphEdges. - void addCallGraphEdges(DenseMap &CallGraphEdges) { + void addCallGraphEdges(DenseMap &CallGraphEdges) { for (auto &EI : CallGraphEdges) addCallGraphEdge(EI.first, EI.second); } - /// Return the list of pairs. + /// Return the list of pairs. std::vector &calls() { return CallGraphEdgeList; } const std::vector &calls() const { return CallGraphEdgeList; } }; diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index b19cffb9925..924b79cfb3a 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -198,6 +198,7 @@ void initializeMachineBlockPlacementPass(PassRegistry&); void initializeMachineBlockPlacementStatsPass(PassRegistry&); void initializeMachineBranchProbabilityInfoPass(PassRegistry&); void initializeMachineCSEPass(PassRegistry&); +void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &); void initializeImplicitNullChecksPass(PassRegistry&); void initializeMachineDominatorTreePass(PassRegistry&); void initializeMachineDominanceFrontierPass(PassRegistry&); @@ -314,6 +315,7 @@ void initializeMachineCombinerPass(PassRegistry &); void initializeLoadCombinePass(PassRegistry&); void initializeRewriteSymbolsPass(PassRegistry&); void initializeWinEHPreparePass(PassRegistry&); +void initializeWriteBitcodePassPass(PassRegistry &); void initializePlaceBackedgeSafepointsImplPass(PassRegistry&); void initializePlaceSafepointsPass(PassRegistry&); void initializeDwarfEHPreparePass(PassRegistry&); diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp index 77d89dcd7b5..63292b2464a 100644 --- a/lib/Analysis/Analysis.cpp +++ b/lib/Analysis/Analysis.cpp @@ -60,6 +60,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) { initializeMemDerefPrinterPass(Registry); initializeMemoryDependenceWrapperPassPass(Registry); initializeModuleDebugInfoPrinterPass(Registry); + initializeModuleSummaryIndexWrapperPassPass(Registry); initializeObjCARCAAWrapperPassPass(Registry); initializePostDominatorTreeWrapperPassPass(Registry); initializeRegionInfoPassPass(Registry); diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt index 72669289b08..0ec27443d8a 100644 --- a/lib/Analysis/CMakeLists.txt +++ b/lib/Analysis/CMakeLists.txt @@ -49,6 +49,7 @@ add_llvm_library(LLVMAnalysis MemoryDependenceAnalysis.cpp MemoryLocation.cpp ModuleDebugInfoPrinter.cpp + ModuleSummaryAnalysis.cpp ObjCARCAliasAnalysis.cpp ObjCARCAnalysisUtils.cpp ObjCARCInstKind.cpp diff --git a/lib/Analysis/ModuleSummaryAnalysis.cpp b/lib/Analysis/ModuleSummaryAnalysis.cpp new file mode 100644 index 00000000000..5c8b0aab193 --- /dev/null +++ b/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -0,0 +1,186 @@ +//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass builds a ModuleSummaryIndex object for the module, to be written +// to bitcode or LLVM assembly. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/ModuleSummaryAnalysis.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BlockFrequencyInfoImpl.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/ValueSymbolTable.h" +#include "llvm/Pass.h" +using namespace llvm; + +#define DEBUG_TYPE "module-summary-analysis" + +// Walk through the operands of a given User via worklist iteration and populate +// the set of GlobalValue references encountered. Invoked either on an +// Instruction or a GlobalVariable (which walks its initializer). +static void findRefEdges(const User *CurUser, DenseSet &RefEdges, + SmallPtrSet &Visited) { + SmallVector Worklist; + Worklist.push_back(CurUser); + + while (!Worklist.empty()) { + const User *U = Worklist.pop_back_val(); + + if (!Visited.insert(U).second) + continue; + + ImmutableCallSite CS(U); + + for (const auto &OI : U->operands()) { + const User *Operand = dyn_cast(OI); + if (!Operand) + continue; + if (isa(Operand)) + continue; + if (isa(Operand)) { + // We have a reference to a global value. This should be added to + // the reference set unless it is a callee. Callees are handled + // specially by WriteFunction and are added to a separate list. + if (!(CS && CS.isCallee(&OI))) + RefEdges.insert(Operand); + continue; + } + Worklist.push_back(Operand); + } + } +} + +void ModuleSummaryIndexBuilder::computeFunctionInfo(const Function &F, + BlockFrequencyInfo *BFI) { + // Summary not currently supported for anonymous functions, they must + // be renamed. + if (!F.hasName()) + return; + + unsigned NumInsts = 0; + // Map from callee ValueId to profile count. Used to accumulate profile + // counts for all static calls to a given callee. + DenseMap CallGraphEdges; + DenseSet RefEdges; + + SmallPtrSet Visited; + for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) + for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; + ++I) { + if (!isa(I)) + ++NumInsts; + + if (auto CS = ImmutableCallSite(&*I)) { + auto *CalledFunction = CS.getCalledFunction(); + if (CalledFunction && CalledFunction->hasName() && + !CalledFunction->isIntrinsic()) { + auto ScaledCount = BFI ? BFI->getBlockProfileCount(&*BB) : None; + auto *CalleeId = + M->getValueSymbolTable().lookup(CalledFunction->getName()); + CallGraphEdges[CalleeId] += + (ScaledCount ? ScaledCount.getValue() : 0); + } + } + findRefEdges(&*I, RefEdges, Visited); + } + + std::unique_ptr FuncSummary = + llvm::make_unique(F.getLinkage(), NumInsts); + FuncSummary->addCallGraphEdges(CallGraphEdges); + FuncSummary->addRefEdges(RefEdges); + std::unique_ptr GVInfo = + llvm::make_unique(0, std::move(FuncSummary)); + Index->addGlobalValueInfo(F.getName(), std::move(GVInfo)); +} + +void ModuleSummaryIndexBuilder::computeVariableInfo(const GlobalVariable &V) { + DenseSet RefEdges; + SmallPtrSet Visited; + findRefEdges(&V, RefEdges, Visited); + std::unique_ptr GVarSummary = + llvm::make_unique(V.getLinkage()); + GVarSummary->addRefEdges(RefEdges); + std::unique_ptr GVInfo = + llvm::make_unique(0, std::move(GVarSummary)); + Index->addGlobalValueInfo(V.getName(), std::move(GVInfo)); +} + +ModuleSummaryIndexBuilder::ModuleSummaryIndexBuilder( + const Module *M, + std::function Ftor) + : Index(llvm::make_unique()), M(M) { + // Compute summaries for all functions defined in module, and save in the + // index. + for (auto &F : *M) { + if (F.isDeclaration()) + continue; + + BlockFrequencyInfo *BFI = nullptr; + std::unique_ptr BFIPtr; + if (Ftor) + BFI = Ftor(F); + else if (F.getEntryCount().hasValue()) { + LoopInfo LI{DominatorTree(const_cast(F))}; + BranchProbabilityInfo BPI{F, LI}; + BFIPtr = llvm::make_unique(F, BPI, LI); + BFI = BFIPtr.get(); + } + + computeFunctionInfo(F, BFI); + } + + // Compute summaries for all variables defined in module, and save in the + // index. + for (const GlobalVariable &G : M->globals()) { + if (G.isDeclaration()) + continue; + computeVariableInfo(G); + } +} + +char ModuleSummaryIndexWrapperPass::ID = 0; +INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis", + "Module Summary Analysis", false, true) +INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) +INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis", + "Module Summary Analysis", false, true) + +ModulePass *llvm::createModuleSummaryIndexWrapperPass() { + return new ModuleSummaryIndexWrapperPass(); +} + +ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass() + : ModulePass(ID) { + initializeModuleSummaryIndexWrapperPassPass(*PassRegistry::getPassRegistry()); +} + +bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) { + IndexBuilder = llvm::make_unique( + &M, [this](const Function &F) { + return &(this->getAnalysis( + *const_cast(&F)) + .getBFI()); + }); + return false; +} + +bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) { + IndexBuilder.reset(); + return false; +} + +void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); + AU.addRequired(); +} diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index d9a4de5ddbd..262dad652fc 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -13,12 +13,7 @@ #include "ValueEnumerator.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/BlockFrequencyInfoImpl.h" -#include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/LoopInfo.h" #include "llvm/Bitcode/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" #include "llvm/Bitcode/ReaderWriter.h" @@ -26,10 +21,8 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Dominators.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" @@ -2282,8 +2275,7 @@ static void WriteValueSymbolTable( const ValueSymbolTable &VST, const ValueEnumerator &VE, BitstreamWriter &Stream, uint64_t VSTOffsetPlaceholder = 0, uint64_t BitcodeStartBit = 0, - DenseMap> * - GlobalValueIndex = nullptr) { + DenseMap *FunctionToBitcodeIndex = nullptr) { if (VST.empty()) { // WriteValueSymbolTableForwardDecl should have returned early as // well. Ensure this handling remains in sync by asserting that @@ -2372,13 +2364,12 @@ static void WriteValueSymbolTable( // Must be the module-level VST, where we pass in the Index and // have a VSTOffsetPlaceholder. The function-level VST should not // contain any Function symbols. - assert(GlobalValueIndex); + assert(FunctionToBitcodeIndex); assert(VSTOffsetPlaceholder > 0); // Save the word offset of the function (from the start of the // actual bitcode written to the stream). - uint64_t BitcodeIndex = - (*GlobalValueIndex)[F]->bitcodeIndex() - BitcodeStartBit; + uint64_t BitcodeIndex = (*FunctionToBitcodeIndex)[F] - BitcodeStartBit; assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned"); NameVals.push_back(BitcodeIndex / 32); @@ -2500,61 +2491,14 @@ static void WriteUseListBlock(const Function *F, ValueEnumerator &VE, Stream.ExitBlock(); } -// Walk through the operands of a given User via worklist iteration and populate -// the set of GlobalValue references encountered. Invoked either on an -// Instruction or a GlobalVariable (which walks its initializer). -static void findRefEdges(const User *CurUser, const ValueEnumerator &VE, - DenseSet &RefEdges, - SmallPtrSet &Visited) { - SmallVector Worklist; - Worklist.push_back(CurUser); - - while (!Worklist.empty()) { - const User *U = Worklist.pop_back_val(); - - if (!Visited.insert(U).second) - continue; - - ImmutableCallSite CS(U); - - for (const auto &OI : U->operands()) { - const User *Operand = dyn_cast(OI); - if (!Operand) - continue; - if (isa(Operand)) - continue; - if (isa(Operand)) { - // We have a reference to a global value. This should be added to - // the reference set unless it is a callee. Callees are handled - // specially by WriteFunction and are added to a separate list. - if (!(CS && CS.isCallee(&OI))) - RefEdges.insert(VE.getValueID(Operand)); - continue; - } - Worklist.push_back(Operand); - } - } -} - /// Emit a function body to the module stream. static void WriteFunction(const Function &F, const Module *M, ValueEnumerator &VE, BitstreamWriter &Stream, - DenseMap> & - GlobalValueIndex, - bool EmitSummaryIndex) { + DenseMap &FunctionToBitcodeIndex) { // Save the bitcode index of the start of this function block for recording // in the VST. - uint64_t BitcodeIndex = Stream.GetCurrentBitNo(); - - bool HasProfileData = F.getEntryCount().hasValue(); - std::unique_ptr BFI; - if (EmitSummaryIndex && HasProfileData) { - Function &Func = const_cast(F); - LoopInfo LI{DominatorTree(Func)}; - BranchProbabilityInfo BPI{Func, LI}; - BFI = llvm::make_unique(Func, BPI, LI); - } + FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo(); Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4); VE.incorporateFunction(F); @@ -2581,40 +2525,15 @@ WriteFunction(const Function &F, const Module *M, ValueEnumerator &VE, bool NeedsMetadataAttachment = F.hasMetadata(); DILocation *LastDL = nullptr; - unsigned NumInsts = 0; - // Map from callee ValueId to profile count. Used to accumulate profile - // counts for all static calls to a given callee. - DenseMap CallGraphEdges; - DenseSet RefEdges; - - SmallPtrSet Visited; // Finally, emit all the instructions, in order. for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB) for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) { WriteInstruction(*I, InstID, VE, Stream, Vals); - if (!isa(I)) - ++NumInsts; - if (!I->getType()->isVoidTy()) ++InstID; - if (EmitSummaryIndex) { - if (auto CS = ImmutableCallSite(&*I)) { - auto *CalledFunction = CS.getCalledFunction(); - if (CalledFunction && CalledFunction->hasName() && - !CalledFunction->isIntrinsic()) { - auto ScaledCount = BFI ? BFI->getBlockProfileCount(&*BB) : None; - unsigned CalleeId = VE.getValueID( - M->getValueSymbolTable().lookup(CalledFunction->getName())); - CallGraphEdges[CalleeId] += - (ScaledCount ? ScaledCount.getValue() : 0); - } - } - findRefEdges(&*I, VE, RefEdges, Visited); - } - // If the instruction has metadata, write a metadata attachment later. NeedsMetadataAttachment |= I->hasMetadataOtherThanDebugLoc(); @@ -2639,15 +2558,6 @@ WriteFunction(const Function &F, const Module *M, ValueEnumerator &VE, LastDL = DL; } - std::unique_ptr FuncSummary; - if (EmitSummaryIndex) { - FuncSummary = llvm::make_unique(F.getLinkage(), NumInsts); - FuncSummary->addCallGraphEdges(CallGraphEdges); - FuncSummary->addRefEdges(RefEdges); - } - GlobalValueIndex[&F] = - llvm::make_unique(BitcodeIndex, std::move(FuncSummary)); - // Emit names for all the instructions etc. WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream); @@ -2915,21 +2825,22 @@ static void WriteModStrings(const ModuleSummaryIndex &I, // Helper to emit a single function summary record. static void WritePerModuleFunctionSummaryRecord( - SmallVector &NameVals, FunctionSummary *FS, unsigned ValueID, - unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, - BitstreamWriter &Stream, const Function &F) { - assert(FS); + SmallVector &NameVals, GlobalValueInfo *Info, + unsigned ValueID, const ValueEnumerator &VE, unsigned FSCallsAbbrev, + unsigned FSCallsProfileAbbrev, BitstreamWriter &Stream, const Function &F) { NameVals.push_back(ValueID); + + FunctionSummary *FS = cast(Info->summary()); NameVals.push_back(getEncodedLinkage(FS->linkage())); NameVals.push_back(FS->instCount()); NameVals.push_back(FS->refs().size()); for (auto &RI : FS->refs()) - NameVals.push_back(RI); + NameVals.push_back(VE.getValueID(RI.getValue())); bool HasProfileData = F.getEntryCount().hasValue(); for (auto &ECI : FS->calls()) { - NameVals.push_back(ECI.first); + NameVals.push_back(VE.getValueID(ECI.first.getValue())); assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite"); NameVals.push_back(ECI.second.CallsiteCount); if (HasProfileData) @@ -2948,6 +2859,7 @@ static void WritePerModuleFunctionSummaryRecord( // Collect the global value references in the given variable's initializer, // and emit them in a summary record. static void WriteModuleLevelReferences(const GlobalVariable &V, + const ModuleSummaryIndex &Index, const ValueEnumerator &VE, SmallVector &NameVals, unsigned FSModRefsAbbrev, @@ -2955,14 +2867,12 @@ static void WriteModuleLevelReferences(const GlobalVariable &V, // Only interested in recording variable defs in the summary. if (V.isDeclaration()) return; - DenseSet RefEdges; - SmallPtrSet Visited; - findRefEdges(&V, VE, RefEdges, Visited); NameVals.push_back(VE.getValueID(&V)); NameVals.push_back(getEncodedLinkage(V.getLinkage())); - for (auto RefId : RefEdges) { - NameVals.push_back(RefId); - } + auto *Info = Index.getGlobalValueInfo(V); + GlobalVarSummary *VS = cast(Info->summary()); + for (auto Ref : VS->refs()) + NameVals.push_back(VE.getValueID(Ref.getValue())); Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, FSModRefsAbbrev); NameVals.clear(); @@ -2970,10 +2880,10 @@ static void WriteModuleLevelReferences(const GlobalVariable &V, /// Emit the per-module summary section alongside the rest of /// the module's bitcode. -static void WritePerModuleGlobalValueSummary( - DenseMap> & - GlobalValueIndex, - const Module *M, const ValueEnumerator &VE, BitstreamWriter &Stream) { +static void WritePerModuleGlobalValueSummary(const Module *M, + const ModuleSummaryIndex &Index, + const ValueEnumerator &VE, + BitstreamWriter &Stream) { if (M->empty()) return; @@ -3013,7 +2923,7 @@ static void WritePerModuleGlobalValueSummary( unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv); SmallVector NameVals; - // Iterate over the list of functions instead of the GlobalValueIndex map to + // Iterate over the list of functions instead of the Index to // ensure the ordering is stable. for (const Function &F : *M) { if (F.isDeclaration()) @@ -3023,39 +2933,17 @@ static void WritePerModuleGlobalValueSummary( if (!F.hasName()) continue; - assert(GlobalValueIndex.count(&F) == 1); - + auto *Info = Index.getGlobalValueInfo(F); WritePerModuleFunctionSummaryRecord( - NameVals, cast(GlobalValueIndex[&F]->summary()), - VE.getValueID(M->getValueSymbolTable().lookup(F.getName())), + NameVals, Info, + VE.getValueID(M->getValueSymbolTable().lookup(F.getName())), VE, FSCallsAbbrev, FSCallsProfileAbbrev, Stream, F); } - for (const GlobalAlias &A : M->aliases()) { - if (!A.getBaseObject()) - continue; - const Function *F = dyn_cast(A.getBaseObject()); - if (!F || F->isDeclaration()) - continue; - - assert(GlobalValueIndex.count(F) == 1); - FunctionSummary *FS = cast(GlobalValueIndex[F]->summary()); - // Add the alias to the reference list of aliasee function. - FS->addRefEdge( - VE.getValueID(M->getValueSymbolTable().lookup(A.getName()))); - WritePerModuleFunctionSummaryRecord( - NameVals, FS, - VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), - FSCallsAbbrev, FSCallsProfileAbbrev, Stream, *F); - } - // Capture references from GlobalVariable initializers, which are outside // of a function scope. for (const GlobalVariable &G : M->globals()) - WriteModuleLevelReferences(G, VE, NameVals, FSModRefsAbbrev, Stream); - for (const GlobalAlias &A : M->aliases()) - if (auto *GV = dyn_cast(A.getBaseObject())) - WriteModuleLevelReferences(*GV, VE, NameVals, FSModRefsAbbrev, Stream); + WriteModuleLevelReferences(G, Index, VE, NameVals, FSModRefsAbbrev, Stream); Stream.ExitBlock(); } @@ -3110,11 +2998,11 @@ static void WriteCombinedGlobalValueSummary( NameVals.push_back(Index.getModuleId(VS->modulePath())); NameVals.push_back(getEncodedLinkage(VS->linkage())); for (auto &RI : VS->refs()) { - const auto &VMI = GUIDToValueIdMap.find(RI); + const auto &VMI = GUIDToValueIdMap.find(RI.getGUID()); unsigned RefId; // If this GUID doesn't have an entry, assign one. if (VMI == GUIDToValueIdMap.end()) { - GUIDToValueIdMap[RI] = ++GlobalValueId; + GUIDToValueIdMap[RI.getGUID()] = ++GlobalValueId; RefId = GlobalValueId; } else { RefId = VMI->second; @@ -3142,11 +3030,11 @@ static void WriteCombinedGlobalValueSummary( NameVals.push_back(FS->refs().size()); for (auto &RI : FS->refs()) { - const auto &VMI = GUIDToValueIdMap.find(RI); + const auto &VMI = GUIDToValueIdMap.find(RI.getGUID()); unsigned RefId; // If this GUID doesn't have an entry, assign one. if (VMI == GUIDToValueIdMap.end()) { - GUIDToValueIdMap[RI] = ++GlobalValueId; + GUIDToValueIdMap[RI.getGUID()] = ++GlobalValueId; RefId = GlobalValueId; } else { RefId = VMI->second; @@ -3162,7 +3050,7 @@ static void WriteCombinedGlobalValueSummary( } for (auto &EI : FS->calls()) { - const auto &VMI = GUIDToValueIdMap.find(EI.first); + const auto &VMI = GUIDToValueIdMap.find(EI.first.getGUID()); // If this GUID doesn't have an entry, it doesn't have a function // summary and we don't need to record any calls to it. if (VMI == GUIDToValueIdMap.end()) @@ -3243,8 +3131,9 @@ static void writeModuleHash(BitstreamWriter &Stream, /// WriteModule - Emit the specified module to the bitstream. static void WriteModule(const Module *M, BitstreamWriter &Stream, bool ShouldPreserveUseListOrder, - uint64_t BitcodeStartBit, bool EmitSummaryIndex, - bool GenerateHash, SmallVectorImpl &Buffer) { + uint64_t BitcodeStartBit, + const ModuleSummaryIndex *Index, bool GenerateHash, + SmallVectorImpl &Buffer) { Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3); size_t BlockStartPos = Buffer.size(); @@ -3290,19 +3179,19 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream, WriteOperandBundleTags(M, Stream); // Emit function bodies. - DenseMap> GlobalValueIndex; + DenseMap FunctionToBitcodeIndex; for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F) if (!F->isDeclaration()) - WriteFunction(*F, M, VE, Stream, GlobalValueIndex, EmitSummaryIndex); + WriteFunction(*F, M, VE, Stream, FunctionToBitcodeIndex); // Need to write after the above call to WriteFunction which populates // the summary information in the index. - if (EmitSummaryIndex) - WritePerModuleGlobalValueSummary(GlobalValueIndex, M, VE, Stream); + if (Index) + WritePerModuleGlobalValueSummary(M, *Index, VE, Stream); WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream, VSTOffsetPlaceholder, BitcodeStartBit, - &GlobalValueIndex); + &FunctionToBitcodeIndex); if (GenerateHash) { writeModuleHash(Stream, Buffer, BlockStartPos); @@ -3392,7 +3281,8 @@ static void WriteBitcodeHeader(BitstreamWriter &Stream) { /// stream. void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, bool ShouldPreserveUseListOrder, - bool EmitSummaryIndex, bool GenerateHash) { + const ModuleSummaryIndex *Index, + bool GenerateHash) { SmallVector Buffer; Buffer.reserve(256*1024); @@ -3417,8 +3307,8 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out, WriteIdentificationBlock(M, Stream); // Emit the module. - WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit, - EmitSummaryIndex, GenerateHash, Buffer); + WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit, Index, + GenerateHash, Buffer); } if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) diff --git a/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 7dbede4a847..2eeeb5d272c 100644 --- a/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" @@ -19,7 +20,11 @@ using namespace llvm; PreservedAnalyses BitcodeWriterPass::run(Module &M) { - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, EmitSummaryIndex, EmitModuleHash); + std::unique_ptr Index; + if (EmitSummaryIndex) + Index = ModuleSummaryIndexBuilder(&M).takeIndex(); + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index.get(), + EmitModuleHash); return PreservedAnalyses::all(); } @@ -32,22 +37,43 @@ namespace { public: static char ID; // Pass identification, replacement for typeid + WriteBitcodePass() : ModulePass(ID), OS(dbgs()) { + initializeWriteBitcodePassPass(*PassRegistry::getPassRegistry()); + } + explicit WriteBitcodePass(raw_ostream &o, bool ShouldPreserveUseListOrder, bool EmitSummaryIndex, bool EmitModuleHash) : ModulePass(ID), OS(o), ShouldPreserveUseListOrder(ShouldPreserveUseListOrder), - EmitSummaryIndex(EmitSummaryIndex), EmitModuleHash(EmitModuleHash) {} + EmitSummaryIndex(EmitSummaryIndex), EmitModuleHash(EmitModuleHash) { + initializeWriteBitcodePassPass(*PassRegistry::getPassRegistry()); + } const char *getPassName() const override { return "Bitcode Writer"; } bool runOnModule(Module &M) override { - WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, EmitSummaryIndex, EmitModuleHash); + const ModuleSummaryIndex *Index = + EmitSummaryIndex + ? &(getAnalysis().getIndex()) + : nullptr; + WriteBitcodeToFile(&M, OS, ShouldPreserveUseListOrder, Index, + EmitModuleHash); return false; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + if (EmitSummaryIndex) + AU.addRequired(); + } }; } char WriteBitcodePass::ID = 0; +INITIALIZE_PASS_BEGIN(WriteBitcodePass, "write-bitcode", "Write Bitcode", false, + true) +INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass) +INITIALIZE_PASS_END(WriteBitcodePass, "write-bitcode", "Write Bitcode", false, + true) ModulePass *llvm::createBitcodeWriterPass(raw_ostream &Str, bool ShouldPreserveUseListOrder, diff --git a/lib/Bitcode/Writer/LLVMBuild.txt b/lib/Bitcode/Writer/LLVMBuild.txt index a450b38fba2..7d9e1de771b 100644 --- a/lib/Bitcode/Writer/LLVMBuild.txt +++ b/lib/Bitcode/Writer/LLVMBuild.txt @@ -19,4 +19,4 @@ type = Library name = BitWriter parent = Bitcode -required_libraries = Analysis Core Support +required_libraries = Core Support diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp index b60668dc670..fea1c7fb8fa 100644 --- a/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/lib/LTO/ThinLTOCodeGenerator.cpp @@ -16,6 +16,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeWriterPass.h" @@ -326,7 +327,8 @@ ProcessThinLTOModule(Module &TheModule, const ModuleSummaryIndex &Index, SmallVector OutputBuffer; { raw_svector_ostream OS(OutputBuffer); - WriteBitcodeToFile(&TheModule, OS, true, true); + ModuleSummaryIndexBuilder IndexBuilder(&TheModule); + WriteBitcodeToFile(&TheModule, OS, true, &IndexBuilder.getIndex()); } return make_unique(std::move(OutputBuffer)); } diff --git a/lib/Transforms/IPO/FunctionImport.cpp b/lib/Transforms/IPO/FunctionImport.cpp index 32853b93f28..63b2e974419 100644 --- a/lib/Transforms/IPO/FunctionImport.cpp +++ b/lib/Transforms/IPO/FunctionImport.cpp @@ -145,7 +145,7 @@ static void computeImportForFunction( FunctionImporter::ImportMapTy &ImportsForModule, StringMap &ExportLists) { for (auto &Edge : Summary.calls()) { - auto GUID = Edge.first; + auto GUID = Edge.first.getGUID(); DEBUG(dbgs() << " edge -> " << GUID << " Threshold:" << Threshold << "\n"); if (DefinedFunctions.count(GUID)) { @@ -181,11 +181,12 @@ static void computeImportForFunction( // Mark all functions and globals referenced by this function as exported to // the outside if they are defined in the same source module. for (auto &Edge : CalleeSummary->calls()) { - auto CalleeGUID = Edge.first; + auto CalleeGUID = Edge.first.getGUID(); if (isGlobalExported(Index, ExportModulePath, CalleeGUID)) ExportList.insert(CalleeGUID); } - for (auto &GUID : CalleeSummary->refs()) { + for (auto &Ref : CalleeSummary->refs()) { + auto GUID = Ref.getGUID(); if (isGlobalExported(Index, ExportModulePath, GUID)) ExportList.insert(GUID); } diff --git a/test/Bitcode/thinlto-function-summary.ll b/test/Bitcode/thinlto-function-summary.ll index 511b1c31409..25afa437483 100644 --- a/test/Bitcode/thinlto-function-summary.ll +++ b/test/Bitcode/thinlto-function-summary.ll @@ -7,7 +7,6 @@ ; BC: record string = 'foo' @@ -37,6 +36,9 @@ entry: ret i32 %x } +; FIXME: Anonymous function and alias not currently in summary until +; follow on fixes to rename anonymous functions and emit alias summary +; entries are committed. ; Check an anonymous function as well, since in that case only the alias ; ends up in the value symbol table and having a summary. @f = alias void (), void ()* @0 ; [#uses=0] diff --git a/test/Transforms/FunctionImport/funcimport.ll b/test/Transforms/FunctionImport/funcimport.ll index 9d30b471022..0ea62021547 100644 --- a/test/Transforms/FunctionImport/funcimport.ll +++ b/test/Transforms/FunctionImport/funcimport.ll @@ -34,12 +34,10 @@ declare void @weakalias(...) #1 ; CHECK-DAG: declare void @analias declare void @analias(...) #1 +; FIXME: Add this checking back when follow on fix to add alias summary +; records is committed. ; Aliases import the aliasee function declare void @linkoncealias(...) #1 -; INSTLIMDEF-DAG: Import linkoncealias -; INSTLIMDEF-DAG: Import linkoncefunc -; CHECK-DAG: define linkonce_odr void @linkoncefunc() -; CHECK-DAG: @linkoncealias = alias void (...), bitcast (void ()* @linkoncefunc to void (...)* ; INSTLIMDEF-DAG: Import referencestatics ; INSTLIMDEF-DAG: define available_externally i32 @referencestatics(i32 %i) @@ -89,7 +87,7 @@ declare void @weakfunc(...) #1 ; INSTLIM5-DAG: declare hidden void @funcwithpersonality.llvm.2() ; INSTLIMDEF-DAG: Import globalfunc2 -; INSTLIMDEF-DAG: 11 function-import - Number of functions imported +; INSTLIMDEF-DAG: 9 function-import - Number of functions imported ; The actual GUID values will depend on path to test. ; GUID-DAG: GUID {{.*}} is weakalias diff --git a/tools/llvm-as/CMakeLists.txt b/tools/llvm-as/CMakeLists.txt index 1b2789a4de4..9b9027b7061 100644 --- a/tools/llvm-as/CMakeLists.txt +++ b/tools/llvm-as/CMakeLists.txt @@ -1,4 +1,5 @@ set(LLVM_LINK_COMPONENTS + Analysis AsmParser BitWriter Core diff --git a/tools/llvm-as/LLVMBuild.txt b/tools/llvm-as/LLVMBuild.txt index 542470bbdd8..cef557ac7a2 100644 --- a/tools/llvm-as/LLVMBuild.txt +++ b/tools/llvm-as/LLVMBuild.txt @@ -19,4 +19,4 @@ type = Tool name = llvm-as parent = Tools -required_libraries = AsmParser BitWriter +required_libraries = Analysis AsmParser BitWriter diff --git a/tools/llvm-as/llvm-as.cpp b/tools/llvm-as/llvm-as.cpp index 31010dbfac1..89397cde3ef 100644 --- a/tools/llvm-as/llvm-as.cpp +++ b/tools/llvm-as/llvm-as.cpp @@ -15,6 +15,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Analysis/ModuleSummaryAnalysis.h" #include "llvm/AsmParser/Parser.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/LLVMContext.h" @@ -83,9 +84,14 @@ static void WriteOutputFile(const Module *M) { exit(1); } - if (Force || !CheckBitcodeOutputToConsole(Out->os(), true)) - WriteBitcodeToFile(M, Out->os(), PreserveBitcodeUseListOrder, - EmitSummaryIndex, EmitModuleHash); + if (Force || !CheckBitcodeOutputToConsole(Out->os(), true)) { + std::unique_ptr Index; + if (EmitSummaryIndex) + Index = ModuleSummaryIndexBuilder(M).takeIndex(); + + WriteBitcodeToFile(M, Out->os(), PreserveBitcodeUseListOrder, Index.get(), + EmitModuleHash); + } // Declare success. Out->keep();