diff --git a/include/llvm/Transforms/PGOInstrumentation.h b/include/llvm/Transforms/PGOInstrumentation.h index 19263f0f807..fa7a68624ec 100644 --- a/include/llvm/Transforms/PGOInstrumentation.h +++ b/include/llvm/Transforms/PGOInstrumentation.h @@ -1,4 +1,4 @@ -//===- Transforms/PGOInstrumentation.h - PGO gen/use passes ---*- C++ -*-===// +//===- Transforms/PGOInstrumentation.h - PGO gen/use passes -----*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -6,19 +6,27 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file provides the interface for IR based instrumentation passes ( /// (profile-gen, and profile-use). +// //===----------------------------------------------------------------------===// #ifndef LLVM_TRANSFORMS_PGOINSTRUMENTATION_H #define LLVM_TRANSFORMS_PGOINSTRUMENTATION_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/IR/PassManager.h" -#include "llvm/Transforms/Instrumentation.h" +#include +#include namespace llvm { +class Function; +class Instruction; +class Module; + /// The instrumentation (profile-instr-gen) pass for IR based PGO. class PGOInstrumentationGen : public PassInfoMixin { public: @@ -28,9 +36,10 @@ public: /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin { public: - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); PGOInstrumentationUse(std::string Filename = ""); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + private: std::string ProfileFileName; }; @@ -40,6 +49,7 @@ class PGOIndirectCallPromotion : public PassInfoMixin public: PGOIndirectCallPromotion(bool IsInLTO = false, bool SamplePGO = false) : InLTO(IsInLTO), SamplePGO(SamplePGO) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: @@ -50,12 +60,14 @@ private: /// The profile size based optimization pass for memory intrinsics. class PGOMemOPSizeOpt : public PassInfoMixin { public: - PGOMemOPSizeOpt() {} + PGOMemOPSizeOpt() = default; + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; void setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, uint64_t MaxCount); -} // End llvm namespace -#endif +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_PGOINSTRUMENTATION_H diff --git a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index d74ccdfccb8..09bcbb28265 100644 --- a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -1,4 +1,4 @@ -//===-- DataFlowSanitizer.cpp - dynamic data flow analysis ----------------===// +//===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===// // // The LLVM Compiler Infrastructure // @@ -6,6 +6,7 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// /// \file /// This file is a part of DataFlowSanitizer, a generalised dynamic data flow /// analysis. @@ -43,32 +44,63 @@ /// /// For more information, please refer to the design document: /// http://clang.llvm.org/docs/DataFlowSanitizerDesign.html +// +//===----------------------------------------------------------------------===// #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/InstVisitor.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/IR/User.h" #include "llvm/IR/Value.h" #include "llvm/Pass.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SpecialCaseList.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Local.h" #include +#include +#include +#include #include +#include #include +#include #include +#include using namespace llvm; @@ -129,10 +161,7 @@ static cl::opt ClDebugNonzeroLabels( "load or return with a nonzero label"), cl::Hidden); - -namespace { - -StringRef GetGlobalTypeString(const GlobalValue &G) { +static StringRef GetGlobalTypeString(const GlobalValue &G) { // Types of GlobalVariables are always pointer types. Type *GType = G.getValueType(); // For now we support blacklisting struct types only. @@ -143,11 +172,13 @@ StringRef GetGlobalTypeString(const GlobalValue &G) { return ""; } +namespace { + class DFSanABIList { std::unique_ptr SCL; public: - DFSanABIList() {} + DFSanABIList() = default; void set(std::unique_ptr List) { SCL = std::move(List); } @@ -256,7 +287,7 @@ class DataFlowSanitizer : public ModulePass { DFSanABIList ABIList; DenseMap UnwrappedFnMap; AttrBuilder ReadOnlyNoneAttrs; - bool DFSanRuntimeShadowMask; + bool DFSanRuntimeShadowMask = false; Value *getShadowAddress(Value *Addr, Instruction *Pos); bool isInstrumented(const Function *F); @@ -272,11 +303,13 @@ class DataFlowSanitizer : public ModulePass { FunctionType *NewFT); Constant *getOrBuildTrampolineFunction(FunctionType *FT, StringRef FName); - public: +public: + static char ID; + DataFlowSanitizer( const std::vector &ABIListFiles = std::vector(), void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr); - static char ID; + bool doInitialization(Module &M) override; bool runOnModule(Module &M) override; }; @@ -287,12 +320,12 @@ struct DFSanFunction { DominatorTree DT; DataFlowSanitizer::InstrumentedABI IA; bool IsNativeABI; - Value *ArgTLSPtr; - Value *RetvalTLSPtr; - AllocaInst *LabelReturnAlloca; + Value *ArgTLSPtr = nullptr; + Value *RetvalTLSPtr = nullptr; + AllocaInst *LabelReturnAlloca = nullptr; DenseMap ValShadowMap; DenseMap AllocaShadowMap; - std::vector > PHIFixups; + std::vector> PHIFixups; DenseSet SkipInsts; std::vector NonZeroChecks; bool AvoidNewBlocks; @@ -306,14 +339,13 @@ struct DFSanFunction { DenseMap> ShadowElements; DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI) - : DFS(DFS), F(F), IA(DFS.getInstrumentedABI()), - IsNativeABI(IsNativeABI), ArgTLSPtr(nullptr), RetvalTLSPtr(nullptr), - LabelReturnAlloca(nullptr) { + : DFS(DFS), F(F), IA(DFS.getInstrumentedABI()), IsNativeABI(IsNativeABI) { DT.recalculate(*F); // FIXME: Need to track down the register allocator issue which causes poor // performance in pathological cases with large numbers of basic blocks. AvoidNewBlocks = F->size() > 1000; } + Value *getArgTLSPtr(); Value *getArgTLS(unsigned Index, Instruction *Pos); Value *getRetvalTLS(); @@ -328,8 +360,9 @@ struct DFSanFunction { }; class DFSanVisitor : public InstVisitor { - public: +public: DFSanFunction &DFSF; + DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {} const DataLayout &getDataLayout() const { @@ -337,7 +370,6 @@ class DFSanVisitor : public InstVisitor { } void visitOperandShadowInst(Instruction &I); - void visitBinaryOperator(BinaryOperator &BO); void visitCastInst(CastInst &CI); void visitCmpInst(CmpInst &CI); @@ -358,9 +390,10 @@ class DFSanVisitor : public InstVisitor { void visitMemTransferInst(MemTransferInst &I); }; -} +} // end anonymous namespace char DataFlowSanitizer::ID; + INITIALIZE_PASS(DataFlowSanitizer, "dfsan", "DataFlowSanitizer: dynamic data flow analysis.", false, false) @@ -374,8 +407,7 @@ llvm::createDataFlowSanitizerPass(const std::vector &ABIListFiles, DataFlowSanitizer::DataFlowSanitizer( const std::vector &ABIListFiles, void *(*getArgTLS)(), void *(*getRetValTLS)()) - : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS), - DFSanRuntimeShadowMask(false) { + : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS) { std::vector AllABIListFiles(std::move(ABIListFiles)); AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(), ClABIListFiles.end()); @@ -383,7 +415,7 @@ DataFlowSanitizer::DataFlowSanitizer( } FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) { - llvm::SmallVector ArgTypes(T->param_begin(), T->param_end()); + SmallVector ArgTypes(T->param_begin(), T->param_end()); ArgTypes.append(T->getNumParams(), ShadowTy); if (T->isVarArg()) ArgTypes.push_back(ShadowPtrTy); @@ -395,7 +427,7 @@ FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) { FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) { assert(!T->isVarArg()); - llvm::SmallVector ArgTypes; + SmallVector ArgTypes; ArgTypes.push_back(T->getPointerTo()); ArgTypes.append(T->param_begin(), T->param_end()); ArgTypes.append(T->getNumParams(), ShadowTy); @@ -406,7 +438,7 @@ FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) { } FunctionType *DataFlowSanitizer::getCustomFunctionType(FunctionType *T) { - llvm::SmallVector ArgTypes; + SmallVector ArgTypes; for (FunctionType::param_iterator i = T->param_begin(), e = T->param_end(); i != e; ++i) { FunctionType *FT; @@ -429,12 +461,12 @@ FunctionType *DataFlowSanitizer::getCustomFunctionType(FunctionType *T) { } bool DataFlowSanitizer::doInitialization(Module &M) { - llvm::Triple TargetTriple(M.getTargetTriple()); - bool IsX86_64 = TargetTriple.getArch() == llvm::Triple::x86_64; - bool IsMIPS64 = TargetTriple.getArch() == llvm::Triple::mips64 || - TargetTriple.getArch() == llvm::Triple::mips64el; - bool IsAArch64 = TargetTriple.getArch() == llvm::Triple::aarch64 || - TargetTriple.getArch() == llvm::Triple::aarch64_be; + Triple TargetTriple(M.getTargetTriple()); + bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64; + bool IsMIPS64 = TargetTriple.getArch() == Triple::mips64 || + TargetTriple.getArch() == Triple::mips64el; + bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64 || + TargetTriple.getArch() == Triple::aarch64_be; const DataLayout &DL = M.getDataLayout(); @@ -655,7 +687,7 @@ bool DataFlowSanitizer::runOnModule(Module &M) { DFSanVarargWrapperFnTy); std::vector FnsToInstrument; - llvm::SmallPtrSet FnsWithNativeABI; + SmallPtrSet FnsWithNativeABI; for (Function &i : M) { if (!i.isIntrinsic() && &i != DFSanUnionFn && @@ -798,11 +830,11 @@ bool DataFlowSanitizer::runOnModule(Module &M) { // DFSanVisitor may create new basic blocks, which confuses df_iterator. // Build a copy of the list before iterating over it. - llvm::SmallVector BBList(depth_first(&i->getEntryBlock())); + SmallVector BBList(depth_first(&i->getEntryBlock())); for (BasicBlock *i : BBList) { Instruction *Inst = &i->front(); - while (1) { + while (true) { // DFSanVisitor may split the current basic block, changing the current // instruction's next pointer and moving the next instruction to the // tail block from which we should continue. @@ -822,7 +854,7 @@ bool DataFlowSanitizer::runOnModule(Module &M) { // until we have visited every block. Therefore, the code that handles phi // nodes adds them to the PHIFixups list so that they can be properly // handled here. - for (std::vector >::iterator + for (std::vector>::iterator i = DFSF.PHIFixups.begin(), e = DFSF.PHIFixups.end(); i != e; ++i) { @@ -1046,8 +1078,7 @@ void DFSanVisitor::visitOperandShadowInst(Instruction &I) { Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, Instruction *Pos) { if (AllocaInst *AI = dyn_cast(Addr)) { - llvm::DenseMap::iterator i = - AllocaShadowMap.find(AI); + const auto i = AllocaShadowMap.find(AI); if (i != AllocaShadowMap.end()) { IRBuilder<> IRB(Pos); return IRB.CreateLoad(i->second); @@ -1188,8 +1219,7 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) { void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, uint64_t Align, Value *Shadow, Instruction *Pos) { if (AllocaInst *AI = dyn_cast(Addr)) { - llvm::DenseMap::iterator i = - AllocaShadowMap.find(AI); + const auto i = AllocaShadowMap.find(AI); if (i != AllocaShadowMap.end()) { IRBuilder<> IRB(Pos); IRB.CreateStore(Shadow, i->second); @@ -1410,24 +1440,21 @@ void DFSanVisitor::visitCallSite(CallSite CS) { if (i != DFSF.DFS.UnwrappedFnMap.end()) { Function *F = i->second; switch (DFSF.DFS.getWrapperKind(F)) { - case DataFlowSanitizer::WK_Warning: { + case DataFlowSanitizer::WK_Warning: CS.setCalledFunction(F); IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn, IRB.CreateGlobalStringPtr(F->getName())); DFSF.setShadow(CS.getInstruction(), DFSF.DFS.ZeroShadow); return; - } - case DataFlowSanitizer::WK_Discard: { + case DataFlowSanitizer::WK_Discard: CS.setCalledFunction(F); DFSF.setShadow(CS.getInstruction(), DFSF.DFS.ZeroShadow); return; - } - case DataFlowSanitizer::WK_Functional: { + case DataFlowSanitizer::WK_Functional: CS.setCalledFunction(F); visitOperandShadowInst(*CS.getInstruction()); return; - } - case DataFlowSanitizer::WK_Custom: { + case DataFlowSanitizer::WK_Custom: // Don't try to handle invokes of custom functions, it's too complicated. // Instead, invoke the dfsw$ wrapper, which will in turn call the __dfsw_ // wrapper. @@ -1527,7 +1554,6 @@ void DFSanVisitor::visitCallSite(CallSite CS) { } break; } - } } FunctionType *FT = cast( diff --git a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp index dcda44d1bd5..8b9bbb49955 100644 --- a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp +++ b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp @@ -1,4 +1,4 @@ -//===-- IndirectCallPromotion.cpp - Optimizations based on value profiling ===// +//===- IndirectCallPromotion.cpp - Optimizations based on value profiling -===// // // The LLVM Compiler Infrastructure // @@ -14,15 +14,15 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" -#include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/IndirectCallPromotionAnalysis.h" #include "llvm/Analysis/IndirectCallSiteVisitor.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DerivedTypes.h" @@ -36,20 +36,22 @@ #include "llvm/IR/MDBuilder.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" #include "llvm/Pass.h" -#include "llvm/PassRegistry.h" -#include "llvm/PassSupport.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include #include +#include +#include +#include #include using namespace llvm; @@ -112,6 +114,7 @@ static cl::opt cl::desc("Dump IR after transformation happens")); namespace { + class PGOIndirectCallPromotionLegacyPass : public ModulePass { public: static char ID; @@ -139,9 +142,11 @@ private: // the promoted direct call. bool SamplePGO; }; + } // end anonymous namespace char PGOIndirectCallPromotionLegacyPass::ID = 0; + INITIALIZE_PASS_BEGIN(PGOIndirectCallPromotionLegacyPass, "pgo-icall-prom", "Use PGO instrumentation profile to promote indirect " "calls to direct calls.", @@ -158,6 +163,7 @@ ModulePass *llvm::createPGOIndirectCallPromotionLegacyPass(bool InLTO, } namespace { + // The class for main data structure to promote indirect calls to conditional // direct calls. class ICallPromotionFunc { @@ -177,6 +183,7 @@ private: struct PromotionCandidate { Function *TargetFunction; uint64_t Count; + PromotionCandidate(Function *F, uint64_t C) : TargetFunction(F), Count(C) {} }; @@ -195,17 +202,16 @@ private: const std::vector &Candidates, uint64_t &TotalCount); - // Noncopyable - ICallPromotionFunc(const ICallPromotionFunc &other) = delete; - ICallPromotionFunc &operator=(const ICallPromotionFunc &other) = delete; - public: ICallPromotionFunc(Function &Func, Module *Modu, InstrProfSymtab *Symtab, bool SamplePGO, OptimizationRemarkEmitter &ORE) : F(Func), M(Modu), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {} + ICallPromotionFunc(const ICallPromotionFunc &) = delete; + ICallPromotionFunc &operator=(const ICallPromotionFunc &) = delete; bool processFunction(ProfileSummaryInfo *PSI); }; + } // end anonymous namespace bool llvm::isLegalToPromote(Instruction *Inst, Function *F, @@ -313,6 +319,7 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite( const char *Reason = nullptr; if (!isLegalToPromote(Inst, TargetFunction, &Reason)) { using namespace ore; + ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", Inst) << "Cannot promote indirect call to " @@ -613,6 +620,7 @@ Instruction *llvm::promoteIndirectCall(Instruction *Inst, DEBUG(dbgs() << *BB << *DirectCallBB << *IndirectCallBB << *MergeBB << "\n"); using namespace ore; + if (ORE) ORE->emit([&]() { return OptimizationRemark(DEBUG_TYPE, "Promoted", Inst) @@ -700,7 +708,7 @@ static bool promoteIndirectCalls(Module &M, ProfileSummaryInfo *PSI, AM->getResult(M).getManager(); ORE = &FAM.getResult(F); } else { - OwnedORE = make_unique(&F); + OwnedORE = llvm::make_unique(&F); ORE = OwnedORE.get(); } diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index f8223496997..11a43e803a9 100644 --- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -1,4 +1,4 @@ -//===-- PGOInstrumentation.cpp - MST-based PGO Instrumentation ------------===// +//===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===// // // The LLVM Compiler Infrastructure // @@ -50,37 +50,69 @@ #include "llvm/Transforms/PGOInstrumentation.h" #include "CFGMST.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" +#include "llvm/ADT/Twine.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/CFG.h" #include "llvm/Analysis/IndirectCallSiteVisitor.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/InstIterator.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/IR/ProfileSummary.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" #include "llvm/Pass.h" +#include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfReader.h" -#include "llvm/ProfileData/ProfileCommon.h" #include "llvm/Support/BranchProbability.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/DOTGraphTraits.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/GraphWriter.h" #include "llvm/Support/JamCRC.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include +#include +#include +#include +#include #include #include #include @@ -202,11 +234,9 @@ extern cl::opt PGOViewCounts; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name= extern cl::opt ViewBlockFreqFuncName; -namespace { - // Return a string describing the branch condition that can be // used in static branch probability heuristics: -std::string getBranchCondString(Instruction *TI) { +static std::string getBranchCondString(Instruction *TI) { BranchInst *BI = dyn_cast(TI); if (!BI || !BI->isConditional()) return std::string(); @@ -237,6 +267,8 @@ std::string getBranchCondString(Instruction *TI) { return result; } +namespace { + /// The select instruction visitor plays three roles specified /// by the mode. In \c VM_counting mode, it simply counts the number of /// select instructions. In \c VM_instrument mode, it inserts code to count @@ -263,6 +295,7 @@ struct SelectInstVisitor : public InstVisitor { Mode = VM_counting; visit(Func); } + // Visit the IR stream and instrument all select instructions. \p // Ind is a pointer to the counter index variable; \p TotalNC // is the total number of counters; \p FNV is the pointer to the @@ -287,8 +320,10 @@ struct SelectInstVisitor : public InstVisitor { void instrumentOneSelectInst(SelectInst &SI); void annotateOneSelectInst(SelectInst &SI); + // Visit \p SI instruction and perform tasks according to visit mode. void visitSelectInst(SelectInst &SI); + // Return the number of select instructions. This needs be called after // countSelects(). unsigned getNumOfSelectInsts() const { return NSIs; } @@ -332,8 +367,10 @@ struct MemIntrinsicVisitor : public InstVisitor { // Visit the IR stream and annotate all mem intrinsic call instructions. void instrumentOneMemIntrinsic(MemIntrinsic &MI); + // Visit \p MI instruction and perform tasks according to visit mode. void visitMemIntrinsic(MemIntrinsic &SI); + unsigned getNumOfMemIntrinsics() const { return NMemIs; } }; @@ -375,6 +412,7 @@ private: std::string ProfileFileName; bool runOnModule(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); } @@ -383,6 +421,7 @@ private: } // end anonymous namespace char PGOInstrumentationGenLegacyPass::ID = 0; + INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) @@ -395,6 +434,7 @@ ModulePass *llvm::createPGOInstrumentationGenLegacyPass() { } char PGOInstrumentationUseLegacyPass::ID = 0; + INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) @@ -407,6 +447,7 @@ ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) { } namespace { + /// \brief An MST based instrumentation for PGO /// /// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO @@ -417,12 +458,13 @@ struct PGOEdge { const BasicBlock *SrcBB; const BasicBlock *DestBB; uint64_t Weight; - bool InMST; - bool Removed; - bool IsCritical; + bool InMST = false; + bool Removed = false; + bool IsCritical = false; + PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1) - : SrcBB(Src), DestBB(Dest), Weight(W), InMST(false), Removed(false), - IsCritical(false) {} + : SrcBB(Src), DestBB(Dest), Weight(W) {} + // Return the information string of an edge. const std::string infoString() const { return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") + @@ -434,9 +476,9 @@ struct PGOEdge { struct BBInfo { BBInfo *Group; uint32_t Index; - uint32_t Rank; + uint32_t Rank = 0; - BBInfo(unsigned IX) : Group(this), Index(IX), Rank(0) {} + BBInfo(unsigned IX) : Group(this), Index(IX) {} // Return the information string of this object. const std::string infoString() const { @@ -448,19 +490,22 @@ struct BBInfo { template class FuncPGOInstrumentation { private: Function &F; - void computeCFGHash(); - void renameComdatFunction(); + // A map that stores the Comdat group in function F. std::unordered_multimap &ComdatMembers; + void computeCFGHash(); + void renameComdatFunction(); + public: std::vector> ValueSites; SelectInstVisitor SIVisitor; MemIntrinsicVisitor MIVisitor; std::string FuncName; GlobalVariable *FuncNameVar; + // CFG hash value for this function. - uint64_t FunctionHash; + uint64_t FunctionHash = 0; // The Minimum Spanning Tree of function CFG. CFGMST MST; @@ -487,8 +532,7 @@ public: bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr) : F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1), - SIVisitor(Func), MIVisitor(Func), FunctionHash(0), MST(F, BPI, BFI) { - + SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) { // This should be done before CFG hash computation. SIVisitor.countSelects(Func); MIVisitor.countMemIntrinsics(Func); @@ -499,7 +543,7 @@ public: FuncName = getPGOFuncName(F); computeCFGHash(); - if (ComdatMembers.size()) + if (!ComdatMembers.empty()) renameComdatFunction(); DEBUG(dumpInfo("after CFGMST")); @@ -527,6 +571,8 @@ public: } }; +} // end anonymous namespace + // Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index // value of each BB in the CFG. The higher 32 bits record the number of edges. template @@ -686,7 +732,7 @@ static void instrumentOneFunc( "Cannot get the Instrumentation point"); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment), - {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), + {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters), Builder.getInt32(I++)}); } @@ -710,7 +756,7 @@ static void instrumentOneFunc( "Cannot get the Instrumentation point"); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), - {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), + {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy), Builder.getInt64(FuncInfo.FunctionHash), Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()), Builder.getInt32(IPVK_IndirectCallTarget), @@ -723,12 +769,15 @@ static void instrumentOneFunc( F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash); } +namespace { + // This class represents a CFG edge in profile use compilation. struct PGOUseEdge : public PGOEdge { - bool CountValid; - uint64_t CountValue; + bool CountValid = false; + uint64_t CountValue = 0; + PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1) - : PGOEdge(Src, Dest, W), CountValid(false), CountValue(0) {} + : PGOEdge(Src, Dest, W) {} // Set edge count value void setEdgeCount(uint64_t Value) { @@ -745,22 +794,21 @@ struct PGOUseEdge : public PGOEdge { } }; -typedef SmallVector DirectEdges; +using DirectEdges = SmallVector; // This class stores the auxiliary information for each BB. struct UseBBInfo : public BBInfo { - uint64_t CountValue; + uint64_t CountValue = 0; bool CountValid; - int32_t UnknownCountInEdge; - int32_t UnknownCountOutEdge; + int32_t UnknownCountInEdge = 0; + int32_t UnknownCountOutEdge = 0; DirectEdges InEdges; DirectEdges OutEdges; - UseBBInfo(unsigned IX) - : BBInfo(IX), CountValue(0), CountValid(false), UnknownCountInEdge(0), - UnknownCountOutEdge(0) {} + + UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {} + UseBBInfo(unsigned IX, uint64_t C) - : BBInfo(IX), CountValue(C), CountValid(true), UnknownCountInEdge(0), - UnknownCountOutEdge(0) {} + : BBInfo(IX), CountValue(C), CountValid(true) {} // Set the profile count value for this BB. void setBBInfoCount(uint64_t Value) { @@ -776,6 +824,8 @@ struct UseBBInfo : public BBInfo { } }; +} // end anonymous namespace + // Sum up the count values for all the edges. static uint64_t sumEdgeCount(const ArrayRef Edges) { uint64_t Total = 0; @@ -787,6 +837,8 @@ static uint64_t sumEdgeCount(const ArrayRef Edges) { return Total; } +namespace { + class PGOUseFunc { public: PGOUseFunc(Function &Func, Module *Modu, @@ -794,7 +846,7 @@ public: BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr) : F(Func), M(Modu), FuncInfo(Func, ComdatMembers, false, BPI, BFI), - CountPosition(0), ProfileCountSize(0), FreqAttr(FFA_Normal) {} + FreqAttr(FFA_Normal) {} // Read counts for the instrumented BB from profile. bool readCounters(IndexedInstrProfReader *PGOReader); @@ -819,6 +871,7 @@ public: // Return the function hash. uint64_t getFuncHash() const { return FuncInfo.FunctionHash; } + // Return the profile record for this function; InstrProfRecord &getProfileRecord() { return ProfileRecord; } @@ -841,6 +894,7 @@ public: private: Function &F; Module *M; + // This member stores the shared information with class PGOGenFunc. FuncPGOInstrumentation FuncInfo; @@ -849,10 +903,10 @@ private: uint64_t ProgramMaxCount; // Position of counter that remains to be read. - uint32_t CountPosition; + uint32_t CountPosition = 0; // Total size of the profile count for this function. - uint32_t ProfileCountSize; + uint32_t ProfileCountSize = 0; // ProfileRecord for this function. InstrProfRecord ProfileRecord; @@ -887,11 +941,12 @@ private: } }; +} // end anonymous namespace + // Visit all the edges and assign the count value for the instrumented // edges and the BB. void PGOUseFunc::setInstrumentedCounts( const std::vector &CountFromProfile) { - assert(FuncInfo.getNumCounters() == CountFromProfile.size()); // Use a worklist as we will update the vector during the iteration. std::vector WorkList; @@ -1136,7 +1191,7 @@ void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) { auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step), - {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs), Builder.getInt32(*CurCtrIdx), Step}); ++(*CurCtrIdx); @@ -1191,7 +1246,7 @@ void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) { assert(!dyn_cast(Length)); Builder.CreateCall( Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile), - {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), + {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), Builder.getInt64(FuncHash), Builder.CreateZExtOrTrunc(Length, Int64Ty), Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)}); ++CurCtrId; @@ -1257,7 +1312,6 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) { ValueSiteIndex++; } } -} // end anonymous namespace // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime // aware this is an ir_level profile so it can set the version flag. @@ -1327,7 +1381,6 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) { PreservedAnalyses PGOInstrumentationGen::run(Module &M, ModuleAnalysisManager &AM) { - auto &FAM = AM.getResult(M).getManager(); auto LookupBPI = [&FAM](Function &F) { return &FAM.getResult(F); @@ -1428,12 +1481,12 @@ static bool annotateAllFunctions( // can affect the BranchProbabilityInfo of any callers, resulting in an // inconsistent MST between prof-gen and prof-use. for (auto &F : HotFunctions) { - F->addFnAttr(llvm::Attribute::InlineHint); + F->addFnAttr(Attribute::InlineHint); DEBUG(dbgs() << "Set inline attribute to function: " << F->getName() << "\n"); } for (auto &F : ColdFunctions) { - F->addFnAttr(llvm::Attribute::Cold); + F->addFnAttr(Attribute::Cold); DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n"); } return true; @@ -1477,9 +1530,19 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) { return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI); } -namespace llvm { -void setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, - uint64_t MaxCount) { +static std::string getSimpleNodeName(const BasicBlock *Node) { + if (!Node->getName().empty()) + return Node->getName(); + + std::string SimpleNodeName; + raw_string_ostream OS(SimpleNodeName); + Node->printAsOperand(OS, false); + return OS.str(); +} + +void llvm::setProfMetadata(Module *M, Instruction *TI, + ArrayRef EdgeCounts, + uint64_t MaxCount) { MDBuilder MDB(M->getContext()); assert(MaxCount > 0 && "Bad max count"); uint64_t Scale = calculateCountScale(MaxCount); @@ -1490,7 +1553,7 @@ void setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, DEBUG(dbgs() << "Weight is: "; for (const auto &W : Weights) { dbgs() << W << " "; } dbgs() << "\n";); - TI->setMetadata(llvm::LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); + TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); if (EmitBranchProbability) { std::string BrCondStr = getBranchCondString(TI); if (BrCondStr.empty()) @@ -1517,36 +1580,32 @@ void setProfMetadata(Module *M, Instruction *TI, ArrayRef EdgeCounts, } } +namespace llvm { + template <> struct GraphTraits { - typedef const BasicBlock *NodeRef; - typedef succ_const_iterator ChildIteratorType; - typedef pointer_iterator nodes_iterator; + using NodeRef = const BasicBlock *; + using ChildIteratorType = succ_const_iterator; + using nodes_iterator = pointer_iterator; static NodeRef getEntryNode(const PGOUseFunc *G) { return &G->getFunc().front(); } + static ChildIteratorType child_begin(const NodeRef N) { return succ_begin(N); } + static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); } + static nodes_iterator nodes_begin(const PGOUseFunc *G) { return nodes_iterator(G->getFunc().begin()); } + static nodes_iterator nodes_end(const PGOUseFunc *G) { return nodes_iterator(G->getFunc().end()); } }; -static std::string getSimpleNodeName(const BasicBlock *Node) { - if (!Node->getName().empty()) - return Node->getName(); - - std::string SimpleNodeName; - raw_string_ostream OS(SimpleNodeName); - Node->printAsOperand(OS, false); - return OS.str(); -} - template <> struct DOTGraphTraits : DefaultDOTGraphTraits { explicit DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} @@ -1586,4 +1645,5 @@ template <> struct DOTGraphTraits : DefaultDOTGraphTraits { return Result; } }; -} // namespace llvm + +} // end namespace llvm diff --git a/lib/Transforms/Utils/LowerSwitch.cpp b/lib/Transforms/Utils/LowerSwitch.cpp index 890afbc46e6..344cb35df98 100644 --- a/lib/Transforms/Utils/LowerSwitch.cpp +++ b/lib/Transforms/Utils/LowerSwitch.cpp @@ -13,46 +13,65 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Value.h" #include "llvm/Pass.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" -#include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h" #include +#include +#include +#include +#include +#include + using namespace llvm; #define DEBUG_TYPE "lower-switch" namespace { + struct IntRange { int64_t Low, High; }; - // Return true iff R is covered by Ranges. - static bool IsInRanges(const IntRange &R, - const std::vector &Ranges) { - // Note: Ranges must be sorted, non-overlapping and non-adjacent. - // Find the first range whose High field is >= R.High, - // then check if the Low field is <= R.Low. If so, we - // have a Range that covers R. - auto I = std::lower_bound( - Ranges.begin(), Ranges.end(), R, - [](const IntRange &A, const IntRange &B) { return A.High < B.High; }); - return I != Ranges.end() && I->Low <= R.Low; - } +} // end anonymous namespace + +// Return true iff R is covered by Ranges. +static bool IsInRanges(const IntRange &R, + const std::vector &Ranges) { + // Note: Ranges must be sorted, non-overlapping and non-adjacent. + + // Find the first range whose High field is >= R.High, + // then check if the Low field is <= R.Low. If so, we + // have a Range that covers R. + auto I = std::lower_bound( + Ranges.begin(), Ranges.end(), R, + [](const IntRange &A, const IntRange &B) { return A.High < B.High; }); + return I != Ranges.end() && I->Low <= R.Low; +} + +namespace { /// Replace all SwitchInst instructions with chained branch instructions. class LowerSwitch : public FunctionPass { public: - static char ID; // Pass identification, replacement for typeid + // Pass identification, replacement for typeid + static char ID; + LowerSwitch() : FunctionPass(ID) { initializeLowerSwitchPass(*PassRegistry::getPassRegistry()); } @@ -68,8 +87,9 @@ namespace { : Low(low), High(high), BB(bb) {} }; - typedef std::vector CaseVector; - typedef std::vector::iterator CaseItr; + using CaseVector = std::vector; + using CaseItr = std::vector::iterator; + private: void processSwitchInst(SwitchInst *SI, SmallPtrSetImpl &DeleteList); @@ -86,22 +106,24 @@ namespace { /// The comparison function for sorting the switch case values in the vector. /// WARNING: Case ranges should be disjoint! struct CaseCmp { - bool operator () (const LowerSwitch::CaseRange& C1, - const LowerSwitch::CaseRange& C2) { - + bool operator()(const LowerSwitch::CaseRange& C1, + const LowerSwitch::CaseRange& C2) { const ConstantInt* CI1 = cast(C1.Low); const ConstantInt* CI2 = cast(C2.High); return CI1->getValue().slt(CI2->getValue()); } }; -} + +} // end anonymous namespace char LowerSwitch::ID = 0; -INITIALIZE_PASS(LowerSwitch, "lowerswitch", - "Lower SwitchInst's to branches", false, false) // Publicly exposed interface to pass... char &llvm::LowerSwitchID = LowerSwitch::ID; + +INITIALIZE_PASS(LowerSwitch, "lowerswitch", + "Lower SwitchInst's to branches", false, false) + // createLowerSwitchPass - Interface to this file... FunctionPass *llvm::createLowerSwitchPass() { return new LowerSwitch(); @@ -136,6 +158,7 @@ bool LowerSwitch::runOnFunction(Function &F) { static raw_ostream& operator<<(raw_ostream &O, const LowerSwitch::CaseVector &C) LLVM_ATTRIBUTE_USED; + static raw_ostream& operator<<(raw_ostream &O, const LowerSwitch::CaseVector &C) { O << "["; @@ -186,7 +209,7 @@ static void fixPhis(BasicBlock *SuccBB, BasicBlock *OrigBB, BasicBlock *NewBB, } // Remove incoming values in the reverse order to prevent invalidating // *successive* index. - for (unsigned III : reverse(Indices)) + for (unsigned III : llvm::reverse(Indices)) PN->removeIncomingValue(III); } } @@ -294,8 +317,7 @@ LowerSwitch::switchConvert(CaseItr Begin, CaseItr End, ConstantInt *LowerBound, /// value, so the jump to the "default" branch is warranted. BasicBlock* LowerSwitch::newLeafBlock(CaseRange& Leaf, Value* Val, BasicBlock* OrigBlock, - BasicBlock* Default) -{ + BasicBlock* Default) { Function* F = OrigBlock->getParent(); BasicBlock* NewLeaf = BasicBlock::Create(Val->getContext(), "LeafBlock"); F->getBasicBlockList().insert(++OrigBlock->getIterator(), NewLeaf); @@ -442,7 +464,8 @@ void LowerSwitch::processSwitchInst(SwitchInst *SI, unsigned MaxPop = 0; BasicBlock *PopSucc = nullptr; - IntRange R = { INT64_MIN, INT64_MAX }; + IntRange R = {std::numeric_limits::min(), + std::numeric_limits::max()}; UnreachableRanges.push_back(R); for (const auto &I : Cases) { int64_t Low = I.Low->getSExtValue(); @@ -457,8 +480,8 @@ void LowerSwitch::processSwitchInst(SwitchInst *SI, assert(Low > LastRange.Low); LastRange.High = Low - 1; } - if (High != INT64_MAX) { - IntRange R = { High + 1, INT64_MAX }; + if (High != std::numeric_limits::max()) { + IntRange R = { High + 1, std::numeric_limits::max() }; UnreachableRanges.push_back(R); } @@ -487,8 +510,8 @@ void LowerSwitch::processSwitchInst(SwitchInst *SI, assert(MaxPop > 0 && PopSucc); Default = PopSucc; Cases.erase( - remove_if(Cases, - [PopSucc](const CaseRange &R) { return R.BB == PopSucc; }), + llvm::remove_if( + Cases, [PopSucc](const CaseRange &R) { return R.BB == PopSucc; }), Cases.end()); // If there are no cases left, just branch. diff --git a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index e2ba5c4cfbb..fcd3bd08482 100644 --- a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -21,25 +21,38 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" -#include "llvm/Analysis/AliasSetTracker.h" +#include "llvm/ADT/TinyPtrVector.h" +#include "llvm/ADT/Twine.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/IteratedDominanceFrontier.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" -#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Metadata.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/Support/Casting.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include +#include +#include +#include +#include + using namespace llvm; #define DEBUG_TYPE "mem2reg" @@ -103,7 +116,7 @@ struct AllocaInfo { bool OnlyUsedInOneBlock; Value *AllocaPointerVal; - TinyPtrVector DbgDeclares; + TinyPtrVector DbgDeclares; void clear() { DefiningBlocks.clear(); @@ -154,10 +167,11 @@ struct AllocaInfo { // Data package used by RenamePass() class RenamePassData { public: - typedef std::vector ValVector; + using ValVector = std::vector; RenamePassData(BasicBlock *B, BasicBlock *P, ValVector V) : BB(B), Pred(P), Values(std::move(V)) {} + BasicBlock *BB; BasicBlock *Pred; ValVector Values; @@ -216,12 +230,15 @@ public: struct PromoteMem2Reg { /// The alloca instructions being promoted. std::vector Allocas; + DominatorTree &DT; DIBuilder DIB; + /// A cache of @llvm.assume intrinsics used by SimplifyInstruction. AssumptionCache *AC; const SimplifyQuery SQ; + /// Reverse mapping of Allocas. DenseMap AllocaLookup; @@ -248,7 +265,6 @@ struct PromoteMem2Reg { SmallVector, 8> AllocaDbgDeclares; /// The set of basic blocks the renamer has already visited. - /// SmallPtrSet Visited; /// Contains a stable numbering of basic blocks to avoid non-determinstic @@ -291,7 +307,7 @@ private: bool QueuePhiNode(BasicBlock *BB, unsigned AllocaIdx, unsigned &Version); }; -} // end of anonymous namespace +} // end anonymous namespace /// Given a LoadInst LI this adds assume(LI != null) after it. static void addAssumeNonNull(AssumptionCache *AC, LoadInst *LI) { @@ -373,7 +389,6 @@ static bool rewriteSingleStoreAlloca(AllocaInst *AI, AllocaInfo &Info, Info.UsingBlocks.push_back(StoreBB); continue; } - } else if (LI->getParent() != StoreBB && !DT.dominates(StoreBB, LI->getParent())) { // If the load and store are in different blocks, use BB dominance to @@ -395,7 +410,7 @@ static bool rewriteSingleStoreAlloca(AllocaInst *AI, AllocaInfo &Info, // that information when we erase this Load. So we preserve // it with an assume. if (AC && LI->getMetadata(LLVMContext::MD_nonnull) && - !llvm::isKnownNonZero(ReplVal, DL, 0, AC, LI, &DT)) + !isKnownNonZero(ReplVal, DL, 0, AC, LI, &DT)) addAssumeNonNull(AC, LI); LI->replaceAllUsesWith(ReplVal); @@ -451,7 +466,7 @@ static bool promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info, // make it efficient to get the index of various operations in the block. // Walk the use-def list of the alloca, getting the locations of all stores. - typedef SmallVector, 64> StoresByIndexTy; + using StoresByIndexTy = SmallVector, 64>; StoresByIndexTy StoresByIndex; for (User *U : AI->users()) @@ -491,7 +506,7 @@ static bool promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info, // information when we erase it. So we preserve it with an assume. Value *ReplVal = std::prev(I)->second->getOperand(0); if (AC && LI->getMetadata(LLVMContext::MD_nonnull) && - !llvm::isKnownNonZero(ReplVal, DL, 0, AC, LI, &DT)) + !isKnownNonZero(ReplVal, DL, 0, AC, LI, &DT)) addAssumeNonNull(AC, LI); LI->replaceAllUsesWith(ReplVal); @@ -598,7 +613,6 @@ void PromoteMem2Reg::run() { // nodes and see if we can optimize out some work by avoiding insertion of // dead phi nodes. - // Unique the set of defining blocks for efficient lookup. SmallPtrSet DefBlocks; DefBlocks.insert(Info.DefiningBlocks.begin(), Info.DefiningBlocks.end()); @@ -635,14 +649,12 @@ void PromoteMem2Reg::run() { // Set the incoming values for the basic block to be null values for all of // the alloca's. We do this in case there is a load of a value that has not // been stored yet. In this case, it will get this null value. - // RenamePassData::ValVector Values(Allocas.size()); for (unsigned i = 0, e = Allocas.size(); i != e; ++i) Values[i] = UndefValue::get(Allocas[i]->getAllocatedType()); // Walks all basic blocks in the function performing the SSA rename algorithm // and inserting the phi nodes we marked as necessary - // std::vector RenamePassWorkList; RenamePassWorkList.emplace_back(&F.front(), nullptr, std::move(Values)); do { @@ -705,7 +717,6 @@ void PromoteMem2Reg::run() { // hasn't traversed. If this is the case, the PHI nodes may not // have incoming values for all predecessors. Loop over all PHI nodes we have // created, inserting undef values if they are missing any incoming values. - // for (DenseMap, PHINode *>::iterator I = NewPhiNodes.begin(), E = NewPhiNodes.end(); @@ -770,7 +781,6 @@ void PromoteMem2Reg::ComputeLiveInBlocks( AllocaInst *AI, AllocaInfo &Info, const SmallPtrSetImpl &DefBlocks, SmallPtrSetImpl &LiveInBlocks) { - // To determine liveness, we must iterate through the predecessors of blocks // where the def is live. Blocks are added to the worklist if we need to // check their predecessors. Start with all the using blocks. @@ -932,7 +942,7 @@ NextIteration: // that information when we erase this Load. So we preserve // it with an assume. if (AC && LI->getMetadata(LLVMContext::MD_nonnull) && - !llvm::isKnownNonZero(V, SQ.DL, 0, AC, LI, &DT)) + !isKnownNonZero(V, SQ.DL, 0, AC, LI, &DT)) addAssumeNonNull(AC, LI); // Anything using the load now uses the current value.