RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 18:54:02 +01:00
Code Issues Projects Releases Wiki Activity
78ddb62141
llvm-mirror/lib/Target/Hexagon/HexagonCommonGEP.cpp
Reid Kleckner 68092989f3 Sink all InitializePasses.h includes 
This file lists every pass in LLVM, and is included by Pass.h, which is
very popular. Every time we add, remove, or rename a pass in LLVM, it
caused lots of recompilation.

I found this fact by looking at this table, which is sorted by the
number of times a file was changed over the last 100,000 git commits
multiplied by the number of object files that depend on it in the
current checkout:
  recompiles    touches affected_files  header
  342380        95      3604    llvm/include/llvm/ADT/STLExtras.h
  314730        234     1345    llvm/include/llvm/InitializePasses.h
  307036        118     2602    llvm/include/llvm/ADT/APInt.h
  213049        59      3611    llvm/include/llvm/Support/MathExtras.h
  170422        47      3626    llvm/include/llvm/Support/Compiler.h
  162225        45      3605    llvm/include/llvm/ADT/Optional.h
  158319        63      2513    llvm/include/llvm/ADT/Triple.h
  140322        39      3598    llvm/include/llvm/ADT/StringRef.h
  137647        59      2333    llvm/include/llvm/Support/Error.h
  131619        73      1803    llvm/include/llvm/Support/FileSystem.h

Before this change, touching InitializePasses.h would cause 1345 files
to recompile. After this change, touching it only causes 550 compiles in
an incremental rebuild.

Reviewers: bkramer, asbirlea, bollu, jdoerfert

Differential Revision: https://reviews.llvm.org/D70211
2019-11-13 16:34:37 -08:00

1317 lines
42 KiB
C++
Raw Blame History

//===- HexagonCommonGEP.cpp -----------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/FoldingSet.h"
#include "llvm/ADT/GraphTraits.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/PostDominators.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/Local.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <map>
#include <set>
#include <utility>
#include <vector>
#define DEBUG_TYPE "commgep"
using namespace llvm;
static cl::opt<bool> OptSpeculate("commgep-speculate", cl::init(true),
cl::Hidden, cl::ZeroOrMore);
static cl::opt<bool> OptEnableInv("commgep-inv", cl::init(true), cl::Hidden,
cl::ZeroOrMore);
static cl::opt<bool> OptEnableConst("commgep-const", cl::init(true),
cl::Hidden, cl::ZeroOrMore);
namespace llvm {
void initializeHexagonCommonGEPPass(PassRegistry&);
} // end namespace llvm
namespace {
struct GepNode;
using NodeSet = std::set<GepNode *>;
using NodeToValueMap = std::map<GepNode *, Value *>;
using NodeVect = std::vector<GepNode *>;
using NodeChildrenMap = std::map<GepNode *, NodeVect>;
using UseSet = SetVector<Use *>;
using NodeToUsesMap = std::map<GepNode *, UseSet>;
// Numbering map for gep nodes. Used to keep track of ordering for
// gep nodes.
struct NodeOrdering {
NodeOrdering() = default;
void insert(const GepNode *N) { Map.insert(std::make_pair(N, ++LastNum)); }
void clear() { Map.clear(); }
bool operator()(const GepNode *N1, const GepNode *N2) const {
auto F1 = Map.find(N1), F2 = Map.find(N2);
assert(F1 != Map.end() && F2 != Map.end());
return F1->second < F2->second;
}
private:
std::map<const GepNode *, unsigned> Map;
unsigned LastNum = 0;
};
class HexagonCommonGEP : public FunctionPass {
public:
static char ID;
HexagonCommonGEP() : FunctionPass(ID) {
initializeHexagonCommonGEPPass(*PassRegistry::getPassRegistry());
}
bool runOnFunction(Function &F) override;
StringRef getPassName() const override { return "Hexagon Common GEP"; }
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<DominatorTreeWrapperPass>();
AU.addPreserved<DominatorTreeWrapperPass>();
AU.addRequired<PostDominatorTreeWrapperPass>();
AU.addPreserved<PostDominatorTreeWrapperPass>();
AU.addRequired<LoopInfoWrapperPass>();
AU.addPreserved<LoopInfoWrapperPass>();
FunctionPass::getAnalysisUsage(AU);
}
private:
using ValueToNodeMap = std::map<Value *, GepNode *>;
using ValueVect = std::vector<Value *>;
using NodeToValuesMap = std::map<GepNode *, ValueVect>;
void getBlockTraversalOrder(BasicBlock *Root, ValueVect &Order);
bool isHandledGepForm(GetElementPtrInst *GepI);
void processGepInst(GetElementPtrInst *GepI, ValueToNodeMap &NM);
void collect();
void common();
BasicBlock *recalculatePlacement(GepNode *Node, NodeChildrenMap &NCM,
NodeToValueMap &Loc);
BasicBlock *recalculatePlacementRec(GepNode *Node, NodeChildrenMap &NCM,
NodeToValueMap &Loc);
bool isInvariantIn(Value *Val, Loop *L);
bool isInvariantIn(GepNode *Node, Loop *L);
bool isInMainPath(BasicBlock *B, Loop *L);
BasicBlock *adjustForInvariance(GepNode *Node, NodeChildrenMap &NCM,
NodeToValueMap &Loc);
void separateChainForNode(GepNode *Node, Use *U, NodeToValueMap &Loc);
void separateConstantChains(GepNode *Node, NodeChildrenMap &NCM,
NodeToValueMap &Loc);
void computeNodePlacement(NodeToValueMap &Loc);
Value *fabricateGEP(NodeVect &NA, BasicBlock::iterator At,
BasicBlock *LocB);
void getAllUsersForNode(GepNode *Node, ValueVect &Values,
NodeChildrenMap &NCM);
void materialize(NodeToValueMap &Loc);
void removeDeadCode();
NodeVect Nodes;
NodeToUsesMap Uses;
NodeOrdering NodeOrder; // Node ordering, for deterministic behavior.
SpecificBumpPtrAllocator<GepNode> *Mem;
LLVMContext *Ctx;
LoopInfo *LI;
DominatorTree *DT;
PostDominatorTree *PDT;
Function *Fn;
};
} // end anonymous namespace
char HexagonCommonGEP::ID = 0;
INITIALIZE_PASS_BEGIN(HexagonCommonGEP, "hcommgep", "Hexagon Common GEP",
false, false)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
INITIALIZE_PASS_END(HexagonCommonGEP, "hcommgep", "Hexagon Common GEP",
false, false)
namespace {
struct GepNode {
enum {
None = 0,
Root = 0x01,
Internal = 0x02,
Used = 0x04,
InBounds = 0x08
};
uint32_t Flags = 0;
union {
GepNode *Parent;
Value *BaseVal;
};
Value *Idx = nullptr;
Type *PTy = nullptr; // Type of the pointer operand.
GepNode() : Parent(nullptr) {}
GepNode(const GepNode *N) : Flags(N->Flags), Idx(N->Idx), PTy(N->PTy) {
if (Flags & Root)
BaseVal = N->BaseVal;
else
Parent = N->Parent;
}
friend raw_ostream &operator<< (raw_ostream &OS, const GepNode &GN);
};
Type *next_type(Type *Ty, Value *Idx) {
if (auto *PTy = dyn_cast<PointerType>(Ty))
return PTy->getElementType();
// Advance the type.
if (!Ty->isStructTy()) {
Type *NexTy = cast<SequentialType>(Ty)->getElementType();
return NexTy;
}
// Otherwise it is a struct type.
ConstantInt *CI = dyn_cast<ConstantInt>(Idx);
assert(CI && "Struct type with non-constant index");
int64_t i = CI->getValue().getSExtValue();
Type *NextTy = cast<StructType>(Ty)->getElementType(i);
return NextTy;
}
raw_ostream &operator<< (raw_ostream &OS, const GepNode &GN) {
OS << "{ {";
bool Comma = false;
if (GN.Flags & GepNode::Root) {
OS << "root";
Comma = true;
}
if (GN.Flags & GepNode::Internal) {
if (Comma)
OS << ',';
OS << "internal";
Comma = true;
}
if (GN.Flags & GepNode::Used) {
if (Comma)
OS << ',';
OS << "used";
}
if (GN.Flags & GepNode::InBounds) {
if (Comma)
OS << ',';
OS << "inbounds";
}
OS << "} ";
if (GN.Flags & GepNode::Root)
OS << "BaseVal:" << GN.BaseVal->getName() << '(' << GN.BaseVal << ')';
else
OS << "Parent:" << GN.Parent;
OS << " Idx:";
if (ConstantInt *CI = dyn_cast<ConstantInt>(GN.Idx))
OS << CI->getValue().getSExtValue();
else if (GN.Idx->hasName())
OS << GN.Idx->getName();
else
OS << "<anon> =" << *GN.Idx;
OS << " PTy:";
if (GN.PTy->isStructTy()) {
StructType *STy = cast<StructType>(GN.PTy);
if (!STy->isLiteral())
OS << GN.PTy->getStructName();
else
OS << "<anon-struct>:" << *STy;
}
else
OS << *GN.PTy;
OS << " }";
return OS;
}
template <typename NodeContainer>
void dump_node_container(raw_ostream &OS, const NodeContainer &S) {
using const_iterator = typename NodeContainer::const_iterator;
for (const_iterator I = S.begin(), E = S.end(); I != E; ++I)
OS << *I << ' ' << **I << '\n';
}
raw_ostream &operator<< (raw_ostream &OS,
const NodeVect &S) LLVM_ATTRIBUTE_UNUSED;
raw_ostream &operator<< (raw_ostream &OS, const NodeVect &S) {
dump_node_container(OS, S);
return OS;
}
raw_ostream &operator<< (raw_ostream &OS,
const NodeToUsesMap &M) LLVM_ATTRIBUTE_UNUSED;
raw_ostream &operator<< (raw_ostream &OS, const NodeToUsesMap &M){
using const_iterator = NodeToUsesMap::const_iterator;
for (const_iterator I = M.begin(), E = M.end(); I != E; ++I) {
const UseSet &Us = I->second;
OS << I->first << " -> #" << Us.size() << '{';
for (UseSet::const_iterator J = Us.begin(), F = Us.end(); J != F; ++J) {
User *R = (*J)->getUser();
if (R->hasName())
OS << ' ' << R->getName();
else
OS << " <?>(" << *R << ')';
}
OS << " }\n";
}
return OS;
}
struct in_set {
in_set(const NodeSet &S) : NS(S) {}
bool operator() (GepNode *N) const {
return NS.find(N) != NS.end();
}
private:
const NodeSet &NS;
};
} // end anonymous namespace
inline void *operator new(size_t, SpecificBumpPtrAllocator<GepNode> &A) {
return A.Allocate();
}
void HexagonCommonGEP::getBlockTraversalOrder(BasicBlock *Root,
ValueVect &Order) {
// Compute block ordering for a typical DT-based traversal of the flow
// graph: "before visiting a block, all of its dominators must have been
// visited".
Order.push_back(Root);
for (auto *DTN : children<DomTreeNode*>(DT->getNode(Root)))
getBlockTraversalOrder(DTN->getBlock(), Order);
}
bool HexagonCommonGEP::isHandledGepForm(GetElementPtrInst *GepI) {
// No vector GEPs.
if (!GepI->getType()->isPointerTy())
return false;
// No GEPs without any indices. (Is this possible?)
if (GepI->idx_begin() == GepI->idx_end())
return false;
return true;
}
void HexagonCommonGEP::processGepInst(GetElementPtrInst *GepI,
ValueToNodeMap &NM) {
LLVM_DEBUG(dbgs() << "Visiting GEP: " << *GepI << '\n');
GepNode *N = new (*Mem) GepNode;
Value *PtrOp = GepI->getPointerOperand();
uint32_t InBounds = GepI->isInBounds() ? GepNode::InBounds : 0;
ValueToNodeMap::iterator F = NM.find(PtrOp);
if (F == NM.end()) {
N->BaseVal = PtrOp;
N->Flags |= GepNode::Root | InBounds;
} else {
// If PtrOp was a GEP instruction, it must have already been processed.
// The ValueToNodeMap entry for it is the last gep node in the generated
// chain. Link to it here.
N->Parent = F->second;
}
N->PTy = PtrOp->getType();
N->Idx = *GepI->idx_begin();
// Collect the list of users of this GEP instruction. Will add it to the
// last node created for it.
UseSet Us;
for (Value::user_iterator UI = GepI->user_begin(), UE = GepI->user_end();
UI != UE; ++UI) {
// Check if this gep is used by anything other than other geps that
// we will process.
if (isa<GetElementPtrInst>(*UI)) {
GetElementPtrInst *UserG = cast<GetElementPtrInst>(*UI);
if (isHandledGepForm(UserG))
continue;
}
Us.insert(&UI.getUse());
}
Nodes.push_back(N);
NodeOrder.insert(N);
// Skip the first index operand, since we only handle 0. This dereferences
// the pointer operand.
GepNode *PN = N;
Type *PtrTy = cast<PointerType>(PtrOp->getType())->getElementType();
for (User::op_iterator OI = GepI->idx_begin()+1, OE = GepI->idx_end();
OI != OE; ++OI) {
Value *Op = *OI;
GepNode *Nx = new (*Mem) GepNode;
Nx->Parent = PN; // Link Nx to the previous node.
Nx->Flags |= GepNode::Internal | InBounds;
Nx->PTy = PtrTy;
Nx->Idx = Op;
Nodes.push_back(Nx);
NodeOrder.insert(Nx);
PN = Nx;
PtrTy = next_type(PtrTy, Op);
}
// After last node has been created, update the use information.
if (!Us.empty()) {
PN->Flags |= GepNode::Used;
Uses[PN].insert(Us.begin(), Us.end());
}
// Link the last node with the originating GEP instruction. This is to
// help with linking chained GEP instructions.
NM.insert(std::make_pair(GepI, PN));
}
void HexagonCommonGEP::collect() {
// Establish depth-first traversal order of the dominator tree.
ValueVect BO;
getBlockTraversalOrder(&Fn->front(), BO);
// The creation of gep nodes requires DT-traversal. When processing a GEP
// instruction that uses another GEP instruction as the base pointer, the
// gep node for the base pointer should already exist.
ValueToNodeMap NM;
for (ValueVect::iterator I = BO.begin(), E = BO.end(); I != E; ++I) {
BasicBlock *B = cast<BasicBlock>(*I);
for (BasicBlock::iterator J = B->begin(), F = B->end(); J != F; ++J) {
if (!isa<GetElementPtrInst>(J))
continue;
GetElementPtrInst *GepI = cast<GetElementPtrInst>(J);
if (isHandledGepForm(GepI))
processGepInst(GepI, NM);
}
}
LLVM_DEBUG(dbgs() << "Gep nodes after initial collection:\n" << Nodes);
}
static void invert_find_roots(const NodeVect &Nodes, NodeChildrenMap &NCM,
NodeVect &Roots) {
using const_iterator = NodeVect::const_iterator;
for (const_iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
GepNode *N = *I;
if (N->Flags & GepNode::Root) {
Roots.push_back(N);
continue;
}
GepNode *PN = N->Parent;
NCM[PN].push_back(N);
}
}
static void nodes_for_root(GepNode *Root, NodeChildrenMap &NCM,
NodeSet &Nodes) {
NodeVect Work;
Work.push_back(Root);
Nodes.insert(Root);
while (!Work.empty()) {
NodeVect::iterator First = Work.begin();
GepNode *N = *First;
Work.erase(First);
NodeChildrenMap::iterator CF = NCM.find(N);
if (CF != NCM.end()) {
Work.insert(Work.end(), CF->second.begin(), CF->second.end());
Nodes.insert(CF->second.begin(), CF->second.end());
}
}
}
namespace {
using NodeSymRel = std::set<NodeSet>;
using NodePair = std::pair<GepNode *, GepNode *>;
using NodePairSet = std::set<NodePair>;
} // end anonymous namespace
static const NodeSet *node_class(GepNode *N, NodeSymRel &Rel) {
for (NodeSymRel::iterator I = Rel.begin(), E = Rel.end(); I != E; ++I)
if (I->count(N))
return &*I;
return nullptr;
}
// Create an ordered pair of GepNode pointers. The pair will be used in
// determining equality. The only purpose of the ordering is to eliminate
// duplication due to the commutativity of equality/non-equality.
static NodePair node_pair(GepNode *N1, GepNode *N2) {
uintptr_t P1 = uintptr_t(N1), P2 = uintptr_t(N2);
if (P1 <= P2)
return std::make_pair(N1, N2);
return std::make_pair(N2, N1);
}
static unsigned node_hash(GepNode *N) {
// Include everything except flags and parent.
FoldingSetNodeID ID;
ID.AddPointer(N->Idx);
ID.AddPointer(N->PTy);
return ID.ComputeHash();
}
static bool node_eq(GepNode *N1, GepNode *N2, NodePairSet &Eq,
NodePairSet &Ne) {
// Don't cache the result for nodes with different hashes. The hash
// comparison is fast enough.
if (node_hash(N1) != node_hash(N2))
return false;
NodePair NP = node_pair(N1, N2);
NodePairSet::iterator FEq = Eq.find(NP);
if (FEq != Eq.end())
return true;
NodePairSet::iterator FNe = Ne.find(NP);
if (FNe != Ne.end())
return false;
// Not previously compared.
bool Root1 = N1->Flags & GepNode::Root;
bool Root2 = N2->Flags & GepNode::Root;
NodePair P = node_pair(N1, N2);
// If the Root flag has different values, the nodes are different.
// If both nodes are root nodes, but their base pointers differ,
// they are different.
if (Root1 != Root2 || (Root1 && N1->BaseVal != N2->BaseVal)) {
Ne.insert(P);
return false;
}
// Here the root flags are identical, and for root nodes the
// base pointers are equal, so the root nodes are equal.
// For non-root nodes, compare their parent nodes.
if (Root1 || node_eq(N1->Parent, N2->Parent, Eq, Ne)) {
Eq.insert(P);
return true;
}
return false;
}
void HexagonCommonGEP::common() {
// The essence of this commoning is finding gep nodes that are equal.
// To do this we need to compare all pairs of nodes. To save time,
// first, partition the set of all nodes into sets of potentially equal
// nodes, and then compare pairs from within each partition.
using NodeSetMap = std::map<unsigned, NodeSet>;
NodeSetMap MaybeEq;
for (NodeVect::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
GepNode *N = *I;
unsigned H = node_hash(N);
MaybeEq[H].insert(N);
}
// Compute the equivalence relation for the gep nodes. Use two caches,
// one for equality and the other for non-equality.
NodeSymRel EqRel; // Equality relation (as set of equivalence classes).
NodePairSet Eq, Ne; // Caches.
for (NodeSetMap::iterator I = MaybeEq.begin(), E = MaybeEq.end();
I != E; ++I) {
NodeSet &S = I->second;
for (NodeSet::iterator NI = S.begin(), NE = S.end(); NI != NE; ++NI) {
GepNode *N = *NI;
// If node already has a class, then the class must have been created
// in a prior iteration of this loop. Since equality is transitive,
// nothing more will be added to that class, so skip it.
if (node_class(N, EqRel))
continue;
// Create a new class candidate now.
NodeSet C;
for (NodeSet::iterator NJ = std::next(NI); NJ != NE; ++NJ)
if (node_eq(N, *NJ, Eq, Ne))
C.insert(*NJ);
// If Tmp is empty, N would be the only element in it. Don't bother
// creating a class for it then.
if (!C.empty()) {
C.insert(N); // Finalize the set before adding it to the relation.
std::pair<NodeSymRel::iterator, bool> Ins = EqRel.insert(C);
(void)Ins;
assert(Ins.second && "Cannot add a class");
}
}
}
LLVM_DEBUG({
dbgs() << "Gep node equality:\n";
for (NodePairSet::iterator I = Eq.begin(), E = Eq.end(); I != E; ++I)
dbgs() << "{ " << I->first << ", " << I->second << " }\n";
dbgs() << "Gep equivalence classes:\n";
for (NodeSymRel::iterator I = EqRel.begin(), E = EqRel.end(); I != E; ++I) {
dbgs() << '{';
const NodeSet &S = *I;
for (NodeSet::const_iterator J = S.begin(), F = S.end(); J != F; ++J) {
if (J != S.begin())
dbgs() << ',';
dbgs() << ' ' << *J;
}
dbgs() << " }\n";
}
});
// Create a projection from a NodeSet to the minimal element in it.
using ProjMap = std::map<const NodeSet *, GepNode *>;
ProjMap PM;
for (NodeSymRel::iterator I = EqRel.begin(), E = EqRel.end(); I != E; ++I) {
const NodeSet &S = *I;
GepNode *Min = *std::min_element(S.begin(), S.end(), NodeOrder);
std::pair<ProjMap::iterator,bool> Ins = PM.insert(std::make_pair(&S, Min));
(void)Ins;
assert(Ins.second && "Cannot add minimal element");
// Update the min element's flags, and user list.
uint32_t Flags = 0;
UseSet &MinUs = Uses[Min];
for (NodeSet::iterator J = S.begin(), F = S.end(); J != F; ++J) {
GepNode *N = *J;
uint32_t NF = N->Flags;
// If N is used, append all original values of N to the list of
// original values of Min.
if (NF & GepNode::Used)
MinUs.insert(Uses[N].begin(), Uses[N].end());
Flags |= NF;
}
if (MinUs.empty())
Uses.erase(Min);
// The collected flags should include all the flags from the min element.
assert((Min->Flags & Flags) == Min->Flags);
Min->Flags = Flags;
}
// Commoning: for each non-root gep node, replace "Parent" with the
// selected (minimum) node from the corresponding equivalence class.
// If a given parent does not have an equivalence class, leave it
// unchanged (it means that it's the only element in its class).
for (NodeVect::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
GepNode *N = *I;
if (N->Flags & GepNode::Root)
continue;
const NodeSet *PC = node_class(N->Parent, EqRel);
if (!PC)
continue;
ProjMap::iterator F = PM.find(PC);
if (F == PM.end())
continue;
// Found a replacement, use it.
GepNode *Rep = F->second;
N->Parent = Rep;
}
LLVM_DEBUG(dbgs() << "Gep nodes after commoning:\n" << Nodes);
// Finally, erase the nodes that are no longer used.
NodeSet Erase;
for (NodeVect::iterator I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {
GepNode *N = *I;
const NodeSet *PC = node_class(N, EqRel);
if (!PC)
continue;
ProjMap::iterator F = PM.find(PC);
if (F == PM.end())
continue;
if (N == F->second)
continue;
// Node for removal.
Erase.insert(*I);
}
NodeVect::iterator NewE = remove_if(Nodes, in_set(Erase));
Nodes.resize(std::distance(Nodes.begin(), NewE));
LLVM_DEBUG(dbgs() << "Gep nodes after post-commoning cleanup:\n" << Nodes);
}
template <typename T>
static BasicBlock *nearest_common_dominator(DominatorTree *DT, T &Blocks) {
LLVM_DEBUG({
dbgs() << "NCD of {";
for (typename T::iterator I = Blocks.begin(), E = Blocks.end(); I != E;
++I) {
if (!*I)
continue;
BasicBlock *B = cast<BasicBlock>(*I);
dbgs() << ' ' << B->getName();
}
dbgs() << " }\n";
});
// Allow null basic blocks in Blocks. In such cases, return nullptr.
typename T::iterator I = Blocks.begin(), E = Blocks.end();
if (I == E || !*I)
return nullptr;
BasicBlock *Dom = cast<BasicBlock>(*I);
while (++I != E) {
BasicBlock *B = cast_or_null<BasicBlock>(*I);
Dom = B ? DT->findNearestCommonDominator(Dom, B) : nullptr;
if (!Dom)
return nullptr;
}
LLVM_DEBUG(dbgs() << "computed:" << Dom->getName() << '\n');
return Dom;
}
template <typename T>
static BasicBlock *nearest_common_dominatee(DominatorTree *DT, T &Blocks) {
// If two blocks, A and B, dominate a block C, then A dominates B,
// or B dominates A.
typename T::iterator I = Blocks.begin(), E = Blocks.end();
// Find the first non-null block.
while (I != E && !*I)
++I;
if (I == E)
return DT->getRoot();
BasicBlock *DomB = cast<BasicBlock>(*I);
while (++I != E) {
if (!*I)
continue;
BasicBlock *B = cast<BasicBlock>(*I);
if (DT->dominates(B, DomB))
continue;
if (!DT->dominates(DomB, B))
return nullptr;
DomB = B;
}
return DomB;
}
// Find the first use in B of any value from Values. If no such use,
// return B->end().
template <typename T>
static BasicBlock::iterator first_use_of_in_block(T &Values, BasicBlock *B) {
BasicBlock::iterator FirstUse = B->end(), BEnd = B->end();
using iterator = typename T::iterator;
for (iterator I = Values.begin(), E = Values.end(); I != E; ++I) {
Value *V = *I;
// If V is used in a PHI node, the use belongs to the incoming block,
// not the block with the PHI node. In the incoming block, the use
// would be considered as being at the end of it, so it cannot
// influence the position of the first use (which is assumed to be
// at the end to start with).
if (isa<PHINode>(V))
continue;
if (!isa<Instruction>(V))
continue;
Instruction *In = cast<Instruction>(V);
if (In->getParent() != B)
continue;
BasicBlock::iterator It = In->getIterator();
if (std::distance(FirstUse, BEnd) < std::distance(It, BEnd))
FirstUse = It;
}
return FirstUse;
}
static bool is_empty(const BasicBlock *B) {
return B->empty() || (&*B->begin() == B->getTerminator());
}
BasicBlock *HexagonCommonGEP::recalculatePlacement(GepNode *Node,
NodeChildrenMap &NCM, NodeToValueMap &Loc) {
LLVM_DEBUG(dbgs() << "Loc for node:" << Node << '\n');
// Recalculate the placement for Node, assuming that the locations of
// its children in Loc are valid.
// Return nullptr if there is no valid placement for Node (for example, it
// uses an index value that is not available at the location required
// to dominate all children, etc.).
// Find the nearest common dominator for:
// - all users, if the node is used, and
// - all children.
ValueVect Bs;
if (Node->Flags & GepNode::Used) {
// Append all blocks with uses of the original values to the
// block vector Bs.
NodeToUsesMap::iterator UF = Uses.find(Node);
assert(UF != Uses.end() && "Used node with no use information");
UseSet &Us = UF->second;
for (UseSet::iterator I = Us.begin(), E = Us.end(); I != E; ++I) {
Use *U = *I;
User *R = U->getUser();
if (!isa<Instruction>(R))
continue;
BasicBlock *PB = isa<PHINode>(R)
? cast<PHINode>(R)->getIncomingBlock(*U)
: cast<Instruction>(R)->getParent();
Bs.push_back(PB);
}
}
// Append the location of each child.
NodeChildrenMap::iterator CF = NCM.find(Node);
if (CF != NCM.end()) {
NodeVect &Cs = CF->second;
for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I) {
GepNode *CN = *I;
NodeToValueMap::iterator LF = Loc.find(CN);
// If the child is only used in GEP instructions (i.e. is not used in
// non-GEP instructions), the nearest dominator computed for it may
// have been null. In such case it won't have a location available.
if (LF == Loc.end())
continue;
Bs.push_back(LF->second);
}
}
BasicBlock *DomB = nearest_common_dominator(DT, Bs);
if (!DomB)
return nullptr;
// Check if the index used by Node dominates the computed dominator.
Instruction *IdxI = dyn_cast<Instruction>(Node->Idx);
if (IdxI && !DT->dominates(IdxI->getParent(), DomB))
return nullptr;
// Avoid putting nodes into empty blocks.
while (is_empty(DomB)) {
DomTreeNode *N = (*DT)[DomB]->getIDom();
if (!N)
break;
DomB = N->getBlock();
}
// Otherwise, DomB is fine. Update the location map.
Loc[Node] = DomB;
return DomB;
}
BasicBlock *HexagonCommonGEP::recalculatePlacementRec(GepNode *Node,
NodeChildrenMap &NCM, NodeToValueMap &Loc) {
LLVM_DEBUG(dbgs() << "LocRec begin for node:" << Node << '\n');
// Recalculate the placement of Node, after recursively recalculating the
// placements of all its children.
NodeChildrenMap::iterator CF = NCM.find(Node);
if (CF != NCM.end()) {
NodeVect &Cs = CF->second;
for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I)
recalculatePlacementRec(*I, NCM, Loc);
}
BasicBlock *LB = recalculatePlacement(Node, NCM, Loc);
LLVM_DEBUG(dbgs() << "LocRec end for node:" << Node << '\n');
return LB;
}
bool HexagonCommonGEP::isInvariantIn(Value *Val, Loop *L) {
if (isa<Constant>(Val) || isa<Argument>(Val))
return true;
Instruction *In = dyn_cast<Instruction>(Val);
if (!In)
return false;
BasicBlock *HdrB = L->getHeader(), *DefB = In->getParent();
return DT->properlyDominates(DefB, HdrB);
}
bool HexagonCommonGEP::isInvariantIn(GepNode *Node, Loop *L) {
if (Node->Flags & GepNode::Root)
if (!isInvariantIn(Node->BaseVal, L))
return false;
return isInvariantIn(Node->Idx, L);
}
bool HexagonCommonGEP::isInMainPath(BasicBlock *B, Loop *L) {
BasicBlock *HB = L->getHeader();
BasicBlock *LB = L->getLoopLatch();
// B must post-dominate the loop header or dominate the loop latch.
if (PDT->dominates(B, HB))
return true;
if (LB && DT->dominates(B, LB))
return true;
return false;
}
static BasicBlock *preheader(DominatorTree *DT, Loop *L) {
if (BasicBlock *PH = L->getLoopPreheader())
return PH;
if (!OptSpeculate)
return nullptr;
DomTreeNode *DN = DT->getNode(L->getHeader());
if (!DN)
return nullptr;
return DN->getIDom()->getBlock();
}
BasicBlock *HexagonCommonGEP::adjustForInvariance(GepNode *Node,
NodeChildrenMap &NCM, NodeToValueMap &Loc) {
// Find the "topmost" location for Node: it must be dominated by both,
// its parent (or the BaseVal, if it's a root node), and by the index
// value.
ValueVect Bs;
if (Node->Flags & GepNode::Root) {
if (Instruction *PIn = dyn_cast<Instruction>(Node->BaseVal))
Bs.push_back(PIn->getParent());
} else {
Bs.push_back(Loc[Node->Parent]);
}
if (Instruction *IIn = dyn_cast<Instruction>(Node->Idx))
Bs.push_back(IIn->getParent());
BasicBlock *TopB = nearest_common_dominatee(DT, Bs);
// Traverse the loop nest upwards until we find a loop in which Node
// is no longer invariant, or until we get to the upper limit of Node's
// placement. The traversal will also stop when a suitable "preheader"
// cannot be found for a given loop. The "preheader" may actually be
// a regular block outside of the loop (i.e. not guarded), in which case
// the Node will be speculated.
// For nodes that are not in the main path of the containing loop (i.e.
// are not executed in each iteration), do not move them out of the loop.
BasicBlock *LocB = cast_or_null<BasicBlock>(Loc[Node]);
if (LocB) {
Loop *Lp = LI->getLoopFor(LocB);
while (Lp) {
if (!isInvariantIn(Node, Lp) || !isInMainPath(LocB, Lp))
break;
BasicBlock *NewLoc = preheader(DT, Lp);
if (!NewLoc || !DT->dominates(TopB, NewLoc))
break;
Lp = Lp->getParentLoop();
LocB = NewLoc;
}
}
Loc[Node] = LocB;
// Recursively compute the locations of all children nodes.
NodeChildrenMap::iterator CF = NCM.find(Node);
if (CF != NCM.end()) {
NodeVect &Cs = CF->second;
for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I)
adjustForInvariance(*I, NCM, Loc);
}
return LocB;
}
namespace {
struct LocationAsBlock {
LocationAsBlock(const NodeToValueMap &L) : Map(L) {}
const NodeToValueMap &Map;
};
raw_ostream &operator<< (raw_ostream &OS,
const LocationAsBlock &Loc) LLVM_ATTRIBUTE_UNUSED ;
raw_ostream &operator<< (raw_ostream &OS, const LocationAsBlock &Loc) {
for (NodeToValueMap::const_iterator I = Loc.Map.begin(), E = Loc.Map.end();
I != E; ++I) {
OS << I->first << " -> ";
BasicBlock *B = cast<BasicBlock>(I->second);
OS << B->getName() << '(' << B << ')';
OS << '\n';
}
return OS;
}
inline bool is_constant(GepNode *N) {
return isa<ConstantInt>(N->Idx);
}
} // end anonymous namespace
void HexagonCommonGEP::separateChainForNode(GepNode *Node, Use *U,
NodeToValueMap &Loc) {
User *R = U->getUser();
LLVM_DEBUG(dbgs() << "Separating chain for node (" << Node << ") user: " << *R
<< '\n');
BasicBlock *PB = cast<Instruction>(R)->getParent();
GepNode *N = Node;
GepNode *C = nullptr, *NewNode = nullptr;
while (is_constant(N) && !(N->Flags & GepNode::Root)) {
// XXX if (single-use) dont-replicate;
GepNode *NewN = new (*Mem) GepNode(N);
Nodes.push_back(NewN);
Loc[NewN] = PB;
if (N == Node)
NewNode = NewN;
NewN->Flags &= ~GepNode::Used;
if (C)
C->Parent = NewN;
C = NewN;
N = N->Parent;
}
if (!NewNode)
return;
// Move over all uses that share the same user as U from Node to NewNode.
NodeToUsesMap::iterator UF = Uses.find(Node);
assert(UF != Uses.end());
UseSet &Us = UF->second;
UseSet NewUs;
for (Use *U : Us) {
if (U->getUser() == R)
NewUs.insert(U);
}
for (Use *U : NewUs)
Us.remove(U); // erase takes an iterator.
if (Us.empty()) {
Node->Flags &= ~GepNode::Used;
Uses.erase(UF);
}
// Should at least have U in NewUs.
NewNode->Flags |= GepNode::Used;
LLVM_DEBUG(dbgs() << "new node: " << NewNode << " " << *NewNode << '\n');
assert(!NewUs.empty());
Uses[NewNode] = NewUs;
}
void HexagonCommonGEP::separateConstantChains(GepNode *Node,
NodeChildrenMap &NCM, NodeToValueMap &Loc) {
// First approximation: extract all chains.
NodeSet Ns;
nodes_for_root(Node, NCM, Ns);
LLVM_DEBUG(dbgs() << "Separating constant chains for node: " << Node << '\n');
// Collect all used nodes together with the uses from loads and stores,
// where the GEP node could be folded into the load/store instruction.
NodeToUsesMap FNs; // Foldable nodes.
for (NodeSet::iterator I = Ns.begin(), E = Ns.end(); I != E; ++I) {
GepNode *N = *I;
if (!(N->Flags & GepNode::Used))
continue;
NodeToUsesMap::iterator UF = Uses.find(N);
assert(UF != Uses.end());
UseSet &Us = UF->second;
// Loads/stores that use the node N.
UseSet LSs;
for (UseSet::iterator J = Us.begin(), F = Us.end(); J != F; ++J) {
Use *U = *J;
User *R = U->getUser();
// We're interested in uses that provide the address. It can happen
// that the value may also be provided via GEP, but we won't handle
// those cases here for now.
if (LoadInst *Ld = dyn_cast<LoadInst>(R)) {
unsigned PtrX = LoadInst::getPointerOperandIndex();
if (&Ld->getOperandUse(PtrX) == U)
LSs.insert(U);
} else if (StoreInst *St = dyn_cast<StoreInst>(R)) {
unsigned PtrX = StoreInst::getPointerOperandIndex();
if (&St->getOperandUse(PtrX) == U)
LSs.insert(U);
}
}
// Even if the total use count is 1, separating the chain may still be
// beneficial, since the constant chain may be longer than the GEP alone
// would be (e.g. if the parent node has a constant index and also has
// other children).
if (!LSs.empty())
FNs.insert(std::make_pair(N, LSs));
}
LLVM_DEBUG(dbgs() << "Nodes with foldable users:\n" << FNs);
for (NodeToUsesMap::iterator I = FNs.begin(), E = FNs.end(); I != E; ++I) {
GepNode *N = I->first;
UseSet &Us = I->second;
for (UseSet::iterator J = Us.begin(), F = Us.end(); J != F; ++J)
separateChainForNode(N, *J, Loc);
}
}
void HexagonCommonGEP::computeNodePlacement(NodeToValueMap &Loc) {
// Compute the inverse of the Node.Parent links. Also, collect the set
// of root nodes.
NodeChildrenMap NCM;
NodeVect Roots;
invert_find_roots(Nodes, NCM, Roots);
// Compute the initial placement determined by the users' locations, and
// the locations of the child nodes.
for (NodeVect::iterator I = Roots.begin(), E = Roots.end(); I != E; ++I)
recalculatePlacementRec(*I, NCM, Loc);
LLVM_DEBUG(dbgs() << "Initial node placement:\n" << LocationAsBlock(Loc));
if (OptEnableInv) {
for (NodeVect::iterator I = Roots.begin(), E = Roots.end(); I != E; ++I)
adjustForInvariance(*I, NCM, Loc);
LLVM_DEBUG(dbgs() << "Node placement after adjustment for invariance:\n"
<< LocationAsBlock(Loc));
}
if (OptEnableConst) {
for (NodeVect::iterator I = Roots.begin(), E = Roots.end(); I != E; ++I)
separateConstantChains(*I, NCM, Loc);
}
LLVM_DEBUG(dbgs() << "Node use information:\n" << Uses);
// At the moment, there is no further refinement of the initial placement.
// Such a refinement could include splitting the nodes if they are placed
// too far from some of its users.
LLVM_DEBUG(dbgs() << "Final node placement:\n" << LocationAsBlock(Loc));
}
Value *HexagonCommonGEP::fabricateGEP(NodeVect &NA, BasicBlock::iterator At,
BasicBlock *LocB) {
LLVM_DEBUG(dbgs() << "Fabricating GEP in " << LocB->getName()
<< " for nodes:\n"
<< NA);
unsigned Num = NA.size();
GepNode *RN = NA[0];
assert((RN->Flags & GepNode::Root) && "Creating GEP for non-root");
GetElementPtrInst *NewInst = nullptr;
Value *Input = RN->BaseVal;
Value **IdxList = new Value*[Num+1];
unsigned nax = 0;
do {
unsigned IdxC = 0;
// If the type of the input of the first node is not a pointer,
// we need to add an artificial i32 0 to the indices (because the
// actual input in the IR will be a pointer).
if (!NA[nax]->PTy->isPointerTy()) {
Type *Int32Ty = Type::getInt32Ty(*Ctx);
IdxList[IdxC++] = ConstantInt::get(Int32Ty, 0);
}
// Keep adding indices from NA until we have to stop and generate
// an "intermediate" GEP.
while (++nax <= Num) {
GepNode *N = NA[nax-1];
IdxList[IdxC++] = N->Idx;
if (nax < Num) {
// We have to stop, if the expected type of the output of this node
// is not the same as the input type of the next node.
Type *NextTy = next_type(N->PTy, N->Idx);
if (NextTy != NA[nax]->PTy)
break;
}
}
ArrayRef<Value*> A(IdxList, IdxC);
Type *InpTy = Input->getType();
Type *ElTy = cast<PointerType>(InpTy->getScalarType())->getElementType();
NewInst = GetElementPtrInst::Create(ElTy, Input, A, "cgep", &*At);
NewInst->setIsInBounds(RN->Flags & GepNode::InBounds);
LLVM_DEBUG(dbgs() << "new GEP: " << *NewInst << '\n');
Input = NewInst;
} while (nax <= Num);
delete[] IdxList;
return NewInst;
}
void HexagonCommonGEP::getAllUsersForNode(GepNode *Node, ValueVect &Values,
NodeChildrenMap &NCM) {
NodeVect Work;
Work.push_back(Node);
while (!Work.empty()) {
NodeVect::iterator First = Work.begin();
GepNode *N = *First;
Work.erase(First);
if (N->Flags & GepNode::Used) {
NodeToUsesMap::iterator UF = Uses.find(N);
assert(UF != Uses.end() && "No use information for used node");
UseSet &Us = UF->second;
for (UseSet::iterator I = Us.begin(), E = Us.end(); I != E; ++I)
Values.push_back((*I)->getUser());
}
NodeChildrenMap::iterator CF = NCM.find(N);
if (CF != NCM.end()) {
NodeVect &Cs = CF->second;
Work.insert(Work.end(), Cs.begin(), Cs.end());
}
}
}
void HexagonCommonGEP::materialize(NodeToValueMap &Loc) {
LLVM_DEBUG(dbgs() << "Nodes before materialization:\n" << Nodes << '\n');
NodeChildrenMap NCM;
NodeVect Roots;
// Compute the inversion again, since computing placement could alter
// "parent" relation between nodes.
invert_find_roots(Nodes, NCM, Roots);
while (!Roots.empty()) {
NodeVect::iterator First = Roots.begin();
GepNode *Root = *First, *Last = *First;
Roots.erase(First);
NodeVect NA; // Nodes to assemble.
// Append to NA all child nodes up to (and including) the first child
// that:
// (1) has more than 1 child, or
// (2) is used, or
// (3) has a child located in a different block.
bool LastUsed = false;
unsigned LastCN = 0;
// The location may be null if the computation failed (it can legitimately
// happen for nodes created from dead GEPs).
Value *LocV = Loc[Last];
if (!LocV)
continue;
BasicBlock *LastB = cast<BasicBlock>(LocV);
do {
NA.push_back(Last);
LastUsed = (Last->Flags & GepNode::Used);
if (LastUsed)
break;
NodeChildrenMap::iterator CF = NCM.find(Last);
LastCN = (CF != NCM.end()) ? CF->second.size() : 0;
if (LastCN != 1)
break;
GepNode *Child = CF->second.front();
BasicBlock *ChildB = cast_or_null<BasicBlock>(Loc[Child]);
if (ChildB != nullptr && LastB != ChildB)
break;
Last = Child;
} while (true);
BasicBlock::iterator InsertAt = LastB->getTerminator()->getIterator();
if (LastUsed || LastCN > 0) {
ValueVect Urs;
getAllUsersForNode(Root, Urs, NCM);
BasicBlock::iterator FirstUse = first_use_of_in_block(Urs, LastB);
if (FirstUse != LastB->end())
InsertAt = FirstUse;
}
// Generate a new instruction for NA.
Value *NewInst = fabricateGEP(NA, InsertAt, LastB);
// Convert all the children of Last node into roots, and append them
// to the Roots list.
if (LastCN > 0) {
NodeVect &Cs = NCM[Last];
for (NodeVect::iterator I = Cs.begin(), E = Cs.end(); I != E; ++I) {
GepNode *CN = *I;
CN->Flags &= ~GepNode::Internal;
CN->Flags |= GepNode::Root;
CN->BaseVal = NewInst;
Roots.push_back(CN);
}
}
// Lastly, if the Last node was used, replace all uses with the new GEP.
// The uses reference the original GEP values.
if (LastUsed) {
NodeToUsesMap::iterator UF = Uses.find(Last);
assert(UF != Uses.end() && "No use information found");
UseSet &Us = UF->second;
for (UseSet::iterator I = Us.begin(), E = Us.end(); I != E; ++I) {
Use *U = *I;
U->set(NewInst);
}
}
}
}
void HexagonCommonGEP::removeDeadCode() {
ValueVect BO;
BO.push_back(&Fn->front());
for (unsigned i = 0; i < BO.size(); ++i) {
BasicBlock *B = cast<BasicBlock>(BO[i]);
for (auto DTN : children<DomTreeNode*>(DT->getNode(B)))
BO.push_back(DTN->getBlock());
}
for (unsigned i = BO.size(); i > 0; --i) {
BasicBlock *B = cast<BasicBlock>(BO[i-1]);
BasicBlock::InstListType &IL = B->getInstList();
using reverse_iterator = BasicBlock::InstListType::reverse_iterator;
ValueVect Ins;
for (reverse_iterator I = IL.rbegin(), E = IL.rend(); I != E; ++I)
Ins.push_back(&*I);
for (ValueVect::iterator I = Ins.begin(), E = Ins.end(); I != E; ++I) {
Instruction *In = cast<Instruction>(*I);
if (isInstructionTriviallyDead(In))
In->eraseFromParent();
}
}
}
bool HexagonCommonGEP::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
// For now bail out on C++ exception handling.
for (Function::iterator A = F.begin(), Z = F.end(); A != Z; ++A)
for (BasicBlock::iterator I = A->begin(), E = A->end(); I != E; ++I)
if (isa<InvokeInst>(I) || isa<LandingPadInst>(I))
return false;
Fn = &F;
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
PDT = &getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
Ctx = &F.getContext();
Nodes.clear();
Uses.clear();
NodeOrder.clear();
SpecificBumpPtrAllocator<GepNode> Allocator;
Mem = &Allocator;
collect();
common();
NodeToValueMap Loc;
computeNodePlacement(Loc);
materialize(Loc);
removeDeadCode();
#ifdef EXPENSIVE_CHECKS
// Run this only when expensive checks are enabled.
verifyFunction(F);
#endif
return true;
}
namespace llvm {
FunctionPass *createHexagonCommonGEP() {
return new HexagonCommonGEP();
}
} // end namespace llvm
Reference in New Issue View Git Blame Copy Permalink