mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-10-19 02:52:53 +02:00
Revert r311077: [LV] Using VPlan ...
This causes LLVM to assert fail on PPC64 and crash / infloop in other cases. Filed http://llvm.org/PR34248 with reproducer attached. llvm-svn: 311304
This commit is contained in:
Chandler Carruth
parent
0ab67021c0
commit
f802228b0f
@ -3,7 +3,6 @@ add_llvm_library(LLVMVectorize
|
||||
LoopVectorize.cpp
|
||||
SLPVectorizer.cpp
|
||||
Vectorize.cpp
|
||||
VPlan.cpp
|
||||
|
||||
ADDITIONAL_HEADER_DIRS
|
||||
${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,401 +0,0 @@
|
||||
//===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
///
|
||||
/// \file
|
||||
/// This is the LLVM vectorization plan. It represents a candidate for
|
||||
/// vectorization, allowing to plan and optimize how to vectorize a given loop
|
||||
/// before generating LLVM-IR.
|
||||
/// The vectorizer uses vectorization plans to estimate the costs of potential
|
||||
/// candidates and if profitable to execute the desired plan, generating vector
|
||||
/// LLVM-IR code.
|
||||
///
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "VPlan.h"
|
||||
#include "llvm/ADT/PostOrderIterator.h"
|
||||
#include "llvm/Analysis/LoopInfo.h"
|
||||
#include "llvm/IR/BasicBlock.h"
|
||||
#include "llvm/IR/Dominators.h"
|
||||
#include "llvm/Support/GraphWriter.h"
|
||||
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
|
||||
|
||||
using namespace llvm;
|
||||
|
||||
#define DEBUG_TYPE "vplan"
|
||||
|
||||
/// \return the VPBasicBlock that is the entry of Block, possibly indirectly.
|
||||
const VPBasicBlock *VPBlockBase::getEntryBasicBlock() const {
|
||||
const VPBlockBase *Block = this;
|
||||
while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
|
||||
Block = Region->getEntry();
|
||||
return cast<VPBasicBlock>(Block);
|
||||
}
|
||||
|
||||
VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
|
||||
VPBlockBase *Block = this;
|
||||
while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
|
||||
Block = Region->getEntry();
|
||||
return cast<VPBasicBlock>(Block);
|
||||
}
|
||||
|
||||
/// \return the VPBasicBlock that is the exit of Block, possibly indirectly.
|
||||
const VPBasicBlock *VPBlockBase::getExitBasicBlock() const {
|
||||
const VPBlockBase *Block = this;
|
||||
while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
|
||||
Block = Region->getExit();
|
||||
return cast<VPBasicBlock>(Block);
|
||||
}
|
||||
|
||||
VPBasicBlock *VPBlockBase::getExitBasicBlock() {
|
||||
VPBlockBase *Block = this;
|
||||
while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
|
||||
Block = Region->getExit();
|
||||
return cast<VPBasicBlock>(Block);
|
||||
}
|
||||
|
||||
VPBlockBase *VPBlockBase::getEnclosingBlockWithSuccessors() {
|
||||
if (!Successors.empty() || !Parent)
|
||||
return this;
|
||||
assert(Parent->getExit() == this &&
|
||||
"Block w/o successors not the exit of its parent.");
|
||||
return Parent->getEnclosingBlockWithSuccessors();
|
||||
}
|
||||
|
||||
VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() {
|
||||
if (!Predecessors.empty() || !Parent)
|
||||
return this;
|
||||
assert(Parent->getEntry() == this &&
|
||||
"Block w/o predecessors not the entry of its parent.");
|
||||
return Parent->getEnclosingBlockWithPredecessors();
|
||||
}
|
||||
|
||||
void VPBlockBase::deleteCFG(VPBlockBase *Entry) {
|
||||
SmallVector<VPBlockBase *, 8> Blocks;
|
||||
for (VPBlockBase *Block : depth_first(Entry))
|
||||
Blocks.push_back(Block);
|
||||
|
||||
for (VPBlockBase *Block : Blocks)
|
||||
delete Block;
|
||||
}
|
||||
|
||||
BasicBlock *
|
||||
VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
|
||||
// BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
|
||||
// Pred stands for Predessor. Prev stands for Previous - last visited/created.
|
||||
BasicBlock *PrevBB = CFG.PrevBB;
|
||||
BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
|
||||
PrevBB->getParent(), CFG.LastBB);
|
||||
DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n');
|
||||
|
||||
// Hook up the new basic block to its predecessors.
|
||||
for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
|
||||
VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock();
|
||||
auto &PredVPSuccessors = PredVPBB->getSuccessors();
|
||||
BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
|
||||
assert(PredBB && "Predecessor basic-block not found building successor.");
|
||||
auto *PredBBTerminator = PredBB->getTerminator();
|
||||
DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
|
||||
if (isa<UnreachableInst>(PredBBTerminator)) {
|
||||
assert(PredVPSuccessors.size() == 1 &&
|
||||
"Predecessor ending w/o branch must have single successor.");
|
||||
PredBBTerminator->eraseFromParent();
|
||||
BranchInst::Create(NewBB, PredBB);
|
||||
} else {
|
||||
assert(PredVPSuccessors.size() == 2 &&
|
||||
"Predecessor ending with branch must have two successors.");
|
||||
unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
|
||||
assert(!PredBBTerminator->getSuccessor(idx) &&
|
||||
"Trying to reset an existing successor block.");
|
||||
PredBBTerminator->setSuccessor(idx, NewBB);
|
||||
}
|
||||
}
|
||||
return NewBB;
|
||||
}
|
||||
|
||||
void VPBasicBlock::execute(VPTransformState *State) {
|
||||
bool Replica = State->Instance &&
|
||||
!(State->Instance->Part == 0 && State->Instance->Lane == 0);
|
||||
VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB;
|
||||
VPBlockBase *SingleHPred = nullptr;
|
||||
BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
|
||||
|
||||
// 1. Create an IR basic block, or reuse the last one if possible.
|
||||
// The last IR basic block is reused, as an optimization, in three cases:
|
||||
// A. the first VPBB reuses the loop header BB - when PrevVPBB is null;
|
||||
// B. when the current VPBB has a single (hierarchical) predecessor which
|
||||
// is PrevVPBB and the latter has a single (hierarchical) successor; and
|
||||
// C. when the current VPBB is an entry of a region replica - where PrevVPBB
|
||||
// is the exit of this region from a previous instance, or the predecessor
|
||||
// of this region.
|
||||
if (PrevVPBB && /* A */
|
||||
!((SingleHPred = getSingleHierarchicalPredecessor()) &&
|
||||
SingleHPred->getExitBasicBlock() == PrevVPBB &&
|
||||
PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
|
||||
!(Replica && getPredecessors().empty())) { /* C */
|
||||
|
||||
NewBB = createEmptyBasicBlock(State->CFG);
|
||||
State->Builder.SetInsertPoint(NewBB);
|
||||
// Temporarily terminate with unreachable until CFG is rewired.
|
||||
UnreachableInst *Terminator = State->Builder.CreateUnreachable();
|
||||
State->Builder.SetInsertPoint(Terminator);
|
||||
// Register NewBB in its loop. In innermost loops its the same for all BB's.
|
||||
Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
|
||||
L->addBasicBlockToLoop(NewBB, *State->LI);
|
||||
State->CFG.PrevBB = NewBB;
|
||||
}
|
||||
|
||||
// 2. Fill the IR basic block with IR instructions.
|
||||
DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
|
||||
<< " in BB:" << NewBB->getName() << '\n');
|
||||
|
||||
State->CFG.VPBB2IRBB[this] = NewBB;
|
||||
State->CFG.PrevVPBB = this;
|
||||
|
||||
for (VPRecipeBase &Recipe : Recipes)
|
||||
Recipe.execute(*State);
|
||||
|
||||
DEBUG(dbgs() << "LV: filled BB:" << *NewBB);
|
||||
}
|
||||
|
||||
void VPRegionBlock::execute(VPTransformState *State) {
|
||||
ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry);
|
||||
|
||||
if (!isReplicator()) {
|
||||
// Visit the VPBlocks connected to "this", starting from it.
|
||||
for (VPBlockBase *Block : RPOT) {
|
||||
DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
|
||||
Block->execute(State);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
assert(!State->Instance && "Replicating a Region with non-null instance.");
|
||||
|
||||
// Enter replicating mode.
|
||||
State->Instance = {0, 0};
|
||||
|
||||
for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) {
|
||||
State->Instance->Part = Part;
|
||||
for (unsigned Lane = 0, VF = State->VF; Lane < VF; ++Lane) {
|
||||
State->Instance->Lane = Lane;
|
||||
// Visit the VPBlocks connected to \p this, starting from it.
|
||||
for (VPBlockBase *Block : RPOT) {
|
||||
DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
|
||||
Block->execute(State);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exit replicating mode.
|
||||
State->Instance.reset();
|
||||
}
|
||||
|
||||
/// Generate the code inside the body of the vectorized loop. Assumes a single
|
||||
/// LoopVectorBody basic-block was created for this. Introduce additional
|
||||
/// basic-blocks as needed, and fill them all.
|
||||
void VPlan::execute(VPTransformState *State) {
|
||||
BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
|
||||
BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
|
||||
assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
|
||||
BasicBlock *VectorLatchBB = VectorHeaderBB;
|
||||
|
||||
// 1. Make room to generate basic-blocks inside loop body if needed.
|
||||
VectorLatchBB = VectorHeaderBB->splitBasicBlock(
|
||||
VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
|
||||
Loop *L = State->LI->getLoopFor(VectorHeaderBB);
|
||||
L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
|
||||
// Remove the edge between Header and Latch to allow other connections.
|
||||
// Temporarily terminate with unreachable until CFG is rewired.
|
||||
// Note: this asserts the generated code's assumption that
|
||||
// getFirstInsertionPt() can be dereferenced into an Instruction.
|
||||
VectorHeaderBB->getTerminator()->eraseFromParent();
|
||||
State->Builder.SetInsertPoint(VectorHeaderBB);
|
||||
UnreachableInst *Terminator = State->Builder.CreateUnreachable();
|
||||
State->Builder.SetInsertPoint(Terminator);
|
||||
|
||||
// 2. Generate code in loop body.
|
||||
State->CFG.PrevVPBB = nullptr;
|
||||
State->CFG.PrevBB = VectorHeaderBB;
|
||||
State->CFG.LastBB = VectorLatchBB;
|
||||
|
||||
for (VPBlockBase *Block : depth_first(Entry))
|
||||
Block->execute(State);
|
||||
|
||||
// 3. Merge the temporary latch created with the last basic-block filled.
|
||||
BasicBlock *LastBB = State->CFG.PrevBB;
|
||||
// Connect LastBB to VectorLatchBB to facilitate their merge.
|
||||
assert(isa<UnreachableInst>(LastBB->getTerminator()) &&
|
||||
"Expected VPlan CFG to terminate with unreachable");
|
||||
LastBB->getTerminator()->eraseFromParent();
|
||||
BranchInst::Create(VectorLatchBB, LastBB);
|
||||
|
||||
// Merge LastBB with Latch.
|
||||
bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
|
||||
(void)Merged;
|
||||
assert(Merged && "Could not merge last basic block with latch.");
|
||||
VectorLatchBB = LastBB;
|
||||
|
||||
updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB);
|
||||
}
|
||||
|
||||
void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
|
||||
BasicBlock *LoopLatchBB) {
|
||||
BasicBlock *LoopHeaderBB = LoopPreHeaderBB->getSingleSuccessor();
|
||||
assert(LoopHeaderBB && "Loop preheader does not have a single successor.");
|
||||
DT->addNewBlock(LoopHeaderBB, LoopPreHeaderBB);
|
||||
// The vector body may be more than a single basic-block by this point.
|
||||
// Update the dominator tree information inside the vector body by propagating
|
||||
// it from header to latch, expecting only triangular control-flow, if any.
|
||||
BasicBlock *PostDomSucc = nullptr;
|
||||
for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
|
||||
// Get the list of successors of this block.
|
||||
std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
|
||||
assert(Succs.size() <= 2 &&
|
||||
"Basic block in vector loop has more than 2 successors.");
|
||||
PostDomSucc = Succs[0];
|
||||
if (Succs.size() == 1) {
|
||||
assert(PostDomSucc->getSinglePredecessor() &&
|
||||
"PostDom successor has more than one predecessor.");
|
||||
DT->addNewBlock(PostDomSucc, BB);
|
||||
continue;
|
||||
}
|
||||
BasicBlock *InterimSucc = Succs[1];
|
||||
if (PostDomSucc->getSingleSuccessor() == InterimSucc) {
|
||||
PostDomSucc = Succs[1];
|
||||
InterimSucc = Succs[0];
|
||||
}
|
||||
assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&
|
||||
"One successor of a basic block does not lead to the other.");
|
||||
assert(InterimSucc->getSinglePredecessor() &&
|
||||
"Interim successor has more than one predecessor.");
|
||||
assert(std::distance(pred_begin(PostDomSucc), pred_end(PostDomSucc)) == 2 &&
|
||||
"PostDom successor has more than two predecessors.");
|
||||
DT->addNewBlock(InterimSucc, BB);
|
||||
DT->addNewBlock(PostDomSucc, BB);
|
||||
}
|
||||
}
|
||||
|
||||
const Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
|
||||
return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
|
||||
Twine(getOrCreateBID(Block));
|
||||
}
|
||||
|
||||
const Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) {
|
||||
const std::string &Name = Block->getName();
|
||||
if (!Name.empty())
|
||||
return Name;
|
||||
return "VPB" + Twine(getOrCreateBID(Block));
|
||||
}
|
||||
|
||||
void VPlanPrinter::dump() {
|
||||
Depth = 1;
|
||||
bumpIndent(0);
|
||||
OS << "digraph VPlan {\n";
|
||||
OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
|
||||
if (!Plan.getName().empty())
|
||||
OS << "\\n" << DOT::EscapeString(Plan.getName());
|
||||
OS << "\"]\n";
|
||||
OS << "node [shape=rect, fontname=Courier, fontsize=30]\n";
|
||||
OS << "edge [fontname=Courier, fontsize=30]\n";
|
||||
OS << "compound=true\n";
|
||||
|
||||
for (VPBlockBase *Block : depth_first(Plan.getEntry()))
|
||||
dumpBlock(Block);
|
||||
|
||||
OS << "}\n";
|
||||
}
|
||||
|
||||
void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
|
||||
if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block))
|
||||
dumpBasicBlock(BasicBlock);
|
||||
else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
|
||||
dumpRegion(Region);
|
||||
else
|
||||
llvm_unreachable("Unsupported kind of VPBlock.");
|
||||
}
|
||||
|
||||
void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
|
||||
bool Hidden, const Twine &Label) {
|
||||
// Due to "dot" we print an edge between two regions as an edge between the
|
||||
// exit basic block and the entry basic of the respective regions.
|
||||
const VPBlockBase *Tail = From->getExitBasicBlock();
|
||||
const VPBlockBase *Head = To->getEntryBasicBlock();
|
||||
OS << Indent << getUID(Tail) << " -> " << getUID(Head);
|
||||
OS << " [ label=\"" << Label << '\"';
|
||||
if (Tail != From)
|
||||
OS << " ltail=" << getUID(From);
|
||||
if (Head != To)
|
||||
OS << " lhead=" << getUID(To);
|
||||
if (Hidden)
|
||||
OS << "; splines=none";
|
||||
OS << "]\n";
|
||||
}
|
||||
|
||||
void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
|
||||
auto &Successors = Block->getSuccessors();
|
||||
if (Successors.size() == 1)
|
||||
drawEdge(Block, Successors.front(), false, "");
|
||||
else if (Successors.size() == 2) {
|
||||
drawEdge(Block, Successors.front(), false, "T");
|
||||
drawEdge(Block, Successors.back(), false, "F");
|
||||
} else {
|
||||
unsigned SuccessorNumber = 0;
|
||||
for (auto *Successor : Successors)
|
||||
drawEdge(Block, Successor, false, Twine(SuccessorNumber++));
|
||||
}
|
||||
}
|
||||
|
||||
void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
|
||||
OS << Indent << getUID(BasicBlock) << " [label =\n";
|
||||
bumpIndent(1);
|
||||
OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
|
||||
bumpIndent(1);
|
||||
for (const VPRecipeBase &Recipe : *BasicBlock)
|
||||
Recipe.print(OS, Indent);
|
||||
bumpIndent(-2);
|
||||
OS << "\n" << Indent << "]\n";
|
||||
dumpEdges(BasicBlock);
|
||||
}
|
||||
|
||||
void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
|
||||
OS << Indent << "subgraph " << getUID(Region) << " {\n";
|
||||
bumpIndent(1);
|
||||
OS << Indent << "fontname=Courier\n"
|
||||
<< Indent << "label=\""
|
||||
<< DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ")
|
||||
<< DOT::EscapeString(Region->getName()) << "\"\n";
|
||||
// Dump the blocks of the region.
|
||||
assert(Region->getEntry() && "Region contains no inner blocks.");
|
||||
for (const VPBlockBase *Block : depth_first(Region->getEntry()))
|
||||
dumpBlock(Block);
|
||||
bumpIndent(-1);
|
||||
OS << Indent << "}\n";
|
||||
dumpEdges(Region);
|
||||
}
|
||||
|
||||
void VPlanPrinter::printAsIngredient(raw_ostream &O, Value *V) {
|
||||
std::string IngredientString;
|
||||
raw_string_ostream RSO(IngredientString);
|
||||
if (auto *Inst = dyn_cast<Instruction>(V)) {
|
||||
if (!Inst->getType()->isVoidTy()) {
|
||||
Inst->printAsOperand(RSO, false);
|
||||
RSO << " = ";
|
||||
}
|
||||
RSO << Inst->getOpcodeName() << " ";
|
||||
unsigned E = Inst->getNumOperands();
|
||||
if (E > 0) {
|
||||
Inst->getOperand(0)->printAsOperand(RSO, false);
|
||||
for (unsigned I = 1; I < E; ++I)
|
||||
Inst->getOperand(I)->printAsOperand(RSO << ", ", false);
|
||||
}
|
||||
} else // !Inst
|
||||
V->printAsOperand(RSO, false);
|
||||
RSO.flush();
|
||||
O << DOT::EscapeString(IngredientString);
|
||||
}
|
||||
@ -1,789 +0,0 @@
|
||||
//===- VPlan.h - Represent A Vectorizer Plan ------------------------------===//
|
||||
//
|
||||
// The LLVM Compiler Infrastructure
|
||||
//
|
||||
// This file is distributed under the University of Illinois Open Source
|
||||
// License. See LICENSE.TXT for details.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
///
|
||||
/// \file
|
||||
/// This file contains the declarations of the Vectorization Plan base classes:
|
||||
/// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
|
||||
/// VPBlockBase, together implementing a Hierarchical CFG;
|
||||
/// 2. Specializations of GraphTraits that allow VPBlockBase graphs to be
|
||||
/// treated as proper graphs for generic algorithms;
|
||||
/// 3. Pure virtual VPRecipeBase serving as the base class for recipes contained
|
||||
/// within VPBasicBlocks;
|
||||
/// 4. The VPlan class holding a candidate for vectorization;
|
||||
/// 5. The VPlanPrinter class providing a way to print a plan in dot format.
|
||||
/// These are documented in docs/VectorizationPlan.rst.
|
||||
///
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
||||
#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
||||
|
||||
#include "llvm/ADT/GraphTraits.h"
|
||||
#include "llvm/ADT/SmallSet.h"
|
||||
#include "llvm/ADT/ilist.h"
|
||||
#include "llvm/ADT/ilist_node.h"
|
||||
#include "llvm/IR/IRBuilder.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
|
||||
// The (re)use of existing LoopVectorize classes is subject to future VPlan
|
||||
// refactoring.
|
||||
namespace {
|
||||
// Forward declarations.
|
||||
//class InnerLoopVectorizer;
|
||||
class LoopVectorizationLegality;
|
||||
class LoopVectorizationCostModel;
|
||||
} // namespace
|
||||
|
||||
namespace llvm {
|
||||
|
||||
// Forward declarations.
|
||||
class BasicBlock;
|
||||
class InnerLoopVectorizer;
|
||||
class VPBasicBlock;
|
||||
|
||||
/// In what follows, the term "input IR" refers to code that is fed into the
|
||||
/// vectorizer whereas the term "output IR" refers to code that is generated by
|
||||
/// the vectorizer.
|
||||
|
||||
/// VPIteration represents a single point in the iteration space of the output
|
||||
/// (vectorized and/or unrolled) IR loop.
|
||||
struct VPIteration {
|
||||
unsigned Part; ///< in [0..UF)
|
||||
unsigned Lane; ///< in [0..VF)
|
||||
};
|
||||
|
||||
/// This is a helper struct for maintaining vectorization state. It's used for
|
||||
/// mapping values from the original loop to their corresponding values in
|
||||
/// the new loop. Two mappings are maintained: one for vectorized values and
|
||||
/// one for scalarized values. Vectorized values are represented with UF
|
||||
/// vector values in the new loop, and scalarized values are represented with
|
||||
/// UF x VF scalar values in the new loop. UF and VF are the unroll and
|
||||
/// vectorization factors, respectively.
|
||||
///
|
||||
/// Entries can be added to either map with setVectorValue and setScalarValue,
|
||||
/// which assert that an entry was not already added before. If an entry is to
|
||||
/// replace an existing one, call resetVectorValue and resetScalarValue. This is
|
||||
/// currently needed to modify the mapped values during "fix-up" operations that
|
||||
/// occur once the first phase of widening is complete. These operations include
|
||||
/// type truncation and the second phase of recurrence widening.
|
||||
///
|
||||
/// Entries from either map can be retrieved using the getVectorValue and
|
||||
/// getScalarValue functions, which assert that the desired value exists.
|
||||
|
||||
struct VectorizerValueMap {
|
||||
private:
|
||||
/// The unroll factor. Each entry in the vector map contains UF vector values.
|
||||
unsigned UF;
|
||||
|
||||
/// The vectorization factor. Each entry in the scalar map contains UF x VF
|
||||
/// scalar values.
|
||||
unsigned VF;
|
||||
|
||||
/// The vector and scalar map storage. We use std::map and not DenseMap
|
||||
/// because insertions to DenseMap invalidate its iterators.
|
||||
typedef SmallVector<Value *, 2> VectorParts;
|
||||
typedef SmallVector<SmallVector<Value *, 4>, 2> ScalarParts;
|
||||
std::map<Value *, VectorParts> VectorMapStorage;
|
||||
std::map<Value *, ScalarParts> ScalarMapStorage;
|
||||
|
||||
public:
|
||||
/// Construct an empty map with the given unroll and vectorization factors.
|
||||
VectorizerValueMap(unsigned UF, unsigned VF) : UF(UF), VF(VF) {}
|
||||
|
||||
/// \return True if the map has any vector entry for \p Key.
|
||||
bool hasAnyVectorValue(Value *Key) const {
|
||||
return VectorMapStorage.count(Key);
|
||||
}
|
||||
|
||||
/// \return True if the map has a vector entry for \p Key and \p Part.
|
||||
bool hasVectorValue(Value *Key, unsigned Part) const {
|
||||
assert(Part < UF && "Queried Vector Part is too large.");
|
||||
if (!hasAnyVectorValue(Key))
|
||||
return false;
|
||||
const VectorParts &Entry = VectorMapStorage.find(Key)->second;
|
||||
assert(Entry.size() == UF && "VectorParts has wrong dimensions.");
|
||||
return Entry[Part] != nullptr;
|
||||
}
|
||||
|
||||
/// \return True if the map has any scalar entry for \p Key.
|
||||
bool hasAnyScalarValue(Value *Key) const {
|
||||
return ScalarMapStorage.count(Key);
|
||||
}
|
||||
|
||||
/// \return True if the map has a scalar entry for \p Key and \p Instance.
|
||||
bool hasScalarValue(Value *Key, const VPIteration &Instance) const {
|
||||
assert(Instance.Part < UF && "Queried Scalar Part is too large.");
|
||||
assert(Instance.Lane < VF && "Queried Scalar Lane is too large.");
|
||||
if (!hasAnyScalarValue(Key))
|
||||
return false;
|
||||
const ScalarParts &Entry = ScalarMapStorage.find(Key)->second;
|
||||
assert(Entry.size() == UF && "ScalarParts has wrong dimensions.");
|
||||
assert(Entry[Instance.Part].size() == VF &&
|
||||
"ScalarParts has wrong dimensions.");
|
||||
return Entry[Instance.Part][Instance.Lane] != nullptr;
|
||||
}
|
||||
|
||||
/// Retrieve the existing vector value that corresponds to \p Key and
|
||||
/// \p Part.
|
||||
Value *getVectorValue(Value *Key, unsigned Part) {
|
||||
assert(hasVectorValue(Key, Part) && "Getting non-existent value.");
|
||||
return VectorMapStorage[Key][Part];
|
||||
}
|
||||
|
||||
/// Retrieve the existing scalar value that corresponds to \p Key and
|
||||
/// \p Instance.
|
||||
Value *getScalarValue(Value *Key, const VPIteration &Instance) {
|
||||
assert(hasScalarValue(Key, Instance) && "Getting non-existent value.");
|
||||
return ScalarMapStorage[Key][Instance.Part][Instance.Lane];
|
||||
}
|
||||
|
||||
/// Set a vector value associated with \p Key and \p Part. Assumes such a
|
||||
/// value is not already set. If it is, use resetVectorValue() instead.
|
||||
void setVectorValue(Value *Key, unsigned Part, Value *Vector) {
|
||||
assert(!hasVectorValue(Key, Part) && "Vector value already set for part");
|
||||
if (!VectorMapStorage.count(Key)) {
|
||||
VectorParts Entry(UF);
|
||||
VectorMapStorage[Key] = Entry;
|
||||
}
|
||||
VectorMapStorage[Key][Part] = Vector;
|
||||
}
|
||||
|
||||
/// Set a scalar value associated with \p Key and \p Instance. Assumes such a
|
||||
/// value is not already set.
|
||||
void setScalarValue(Value *Key, const VPIteration &Instance, Value *Scalar) {
|
||||
assert(!hasScalarValue(Key, Instance) && "Scalar value already set");
|
||||
if (!ScalarMapStorage.count(Key)) {
|
||||
ScalarParts Entry(UF);
|
||||
// TODO: Consider storing uniform values only per-part, as they occupy
|
||||
// lane 0 only, keeping the other VF-1 redundant entries null.
|
||||
for (unsigned Part = 0; Part < UF; ++Part)
|
||||
Entry[Part].resize(VF, nullptr);
|
||||
ScalarMapStorage[Key] = Entry;
|
||||
}
|
||||
ScalarMapStorage[Key][Instance.Part][Instance.Lane] = Scalar;
|
||||
}
|
||||
|
||||
/// Reset the vector value associated with \p Key for the given \p Part.
|
||||
/// This function can be used to update values that have already been
|
||||
/// vectorized. This is the case for "fix-up" operations including type
|
||||
/// truncation and the second phase of recurrence vectorization.
|
||||
void resetVectorValue(Value *Key, unsigned Part, Value *Vector) {
|
||||
assert(hasVectorValue(Key, Part) && "Vector value not set for part");
|
||||
VectorMapStorage[Key][Part] = Vector;
|
||||
}
|
||||
|
||||
/// Reset the scalar value associated with \p Key for \p Part and \p Lane.
|
||||
/// This function can be used to update values that have already been
|
||||
/// scalarized. This is the case for "fix-up" operations including scalar phi
|
||||
/// nodes for scalarized and predicated instructions.
|
||||
void resetScalarValue(Value *Key, const VPIteration &Instance,
|
||||
Value *Scalar) {
|
||||
assert(hasScalarValue(Key, Instance) &&
|
||||
"Scalar value not set for part and lane");
|
||||
ScalarMapStorage[Key][Instance.Part][Instance.Lane] = Scalar;
|
||||
}
|
||||
};
|
||||
|
||||
/// VPTransformState holds information passed down when "executing" a VPlan,
|
||||
/// needed for generating the output IR.
|
||||
struct VPTransformState {
|
||||
|
||||
VPTransformState(unsigned VF, unsigned UF, class LoopInfo *LI,
|
||||
class DominatorTree *DT, IRBuilder<> &Builder,
|
||||
VectorizerValueMap &ValueMap, InnerLoopVectorizer *ILV)
|
||||
: VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder),
|
||||
ValueMap(ValueMap), ILV(ILV) {}
|
||||
|
||||
/// The chosen Vectorization and Unroll Factors of the loop being vectorized.
|
||||
unsigned VF;
|
||||
unsigned UF;
|
||||
|
||||
/// Hold the indices to generate specific scalar instructions. Null indicates
|
||||
/// that all instances are to be generated, using either scalar or vector
|
||||
/// instructions.
|
||||
Optional<VPIteration> Instance;
|
||||
|
||||
/// Hold state information used when constructing the CFG of the output IR,
|
||||
/// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
|
||||
struct CFGState {
|
||||
/// The previous VPBasicBlock visited. Initially set to null.
|
||||
VPBasicBlock *PrevVPBB;
|
||||
/// The previous IR BasicBlock created or used. Initially set to the new
|
||||
/// header BasicBlock.
|
||||
BasicBlock *PrevBB;
|
||||
/// The last IR BasicBlock in the output IR. Set to the new latch
|
||||
/// BasicBlock, used for placing the newly created BasicBlocks.
|
||||
BasicBlock *LastBB;
|
||||
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
|
||||
/// of replication, maps the BasicBlock of the last replica created.
|
||||
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
|
||||
|
||||
CFGState() : PrevVPBB(nullptr), PrevBB(nullptr), LastBB(nullptr) {}
|
||||
} CFG;
|
||||
|
||||
/// Hold a pointer to LoopInfo to register new basic blocks in the loop.
|
||||
class LoopInfo *LI;
|
||||
|
||||
/// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
|
||||
class DominatorTree *DT;
|
||||
|
||||
/// Hold a reference to the IRBuilder used to generate output IR code.
|
||||
IRBuilder<> &Builder;
|
||||
|
||||
/// Hold a reference to the Value state information used when generating the
|
||||
/// Values of the output IR.
|
||||
VectorizerValueMap &ValueMap;
|
||||
|
||||
/// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
|
||||
class InnerLoopVectorizer *ILV;
|
||||
};
|
||||
|
||||
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
|
||||
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
|
||||
class VPBlockBase {
|
||||
private:
|
||||
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
|
||||
|
||||
/// An optional name for the block.
|
||||
std::string Name;
|
||||
|
||||
/// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
|
||||
/// it is a topmost VPBlockBase.
|
||||
class VPRegionBlock *Parent;
|
||||
|
||||
/// List of predecessor blocks.
|
||||
SmallVector<VPBlockBase *, 1> Predecessors;
|
||||
|
||||
/// List of successor blocks.
|
||||
SmallVector<VPBlockBase *, 1> Successors;
|
||||
|
||||
/// Add \p Successor as the last successor to this block.
|
||||
void appendSuccessor(VPBlockBase *Successor) {
|
||||
assert(Successor && "Cannot add nullptr successor!");
|
||||
Successors.push_back(Successor);
|
||||
}
|
||||
|
||||
/// Add \p Predecessor as the last predecessor to this block.
|
||||
void appendPredecessor(VPBlockBase *Predecessor) {
|
||||
assert(Predecessor && "Cannot add nullptr predecessor!");
|
||||
Predecessors.push_back(Predecessor);
|
||||
}
|
||||
|
||||
/// Remove \p Predecessor from the predecessors of this block.
|
||||
void removePredecessor(VPBlockBase *Predecessor) {
|
||||
auto Pos = std::find(Predecessors.begin(), Predecessors.end(), Predecessor);
|
||||
assert(Pos && "Predecessor does not exist");
|
||||
Predecessors.erase(Pos);
|
||||
}
|
||||
|
||||
/// Remove \p Successor from the successors of this block.
|
||||
void removeSuccessor(VPBlockBase *Successor) {
|
||||
auto Pos = std::find(Successors.begin(), Successors.end(), Successor);
|
||||
assert(Pos && "Successor does not exist");
|
||||
Successors.erase(Pos);
|
||||
}
|
||||
|
||||
protected:
|
||||
VPBlockBase(const unsigned char SC, const std::string &N)
|
||||
: SubclassID(SC), Name(N), Parent(nullptr) {}
|
||||
|
||||
public:
|
||||
/// An enumeration for keeping track of the concrete subclass of VPBlockBase
|
||||
/// that are actually instantiated. Values of this enumeration are kept in the
|
||||
/// SubclassID field of the VPBlockBase objects. They are used for concrete
|
||||
/// type identification.
|
||||
typedef enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy;
|
||||
|
||||
typedef SmallVectorImpl<VPBlockBase *> VPBlocksTy;
|
||||
|
||||
virtual ~VPBlockBase() {}
|
||||
|
||||
const std::string &getName() const { return Name; }
|
||||
|
||||
void setName(const Twine &newName) { Name = newName.str(); }
|
||||
|
||||
/// \return an ID for the concrete type of this object.
|
||||
/// This is used to implement the classof checks. This should not be used
|
||||
/// for any other purpose, as the values may change as LLVM evolves.
|
||||
unsigned getVPBlockID() const { return SubclassID; }
|
||||
|
||||
const VPRegionBlock *getParent() const { return Parent; }
|
||||
|
||||
void setParent(VPRegionBlock *P) { Parent = P; }
|
||||
|
||||
/// \return the VPBasicBlock that is the entry of this VPBlockBase,
|
||||
/// recursively, if the latter is a VPRegionBlock. Otherwise, if this
|
||||
/// VPBlockBase is a VPBasicBlock, it is returned.
|
||||
const VPBasicBlock *getEntryBasicBlock() const;
|
||||
VPBasicBlock *getEntryBasicBlock();
|
||||
|
||||
/// \return the VPBasicBlock that is the exit of this VPBlockBase,
|
||||
/// recursively, if the latter is a VPRegionBlock. Otherwise, if this
|
||||
/// VPBlockBase is a VPBasicBlock, it is returned.
|
||||
const VPBasicBlock *getExitBasicBlock() const;
|
||||
VPBasicBlock *getExitBasicBlock();
|
||||
|
||||
const VPBlocksTy &getSuccessors() const { return Successors; }
|
||||
VPBlocksTy &getSuccessors() { return Successors; }
|
||||
|
||||
const VPBlocksTy &getPredecessors() const { return Predecessors; }
|
||||
VPBlocksTy &getPredecessors() { return Predecessors; }
|
||||
|
||||
/// \return the successor of this VPBlockBase if it has a single successor.
|
||||
/// Otherwise return a null pointer.
|
||||
VPBlockBase *getSingleSuccessor() const {
|
||||
return (Successors.size() == 1 ? *Successors.begin() : nullptr);
|
||||
}
|
||||
|
||||
/// \return the predecessor of this VPBlockBase if it has a single
|
||||
/// predecessor. Otherwise return a null pointer.
|
||||
VPBlockBase *getSinglePredecessor() const {
|
||||
return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
|
||||
}
|
||||
|
||||
/// An Enclosing Block of a block B is any block containing B, including B
|
||||
/// itself. \return the closest enclosing block starting from "this", which
|
||||
/// has successors. \return the root enclosing block if all enclosing blocks
|
||||
/// have no successors.
|
||||
VPBlockBase *getEnclosingBlockWithSuccessors();
|
||||
|
||||
/// \return the closest enclosing block starting from "this", which has
|
||||
/// predecessors. \return the root enclosing block if all enclosing blocks
|
||||
/// have no predecessors.
|
||||
VPBlockBase *getEnclosingBlockWithPredecessors();
|
||||
|
||||
/// \return the successors either attached directly to this VPBlockBase or, if
|
||||
/// this VPBlockBase is the exit block of a VPRegionBlock and has no
|
||||
/// successors of its own, search recursively for the first enclosing
|
||||
/// VPRegionBlock that has successors and return them. If no such
|
||||
/// VPRegionBlock exists, return the (empty) successors of the topmost
|
||||
/// VPBlockBase reached.
|
||||
const VPBlocksTy &getHierarchicalSuccessors() {
|
||||
return getEnclosingBlockWithSuccessors()->getSuccessors();
|
||||
}
|
||||
|
||||
/// \return the hierarchical successor of this VPBlockBase if it has a single
|
||||
/// hierarchical successor. Otherwise return a null pointer.
|
||||
VPBlockBase *getSingleHierarchicalSuccessor() {
|
||||
return getEnclosingBlockWithSuccessors()->getSingleSuccessor();
|
||||
}
|
||||
|
||||
/// \return the predecessors either attached directly to this VPBlockBase or,
|
||||
/// if this VPBlockBase is the entry block of a VPRegionBlock and has no
|
||||
/// predecessors of its own, search recursively for the first enclosing
|
||||
/// VPRegionBlock that has predecessors and return them. If no such
|
||||
/// VPRegionBlock exists, return the (empty) predecessors of the topmost
|
||||
/// VPBlockBase reached.
|
||||
const VPBlocksTy &getHierarchicalPredecessors() {
|
||||
return getEnclosingBlockWithPredecessors()->getPredecessors();
|
||||
}
|
||||
|
||||
/// \return the hierarchical predecessor of this VPBlockBase if it has a
|
||||
/// single hierarchical predecessor. Otherwise return a null pointer.
|
||||
VPBlockBase *getSingleHierarchicalPredecessor() {
|
||||
return getEnclosingBlockWithPredecessors()->getSinglePredecessor();
|
||||
}
|
||||
|
||||
/// Sets a given VPBlockBase \p Successor as the single successor and \return
|
||||
/// \p Successor. The parent of this Block is copied to be the parent of
|
||||
/// \p Successor.
|
||||
VPBlockBase *setOneSuccessor(VPBlockBase *Successor) {
|
||||
assert(Successors.empty() && "Setting one successor when others exist.");
|
||||
appendSuccessor(Successor);
|
||||
Successor->appendPredecessor(this);
|
||||
Successor->Parent = Parent;
|
||||
return Successor;
|
||||
}
|
||||
|
||||
/// Sets two given VPBlockBases \p IfTrue and \p IfFalse to be the two
|
||||
/// successors. The parent of this Block is copied to be the parent of both
|
||||
/// \p IfTrue and \p IfFalse.
|
||||
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
|
||||
assert(Successors.empty() && "Setting two successors when others exist.");
|
||||
appendSuccessor(IfTrue);
|
||||
appendSuccessor(IfFalse);
|
||||
IfTrue->appendPredecessor(this);
|
||||
IfFalse->appendPredecessor(this);
|
||||
IfTrue->Parent = Parent;
|
||||
IfFalse->Parent = Parent;
|
||||
}
|
||||
|
||||
void disconnectSuccessor(VPBlockBase *Successor) {
|
||||
assert(Successor && "Successor to disconnect is null.");
|
||||
removeSuccessor(Successor);
|
||||
Successor->removePredecessor(this);
|
||||
}
|
||||
|
||||
/// The method which generates the output IR that correspond to this
|
||||
/// VPBlockBase, thereby "executing" the VPlan.
|
||||
virtual void execute(struct VPTransformState *State) = 0;
|
||||
|
||||
/// Delete all blocks reachable from a given VPBlockBase, inclusive.
|
||||
static void deleteCFG(VPBlockBase *Entry);
|
||||
};
|
||||
|
||||
/// VPRecipeBase is a base class modeling a sequence of one or more output IR
|
||||
/// instructions.
|
||||
class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock> {
|
||||
friend VPBasicBlock;
|
||||
|
||||
private:
|
||||
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
|
||||
|
||||
/// Each VPRecipe belongs to a single VPBasicBlock.
|
||||
VPBasicBlock *Parent;
|
||||
|
||||
public:
|
||||
/// An enumeration for keeping track of the concrete subclass of VPRecipeBase
|
||||
/// that is actually instantiated. Values of this enumeration are kept in the
|
||||
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
|
||||
/// type identification.
|
||||
typedef enum {
|
||||
VPBranchOnMaskSC,
|
||||
VPInterleaveSC,
|
||||
VPPredInstPHISC,
|
||||
VPReplicateSC,
|
||||
VPWidenIntOrFpInductionSC,
|
||||
VPWidenPHISC,
|
||||
VPWidenSC,
|
||||
} VPRecipeTy;
|
||||
|
||||
VPRecipeBase(const unsigned char SC) : SubclassID(SC), Parent(nullptr) {}
|
||||
|
||||
virtual ~VPRecipeBase() {}
|
||||
|
||||
/// \return an ID for the concrete type of this object.
|
||||
/// This is used to implement the classof checks. This should not be used
|
||||
/// for any other purpose, as the values may change as LLVM evolves.
|
||||
unsigned getVPRecipeID() const { return SubclassID; }
|
||||
|
||||
/// \return the VPBasicBlock which this VPRecipe belongs to.
|
||||
VPBasicBlock *getParent() { return Parent; }
|
||||
const VPBasicBlock *getParent() const { return Parent; }
|
||||
|
||||
/// The method which generates the output IR instructions that correspond to
|
||||
/// this VPRecipe, thereby "executing" the VPlan.
|
||||
virtual void execute(struct VPTransformState &State) = 0;
|
||||
|
||||
/// Each recipe prints itself.
|
||||
virtual void print(raw_ostream &O, const Twine &Indent) const = 0;
|
||||
};
|
||||
|
||||
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
|
||||
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
|
||||
/// output IR instructions.
|
||||
class VPBasicBlock : public VPBlockBase {
|
||||
public:
|
||||
typedef iplist<VPRecipeBase> RecipeListTy;
|
||||
|
||||
private:
|
||||
/// The VPRecipes held in the order of output instructions to generate.
|
||||
RecipeListTy Recipes;
|
||||
|
||||
public:
|
||||
/// Instruction iterators...
|
||||
typedef RecipeListTy::iterator iterator;
|
||||
typedef RecipeListTy::const_iterator const_iterator;
|
||||
typedef RecipeListTy::reverse_iterator reverse_iterator;
|
||||
typedef RecipeListTy::const_reverse_iterator const_reverse_iterator;
|
||||
|
||||
//===--------------------------------------------------------------------===//
|
||||
/// Recipe iterator methods
|
||||
///
|
||||
inline iterator begin() { return Recipes.begin(); }
|
||||
inline const_iterator begin() const { return Recipes.begin(); }
|
||||
inline iterator end() { return Recipes.end(); }
|
||||
inline const_iterator end() const { return Recipes.end(); }
|
||||
|
||||
inline reverse_iterator rbegin() { return Recipes.rbegin(); }
|
||||
inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
|
||||
inline reverse_iterator rend() { return Recipes.rend(); }
|
||||
inline const_reverse_iterator rend() const { return Recipes.rend(); }
|
||||
|
||||
inline size_t size() const { return Recipes.size(); }
|
||||
inline bool empty() const { return Recipes.empty(); }
|
||||
inline const VPRecipeBase &front() const { return Recipes.front(); }
|
||||
inline VPRecipeBase &front() { return Recipes.front(); }
|
||||
inline const VPRecipeBase &back() const { return Recipes.back(); }
|
||||
inline VPRecipeBase &back() { return Recipes.back(); }
|
||||
|
||||
/// \brief Returns a pointer to a member of the recipe list.
|
||||
static RecipeListTy VPBasicBlock::*getSublistAccess(VPRecipeBase *) {
|
||||
return &VPBasicBlock::Recipes;
|
||||
}
|
||||
|
||||
VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
|
||||
: VPBlockBase(VPBasicBlockSC, Name.str()) {
|
||||
if (Recipe)
|
||||
appendRecipe(Recipe);
|
||||
}
|
||||
|
||||
~VPBasicBlock() { Recipes.clear(); }
|
||||
|
||||
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
||||
static inline bool classof(const VPBlockBase *V) {
|
||||
return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
|
||||
}
|
||||
|
||||
/// Augment the existing recipes of a VPBasicBlock with an additional
|
||||
/// \p Recipe as the last recipe.
|
||||
void appendRecipe(VPRecipeBase *Recipe) {
|
||||
assert(Recipe && "No recipe to append.");
|
||||
assert(!Recipe->Parent && "Recipe already in VPlan");
|
||||
Recipe->Parent = this;
|
||||
return Recipes.push_back(Recipe);
|
||||
}
|
||||
|
||||
/// The method which generates the output IR instructions that correspond to
|
||||
/// this VPBasicBlock, thereby "executing" the VPlan.
|
||||
void execute(struct VPTransformState *State) override;
|
||||
|
||||
private:
|
||||
/// Create an IR BasicBlock to hold the output instructions generated by this
|
||||
/// VPBasicBlock, and return it. Update the CFGState accordingly.
|
||||
BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
|
||||
};
|
||||
|
||||
/// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
|
||||
/// which form a Single-Entry-Single-Exit subgraph of the output IR CFG.
|
||||
/// A VPRegionBlock may indicate that its contents are to be replicated several
|
||||
/// times. This is designed to support predicated scalarization, in which a
|
||||
/// scalar if-then code structure needs to be generated VF * UF times. Having
|
||||
/// this replication indicator helps to keep a single model for multiple
|
||||
/// candidate VF's. The actual replication takes place only once the desired VF
|
||||
/// and UF have been determined.
|
||||
class VPRegionBlock : public VPBlockBase {
|
||||
private:
|
||||
/// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
|
||||
VPBlockBase *Entry;
|
||||
|
||||
/// Hold the Single Exit of the SESE region modelled by the VPRegionBlock.
|
||||
VPBlockBase *Exit;
|
||||
|
||||
/// An indicator whether this region is to generate multiple replicated
|
||||
/// instances of output IR corresponding to its VPBlockBases.
|
||||
bool IsReplicator;
|
||||
|
||||
public:
|
||||
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exit,
|
||||
const std::string &Name = "", bool IsReplicator = false)
|
||||
: VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exit(Exit),
|
||||
IsReplicator(IsReplicator) {
|
||||
assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
|
||||
assert(Exit->getSuccessors().empty() && "Exit block has successors.");
|
||||
Entry->setParent(this);
|
||||
Exit->setParent(this);
|
||||
}
|
||||
|
||||
~VPRegionBlock() {
|
||||
if (Entry)
|
||||
deleteCFG(Entry);
|
||||
}
|
||||
|
||||
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
||||
static inline bool classof(const VPBlockBase *V) {
|
||||
return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
|
||||
}
|
||||
|
||||
const VPBlockBase *getEntry() const { return Entry; }
|
||||
VPBlockBase *getEntry() { return Entry; }
|
||||
|
||||
const VPBlockBase *getExit() const { return Exit; }
|
||||
VPBlockBase *getExit() { return Exit; }
|
||||
|
||||
/// An indicator whether this region is to generate multiple replicated
|
||||
/// instances of output IR corresponding to its VPBlockBases.
|
||||
bool isReplicator() const { return IsReplicator; }
|
||||
|
||||
/// The method which generates the output IR instructions that correspond to
|
||||
/// this VPRegionBlock, thereby "executing" the VPlan.
|
||||
void execute(struct VPTransformState *State) override;
|
||||
};
|
||||
|
||||
/// VPlan models a candidate for vectorization, encoding various decisions take
|
||||
/// to produce efficient output IR, including which branches, basic-blocks and
|
||||
/// output IR instructions to generate, and their cost. VPlan holds a
|
||||
/// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
|
||||
/// VPBlock.
|
||||
class VPlan {
|
||||
private:
|
||||
/// Hold the single entry to the Hierarchical CFG of the VPlan.
|
||||
VPBlockBase *Entry;
|
||||
|
||||
/// Holds the VFs applicable to this VPlan.
|
||||
SmallSet<unsigned, 2> VFs;
|
||||
|
||||
/// Holds the name of the VPlan, for printing.
|
||||
std::string Name;
|
||||
|
||||
public:
|
||||
VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {}
|
||||
|
||||
~VPlan() {
|
||||
if (Entry)
|
||||
VPBlockBase::deleteCFG(Entry);
|
||||
}
|
||||
|
||||
/// Generate the IR code for this VPlan.
|
||||
void execute(struct VPTransformState *State);
|
||||
|
||||
VPBlockBase *getEntry() { return Entry; }
|
||||
const VPBlockBase *getEntry() const { return Entry; }
|
||||
|
||||
VPBlockBase *setEntry(VPBlockBase *Block) { return Entry = Block; }
|
||||
|
||||
void addVF(unsigned VF) { VFs.insert(VF); }
|
||||
|
||||
bool hasVF(unsigned VF) { return VFs.count(VF); }
|
||||
|
||||
const std::string &getName() const { return Name; }
|
||||
|
||||
void setName(const Twine &newName) { Name = newName.str(); }
|
||||
|
||||
private:
|
||||
/// Add to the given dominator tree the header block and every new basic block
|
||||
/// that was created between it and the latch block, inclusive.
|
||||
static void updateDominatorTree(class DominatorTree *DT,
|
||||
BasicBlock *LoopPreHeaderBB,
|
||||
BasicBlock *LoopLatchBB);
|
||||
};
|
||||
|
||||
/// VPlanPrinter prints a given VPlan to a given output stream. The printing is
|
||||
/// indented and follows the dot format.
|
||||
class VPlanPrinter {
|
||||
friend inline raw_ostream &operator<<(raw_ostream &OS, VPlan &Plan);
|
||||
friend inline raw_ostream &operator<<(raw_ostream &OS,
|
||||
const struct VPlanIngredient &I);
|
||||
|
||||
private:
|
||||
raw_ostream &OS;
|
||||
VPlan &Plan;
|
||||
unsigned Depth;
|
||||
unsigned TabWidth = 2;
|
||||
std::string Indent;
|
||||
|
||||
unsigned BID = 0;
|
||||
|
||||
SmallDenseMap<const VPBlockBase *, unsigned> BlockID;
|
||||
|
||||
/// Handle indentation.
|
||||
void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
|
||||
|
||||
/// Print a given \p Block of the Plan.
|
||||
void dumpBlock(const VPBlockBase *Block);
|
||||
|
||||
/// Print the information related to the CFG edges going out of a given
|
||||
/// \p Block, followed by printing the successor blocks themselves.
|
||||
void dumpEdges(const VPBlockBase *Block);
|
||||
|
||||
/// Print a given \p BasicBlock, including its VPRecipes, followed by printing
|
||||
/// its successor blocks.
|
||||
void dumpBasicBlock(const VPBasicBlock *BasicBlock);
|
||||
|
||||
/// Print a given \p Region of the Plan.
|
||||
void dumpRegion(const VPRegionBlock *Region);
|
||||
|
||||
unsigned getOrCreateBID(const VPBlockBase *Block) {
|
||||
return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
|
||||
}
|
||||
|
||||
const Twine getOrCreateName(const VPBlockBase *Block);
|
||||
|
||||
const Twine getUID(const VPBlockBase *Block);
|
||||
|
||||
/// Print the information related to a CFG edge between two VPBlockBases.
|
||||
void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
|
||||
const Twine &Label);
|
||||
|
||||
VPlanPrinter(raw_ostream &O, VPlan &P) : OS(O), Plan(P) {}
|
||||
|
||||
void dump();
|
||||
|
||||
static void printAsIngredient(raw_ostream &O, Value *V);
|
||||
};
|
||||
|
||||
struct VPlanIngredient {
|
||||
Value *V;
|
||||
VPlanIngredient(Value *V) : V(V) {}
|
||||
};
|
||||
|
||||
inline raw_ostream &operator<<(raw_ostream &OS, const VPlanIngredient &I) {
|
||||
VPlanPrinter::printAsIngredient(OS, I.V);
|
||||
return OS;
|
||||
}
|
||||
|
||||
inline raw_ostream &operator<<(raw_ostream &OS, VPlan &Plan) {
|
||||
VPlanPrinter Printer(OS, Plan);
|
||||
Printer.dump();
|
||||
return OS;
|
||||
}
|
||||
|
||||
//===--------------------------------------------------------------------===//
|
||||
// GraphTraits specializations for VPlan/VPRegionBlock Control-Flow Graphs //
|
||||
//===--------------------------------------------------------------------===//
|
||||
|
||||
// Provide specializations of GraphTraits to be able to treat a VPBlockBase as a
|
||||
// graph of VPBlockBase nodes...
|
||||
|
||||
template <> struct GraphTraits<VPBlockBase *> {
|
||||
typedef VPBlockBase *NodeRef;
|
||||
typedef SmallVectorImpl<VPBlockBase *>::iterator ChildIteratorType;
|
||||
|
||||
static NodeRef getEntryNode(NodeRef N) { return N; }
|
||||
|
||||
static inline ChildIteratorType child_begin(NodeRef N) {
|
||||
return N->getSuccessors().begin();
|
||||
}
|
||||
|
||||
static inline ChildIteratorType child_end(NodeRef N) {
|
||||
return N->getSuccessors().end();
|
||||
}
|
||||
};
|
||||
|
||||
template <> struct GraphTraits<const VPBlockBase *> {
|
||||
typedef const VPBlockBase *NodeRef;
|
||||
typedef SmallVectorImpl<VPBlockBase *>::const_iterator ChildIteratorType;
|
||||
|
||||
static NodeRef getEntryNode(NodeRef N) { return N; }
|
||||
|
||||
static inline ChildIteratorType child_begin(NodeRef N) {
|
||||
return N->getSuccessors().begin();
|
||||
}
|
||||
|
||||
static inline ChildIteratorType child_end(NodeRef N) {
|
||||
return N->getSuccessors().end();
|
||||
}
|
||||
};
|
||||
|
||||
// Provide specializations of GraphTraits to be able to treat a VPBlockBase as a
|
||||
// graph of VPBlockBase nodes... and to walk it in inverse order. Inverse order
|
||||
// for a VPBlockBase is considered to be when traversing the predecessors of a
|
||||
// VPBlockBase instead of its successors.
|
||||
//
|
||||
|
||||
template <> struct GraphTraits<Inverse<VPBlockBase *>> {
|
||||
typedef VPBlockBase *NodeRef;
|
||||
typedef SmallVectorImpl<VPBlockBase *>::iterator ChildIteratorType;
|
||||
|
||||
static Inverse<VPBlockBase *> getEntryNode(Inverse<VPBlockBase *> B) {
|
||||
return B;
|
||||
}
|
||||
|
||||
static inline ChildIteratorType child_begin(NodeRef N) {
|
||||
return N->getPredecessors().begin();
|
||||
}
|
||||
|
||||
static inline ChildIteratorType child_end(NodeRef N) {
|
||||
return N->getPredecessors().end();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace llvm
|
||||
|
||||
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
||||
@ -26,9 +26,9 @@ target triple = "aarch64--linux-gnu"
|
||||
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
|
||||
; CHECK: [[PRED_UDIV_IF]]:
|
||||
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
|
||||
; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], %x
|
||||
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
|
||||
; CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[TMP6]], [[TMP5]]
|
||||
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
|
||||
; CHECK-NEXT: [[TMP6:%.*]] = add nsw i64 [[TMP5]], %x
|
||||
; CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[TMP4]], [[TMP6]]
|
||||
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> undef, i64 [[TMP7]], i32 0
|
||||
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE]]
|
||||
; CHECK: [[PRED_UDIV_CONTINUE]]:
|
||||
@ -37,9 +37,9 @@ target triple = "aarch64--linux-gnu"
|
||||
; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2]]
|
||||
; CHECK: [[PRED_UDIV_IF1]]:
|
||||
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
|
||||
; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[TMP11]], %x
|
||||
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
|
||||
; CHECK-NEXT: [[TMP14:%.*]] = udiv i64 [[TMP13]], [[TMP12]]
|
||||
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
|
||||
; CHECK-NEXT: [[TMP13:%.*]] = add nsw i64 [[TMP12]], %x
|
||||
; CHECK-NEXT: [[TMP14:%.*]] = udiv i64 [[TMP11]], [[TMP13]]
|
||||
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP14]], i32 1
|
||||
; CHECK-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
|
||||
; CHECK: [[PRED_UDIV_CONTINUE2]]:
|
||||
|
||||
@ -18,8 +18,8 @@ target triple = "aarch64--linux-gnu"
|
||||
; Cost of udiv:
|
||||
; (udiv(2) + extractelement(6) + insertelement(3)) / 2 = 5
|
||||
;
|
||||
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
|
||||
; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
|
||||
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
|
||||
;
|
||||
define i32 @predicated_udiv(i32* %a, i32* %b, i1 %c, i64 %n) {
|
||||
entry:
|
||||
@ -59,8 +59,8 @@ for.end:
|
||||
; Cost of store:
|
||||
; (store(4) + extractelement(3)) / 2 = 3
|
||||
;
|
||||
; CHECK: Scalarizing and predicating: store i32 %tmp2, i32* %tmp0, align 4
|
||||
; CHECK: Found an estimated cost of 3 for VF 2 For instruction: store i32 %tmp2, i32* %tmp0, align 4
|
||||
; CHECK: Scalarizing and predicating: store i32 %tmp2, i32* %tmp0, align 4
|
||||
;
|
||||
define void @predicated_store(i32* %a, i1 %c, i32 %x, i64 %n) {
|
||||
entry:
|
||||
@ -98,10 +98,10 @@ for.end:
|
||||
; Cost of udiv:
|
||||
; (udiv(2) + extractelement(3) + insertelement(3)) / 2 = 4
|
||||
;
|
||||
; CHECK: Scalarizing: %tmp3 = add nsw i32 %tmp2, %x
|
||||
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
|
||||
; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp3 = add nsw i32 %tmp2, %x
|
||||
; CHECK: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
|
||||
; CHECK: Scalarizing: %tmp3 = add nsw i32 %tmp2, %x
|
||||
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
|
||||
;
|
||||
define i32 @predicated_udiv_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
|
||||
entry:
|
||||
@ -143,10 +143,10 @@ for.end:
|
||||
; Cost of store:
|
||||
; store(4) / 2 = 2
|
||||
;
|
||||
; CHECK: Scalarizing: %tmp2 = add nsw i32 %tmp1, %x
|
||||
; CHECK: Scalarizing and predicating: store i32 %tmp2, i32* %tmp0, align 4
|
||||
; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp2 = add nsw i32 %tmp1, %x
|
||||
; CHECK: Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp2, i32* %tmp0, align 4
|
||||
; CHECK: Scalarizing: %tmp2 = add nsw i32 %tmp1, %x
|
||||
; CHECK: Scalarizing and predicating: store i32 %tmp2, i32* %tmp0, align 4
|
||||
;
|
||||
define void @predicated_store_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
|
||||
entry:
|
||||
@ -192,16 +192,16 @@ for.end:
|
||||
; Cost of store:
|
||||
; store(4) / 2 = 2
|
||||
;
|
||||
; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x
|
||||
; CHECK: Scalarizing and predicating: %tmp3 = sdiv i32 %tmp1, %tmp2
|
||||
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp3, %tmp2
|
||||
; CHECK: Scalarizing: %tmp5 = sub i32 %tmp4, %x
|
||||
; CHECK: Scalarizing and predicating: store i32 %tmp5, i32* %tmp0, align 4
|
||||
; CHECK: Found an estimated cost of 1 for VF 2 For instruction: %tmp2 = add i32 %tmp1, %x
|
||||
; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp3 = sdiv i32 %tmp1, %tmp2
|
||||
; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp3, %tmp2
|
||||
; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp5 = sub i32 %tmp4, %x
|
||||
; CHECK: Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp5, i32* %tmp0, align 4
|
||||
; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x
|
||||
; CHECK: Scalarizing and predicating: %tmp3 = sdiv i32 %tmp1, %tmp2
|
||||
; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp3, %tmp2
|
||||
; CHECK: Scalarizing: %tmp5 = sub i32 %tmp4, %x
|
||||
; CHECK: Scalarizing and predicating: store i32 %tmp5, i32* %tmp0, align 4
|
||||
;
|
||||
define void @predication_multi_context(i32* %a, i1 %c, i32 %x, i64 %n) {
|
||||
entry:
|
||||
|
||||
@ -24,10 +24,10 @@ for.body:
|
||||
for.end:
|
||||
ret void
|
||||
|
||||
; CHECK: LV: Scalarizing: %tmp1 = load i32, i32* %tmp0, align 4
|
||||
; CHECK: LV: Scalarizing: store i32 %tmp2, i32* %tmp0, align 4
|
||||
|
||||
; CHECK: LV: Found an estimated cost of 4 for VF 4 For instruction: %tmp1 = load i32, i32* %tmp0, align 4
|
||||
; CHECK: LV: Found an estimated cost of 4 for VF 4 For instruction: store i32 %tmp2, i32* %tmp0, align 4
|
||||
|
||||
; CHECK: LV: Scalarizing: %tmp1 = load i32, i32* %tmp0, align 4
|
||||
; CHECK: LV: Scalarizing: store i32 %tmp2, i32* %tmp0, align 4
|
||||
}
|
||||
|
||||
|
||||
@ -467,6 +467,13 @@ for.body:
|
||||
; SINK-AFTER: %[[VCONV:.+]] = sext <4 x i16> %[[VSHUF]] to <4 x i32>
|
||||
; SINK-AFTER: %[[VCONV3:.+]] = sext <4 x i16> %wide.load to <4 x i32>
|
||||
; SINK-AFTER: mul nsw <4 x i32> %[[VCONV3]], %[[VCONV]]
|
||||
; Check also that the sext sank after the load in the scalar loop.
|
||||
; SINK-AFTER: for.body
|
||||
; SINK-AFTER: %scalar.recur = phi i16 [ %scalar.recur.init, %scalar.ph ], [ %[[LOAD:.+]], %for.body ]
|
||||
; SINK-AFTER: %[[LOAD]] = load i16, i16* %arrayidx2
|
||||
; SINK-AFTER: %[[CONV:.+]] = sext i16 %scalar.recur to i32
|
||||
; SINK-AFTER: %[[CONV3:.+]] = sext i16 %[[LOAD]] to i32
|
||||
; SINK-AFTER: %mul = mul nsw i32 %[[CONV3]], %[[CONV]]
|
||||
;
|
||||
define void @sink_after(i16* %a, i32* %b, i64 %n) {
|
||||
entry:
|
||||
|
||||
@ -209,9 +209,9 @@ entry:
|
||||
; CHECK: br i1 {{.*}}, label %[[IF0:.+]], label %[[CONT0:.+]]
|
||||
; CHECK: [[IF0]]:
|
||||
; CHECK: %[[T00:.+]] = extractelement <2 x i32> %wide.load, i32 0
|
||||
; CHECK: %[[T01:.+]] = add nsw i32 %[[T00]], %x
|
||||
; CHECK: %[[T02:.+]] = extractelement <2 x i32> %wide.load, i32 0
|
||||
; CHECK: %[[T03:.+]] = udiv i32 %[[T02]], %[[T01]]
|
||||
; CHECK: %[[T01:.+]] = extractelement <2 x i32> %wide.load, i32 0
|
||||
; CHECK: %[[T02:.+]] = add nsw i32 %[[T01]], %x
|
||||
; CHECK: %[[T03:.+]] = udiv i32 %[[T00]], %[[T02]]
|
||||
; CHECK: %[[T04:.+]] = insertelement <2 x i32> undef, i32 %[[T03]], i32 0
|
||||
; CHECK: br label %[[CONT0]]
|
||||
; CHECK: [[CONT0]]:
|
||||
@ -219,9 +219,9 @@ entry:
|
||||
; CHECK: br i1 {{.*}}, label %[[IF1:.+]], label %[[CONT1:.+]]
|
||||
; CHECK: [[IF1]]:
|
||||
; CHECK: %[[T06:.+]] = extractelement <2 x i32> %wide.load, i32 1
|
||||
; CHECK: %[[T07:.+]] = add nsw i32 %[[T06]], %x
|
||||
; CHECK: %[[T08:.+]] = extractelement <2 x i32> %wide.load, i32 1
|
||||
; CHECK: %[[T09:.+]] = udiv i32 %[[T08]], %[[T07]]
|
||||
; CHECK: %[[T07:.+]] = extractelement <2 x i32> %wide.load, i32 1
|
||||
; CHECK: %[[T08:.+]] = add nsw i32 %[[T07]], %x
|
||||
; CHECK: %[[T09:.+]] = udiv i32 %[[T06]], %[[T08]]
|
||||
; CHECK: %[[T10:.+]] = insertelement <2 x i32> %[[T05]], i32 %[[T09]], i32 1
|
||||
; CHECK: br label %[[CONT1]]
|
||||
; CHECK: [[CONT1]]:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user