diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp index a558e142e01..4433fb1aaa2 100644 --- a/lib/CodeGen/LiveRangeCalc.cpp +++ b/lib/CodeGen/LiveRangeCalc.cpp @@ -29,14 +29,75 @@ void LiveRangeCalc::reset(const MachineFunction *mf, DomTree = MDT; Alloc = VNIA; - unsigned N = MF->getNumBlockIDs(); - Seen.clear(); - Seen.resize(N); - LiveOut.resize(N); + MainLiveOutData.reset(MF->getNumBlockIDs()); LiveIn.clear(); } +static SlotIndex getDefIndex(const SlotIndexes &Indexes, const MachineInstr &MI, + bool EarlyClobber) { + // PHI defs begin at the basic block start index. + if (MI.isPHI()) + return Indexes.getMBBStartIdx(MI.getParent()); + + // Instructions are either normal 'r', or early clobber 'e'. + return Indexes.getInstructionIndex(&MI).getRegSlot(EarlyClobber); +} + +void LiveRangeCalc::createDeadDefs(LiveInterval &LI) { + assert(MRI && Indexes && "call reset() first"); + + // Visit all def operands. If the same instruction has multiple defs of Reg, + // LR.createDeadDef() will deduplicate. + const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo(); + unsigned Reg = LI.reg; + for (const MachineOperand &MO : MRI->def_operands(Reg)) { + const MachineInstr *MI = MO.getParent(); + SlotIndex Idx = getDefIndex(*Indexes, *MI, MO.isEarlyClobber()); + unsigned SubReg = MO.getSubReg(); + if (SubReg != 0 || LI.hasSubRanges()) { + unsigned Mask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg) + : MRI->getMaxLaneMaskForVReg(Reg); + + // If this is the first time we see a subregister def, initialize + // subranges by creating a copy of the main range. + if (!LI.hasSubRanges() && !LI.empty()) { + unsigned ClassMask = MRI->getMaxLaneMaskForVReg(Reg); + LI.createSubRangeFrom(*Alloc, ClassMask, LI); + } + + for (LiveInterval::subrange_iterator S = LI.subrange_begin(), + SE = LI.subrange_end(); S != SE; ++S) { + // A Mask for subregs common to the existing subrange and current def. + unsigned Common = S->LaneMask & Mask; + if (Common == 0) + continue; + // A Mask for subregs covered by the subrange but not the current def. + unsigned LRest = S->LaneMask & ~Mask; + LiveInterval::SubRange *CommonRange; + if (LRest != 0) { + // Split current subrange into Common and LRest ranges. + S->LaneMask = LRest; + CommonRange = LI.createSubRangeFrom(*Alloc, Common, *S); + } else { + assert(Common == S->LaneMask); + CommonRange = &*S; + } + CommonRange->createDeadDef(Idx, *Alloc); + Mask &= ~Common; + } + if (Mask != 0) { + LiveInterval::SubRange *SubRange = LI.createSubRange(*Alloc, Mask); + SubRange->createDeadDef(Idx, *Alloc); + } + } + + // Create the def in LR. This may find an existing def. + LI.createDeadDef(Idx, *Alloc); + } +} + + void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) { assert(MRI && Indexes && "call reset() first"); @@ -44,22 +105,38 @@ void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) { // LR.createDeadDef() will deduplicate. for (MachineOperand &MO : MRI->def_operands(Reg)) { const MachineInstr *MI = MO.getParent(); - // Find the corresponding slot index. - SlotIndex Idx; - if (MI->isPHI()) - // PHI defs begin at the basic block start index. - Idx = Indexes->getMBBStartIdx(MI->getParent()); - else - // Instructions are either normal 'r', or early clobber 'e'. - Idx = Indexes->getInstructionIndex(MI) - .getRegSlot(MO.isEarlyClobber()); - + SlotIndex Idx = getDefIndex(*Indexes, *MI, MO.isEarlyClobber()); // Create the def in LR. This may find an existing def. LR.createDeadDef(Idx, *Alloc); } } +static SlotIndex getUseIndex(const SlotIndexes &Indexes, + const MachineOperand &MO) { + const MachineInstr *MI = MO.getParent(); + unsigned OpNo = (&MO - &MI->getOperand(0)); + if (MI->isPHI()) { + assert(!MO.isDef() && "Cannot handle PHI def of partial register."); + // The actual place where a phi operand is used is the end of the pred MBB. + // PHI operands are paired: (Reg, PredMBB). + return Indexes.getMBBEndIdx(MI->getOperand(OpNo+1).getMBB()); + } + + // Check for early-clobber redefs. + bool isEarlyClobber = false; + unsigned DefIdx; + if (MO.isDef()) { + isEarlyClobber = MO.isEarlyClobber(); + } else if (MI->isRegTiedToDefOperand(OpNo, &DefIdx)) { + // FIXME: This would be a lot easier if tied early-clobber uses also + // had an early-clobber flag. + isEarlyClobber = MI->getOperand(DefIdx).isEarlyClobber(); + } + return Indexes.getInstructionIndex(MI).getRegSlot(isEarlyClobber); +} + + void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg) { assert(MRI && Indexes && "call reset() first"); @@ -73,38 +150,86 @@ void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg) { continue; // MI is reading Reg. We may have visited MI before if it happens to be // reading Reg multiple times. That is OK, extend() is idempotent. - const MachineInstr *MI = MO.getParent(); - unsigned OpNo = (&MO - &MI->getOperand(0)); - - // Find the SlotIndex being read. - SlotIndex Idx; - if (MI->isPHI()) { - assert(!MO.isDef() && "Cannot handle PHI def of partial register."); - // PHI operands are paired: (Reg, PredMBB). - // Extend the live range to be live-out from PredMBB. - Idx = Indexes->getMBBEndIdx(MI->getOperand(OpNo+1).getMBB()); - } else { - // This is a normal instruction. - Idx = Indexes->getInstructionIndex(MI).getRegSlot(); - // Check for early-clobber redefs. - unsigned DefIdx; - if (MO.isDef()) { - if (MO.isEarlyClobber()) - Idx = Idx.getRegSlot(true); - } else if (MI->isRegTiedToDefOperand(OpNo, &DefIdx)) { - // FIXME: This would be a lot easier if tied early-clobber uses also - // had an early-clobber flag. - if (MI->getOperand(DefIdx).isEarlyClobber()) - Idx = Idx.getRegSlot(true); - } - } - extend(LR, Idx, Reg); + SlotIndex Idx = getUseIndex(*Indexes, MO); + extend(LR, Idx, Reg, MainLiveOutData); } } -// Transfer information from the LiveIn vector to the live ranges. -void LiveRangeCalc::updateLiveIns() { +void LiveRangeCalc::extendToUses(LiveInterval &LI) { + assert(MRI && Indexes && "call reset() first"); + + const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo(); + SmallVector LiveOuts; + unsigned NumSubRanges = 0; + for (LiveInterval::subrange_iterator S = LI.subrange_begin(), + SE = LI.subrange_end(); S != SE; ++S, ++NumSubRanges) { + LiveOuts.push_back(LiveOutData()); + LiveOuts.back().reset(MF->getNumBlockIDs()); + } + + // Visit all operands that read Reg. This may include partial defs. + unsigned Reg = LI.reg; + for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) { + // Clear all kill flags. They will be reinserted after register allocation + // by LiveIntervalAnalysis::addKillFlags(). + if (MO.isUse()) + MO.setIsKill(false); + if (!MO.readsReg()) + continue; + SlotIndex Idx = getUseIndex(*Indexes, MO); + unsigned SubReg = MO.getSubReg(); + if (MO.isUse() && (LI.hasSubRanges() || SubReg != 0)) { + unsigned Mask = SubReg != 0 + ? TRI.getSubRegIndexLaneMask(SubReg) + : Mask = MRI->getMaxLaneMaskForVReg(Reg); + + // If this is the first time we see a subregister def/use. Initialize + // subranges by creating a copy of the main range. + if (!LI.hasSubRanges()) { + unsigned ClassMask = MRI->getMaxLaneMaskForVReg(Reg); + LI.createSubRangeFrom(*Alloc, ClassMask, LI); + LiveOuts.insert(LiveOuts.begin(), LiveOutData()); + LiveOuts.front().reset(MF->getNumBlockIDs()); + ++NumSubRanges; + } + unsigned SubRangeIdx = 0; + for (LiveInterval::subrange_iterator S = LI.subrange_begin(), + SE = LI.subrange_end(); S != SE; ++S, ++SubRangeIdx) { + // A Mask for subregs common to the existing subrange and current def. + unsigned Common = S->LaneMask & Mask; + if (Common == 0) + continue; + // A Mask for subregs covered by the subrange but not the current def. + unsigned LRest = S->LaneMask & ~Mask; + LiveInterval::SubRange *CommonRange; + unsigned CommonRangeIdx; + if (LRest != 0) { + // Split current subrange into Common and LRest ranges. + S->LaneMask = LRest; + CommonRange = LI.createSubRangeFrom(*Alloc, Common, *S); + CommonRangeIdx = 0; + LiveOuts.insert(LiveOuts.begin(), LiveOutData()); + LiveOuts.front().reset(MF->getNumBlockIDs()); + ++NumSubRanges; + ++SubRangeIdx; + } else { + // The subrange and current def lanemasks match completely. + assert(Common == S->LaneMask); + CommonRange = &*S; + CommonRangeIdx = SubRangeIdx; + } + extend(*CommonRange, Idx, Reg, LiveOuts[CommonRangeIdx]); + Mask &= ~Common; + } + assert(SubRangeIdx == NumSubRanges); + } + extend(LI, Idx, Reg, MainLiveOutData); + } +} + + +void LiveRangeCalc::updateFromLiveIns(LiveOutData &LiveOuts) { LiveRangeUpdater Updater; for (SmallVectorImpl::iterator I = LiveIn.begin(), E = LiveIn.end(); I != E; ++I) { @@ -121,8 +246,8 @@ void LiveRangeCalc::updateLiveIns() { else { // The value is live-through, update LiveOut as well. // Defer the Domtree lookup until it is needed. - assert(Seen.test(MBB->getNumber())); - LiveOut[MBB] = LiveOutPair(I->Value, (MachineDomTreeNode *)nullptr); + assert(LiveOuts.Seen.test(MBB->getNumber())); + LiveOuts.Map[MBB] = LiveOutPair(I->Value, nullptr); } Updater.setDest(&I->LR); Updater.add(Start, End, I->Value); @@ -131,7 +256,8 @@ void LiveRangeCalc::updateLiveIns() { } -void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Kill, unsigned PhysReg) { +void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Kill, unsigned PhysReg, + LiveOutData &LiveOuts) { assert(Kill.isValid() && "Invalid SlotIndex"); assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); @@ -147,27 +273,28 @@ void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Kill, unsigned PhysReg) { // multiple values, and we may need to create even more phi-defs to preserve // VNInfo SSA form. Perform a search for all predecessor blocks where we // know the dominating VNInfo. - if (findReachingDefs(LR, *KillMBB, Kill, PhysReg)) + if (findReachingDefs(LR, *KillMBB, Kill, PhysReg, LiveOuts)) return; // When there were multiple different values, we may need new PHIs. - calculateValues(); + calculateValues(LiveOuts); } // This function is called by a client after using the low-level API to add // live-out and live-in blocks. The unique value optimization is not // available, SplitEditor::transferValues handles that case directly anyway. -void LiveRangeCalc::calculateValues() { +void LiveRangeCalc::calculateValues(LiveOutData &LiveOuts) { assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); - updateSSA(); - updateLiveIns(); + updateSSA(LiveOuts); + updateFromLiveIns(LiveOuts); } bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, - SlotIndex Kill, unsigned PhysReg) { + SlotIndex Kill, unsigned PhysReg, + LiveOutData &LiveOuts) { unsigned KillMBBNum = KillMBB.getNumber(); // Block numbers where LR should be live-in. @@ -201,8 +328,8 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, MachineBasicBlock *Pred = *PI; // Is this a known live-out block? - if (Seen.test(Pred->getNumber())) { - if (VNInfo *VNI = LiveOut[Pred].first) { + if (LiveOuts.Seen.test(Pred->getNumber())) { + if (VNInfo *VNI = LiveOuts.Map[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; @@ -216,7 +343,7 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. VNInfo *VNI = LR.extendInBlock(Start, End); - setLiveOutValue(Pred, VNI); + LiveOuts.setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; @@ -251,7 +378,7 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, if (*I == KillMBBNum && Kill.isValid()) End = Kill; else - LiveOut[MF->getBlockNumbered(*I)] = + LiveOuts.Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr); Updater.add(Start, End, TheVNI); } @@ -275,7 +402,7 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, // This is essentially the same iterative algorithm that SSAUpdater uses, // except we already have a dominator tree, so we don't have to recompute it. -void LiveRangeCalc::updateSSA() { +void LiveRangeCalc::updateSSA(LiveOutData &LiveOuts) { assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); @@ -297,22 +424,23 @@ void LiveRangeCalc::updateSSA() { // We need a live-in value to a block with no immediate dominator? // This is probably an unreachable block that has survived somehow. - bool needPHI = !IDom || !Seen.test(IDom->getBlock()->getNumber()); + bool needPHI = !IDom + || !LiveOuts.Seen.test(IDom->getBlock()->getNumber()); // IDom dominates all of our predecessors, but it may not be their // immediate dominator. Check if any of them have live-out values that are // properly dominated by IDom. If so, we need a phi-def here. if (!needPHI) { - IDomValue = LiveOut[IDom->getBlock()]; + IDomValue = LiveOuts.Map[IDom->getBlock()]; // Cache the DomTree node that defined the value. if (IDomValue.first && !IDomValue.second) - LiveOut[IDom->getBlock()].second = IDomValue.second = + LiveOuts.Map[IDom->getBlock()].second = IDomValue.second = DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def)); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { - LiveOutPair &Value = LiveOut[*PI]; + LiveOutPair &Value = LiveOuts.Map[*PI]; if (!Value.first || Value.first == IDomValue.first) continue; @@ -334,7 +462,7 @@ void LiveRangeCalc::updateSSA() { // The value may be live-through even if Kill is set, as can happen when // we are called from extendRange. In that case LiveOutSeen is true, and // LiveOut indicates a foreign or missing value. - LiveOutPair &LOP = LiveOut[MBB]; + LiveOutPair &LOP = LiveOuts.Map[MBB]; // Create a phi-def if required. if (needPHI) { @@ -348,7 +476,7 @@ void LiveRangeCalc::updateSSA() { // This block is done, we know the final value. I->DomNode = nullptr; - // Add liveness since updateLiveIns now skips this node. + // Add liveness since updateFromLiveIns now skips this node. if (I->Kill.isValid()) LR.addSegment(LiveInterval::Segment(Start, I->Kill, VNI)); else { diff --git a/lib/CodeGen/LiveRangeCalc.h b/lib/CodeGen/LiveRangeCalc.h index 345d6c43081..fd494797918 100644 --- a/lib/CodeGen/LiveRangeCalc.h +++ b/lib/CodeGen/LiveRangeCalc.h @@ -40,12 +40,6 @@ class LiveRangeCalc { MachineDominatorTree *DomTree; VNInfo::Allocator *Alloc; - /// Seen - Bit vector of active entries in LiveOut, also used as a visited - /// set by findReachingDefs. One entry per basic block, indexed by block - /// number. This is kept as a separate bit vector because it can be cleared - /// quickly when switching live ranges. - BitVector Seen; - /// LiveOutPair - A value and the block that defined it. The domtree node is /// redundant, it can be computed as: MDT[Indexes.getMBBFromIndex(VNI->def)]. typedef std::pair LiveOutPair; @@ -53,24 +47,44 @@ class LiveRangeCalc { /// LiveOutMap - Map basic blocks to the value leaving the block. typedef IndexedMap LiveOutMap; - /// LiveOut - Map each basic block where a live range is live out to the - /// live-out value and its defining block. - /// - /// For every basic block, MBB, one of these conditions shall be true: - /// - /// 1. !Seen.count(MBB->getNumber()) - /// Blocks without a Seen bit are ignored. - /// 2. LiveOut[MBB].second.getNode() == MBB - /// The live-out value is defined in MBB. - /// 3. forall P in preds(MBB): LiveOut[P] == LiveOut[MBB] - /// The live-out value passses through MBB. All predecessors must carry - /// the same value. - /// - /// The domtree node may be null, it can be computed. - /// - /// The map can be shared by multiple live ranges as long as no two are - /// live-out of the same block. - LiveOutMap LiveOut; + struct LiveOutData { + /// Seen - Bit vector of active entries in LiveOut, also used as a visited + /// set by findReachingDefs. One entry per basic block, indexed by block + /// number. This is kept as a separate bit vector because it can be cleared + /// quickly when switching live ranges. + BitVector Seen; + + /// LiveOut - Map each basic block where a live range is live out to the + /// live-out value and its defining block. + /// + /// For every basic block, MBB, one of these conditions shall be true: + /// + /// 1. !Seen.count(MBB->getNumber()) + /// Blocks without a Seen bit are ignored. + /// 2. LiveOut[MBB].second.getNode() == MBB + /// The live-out value is defined in MBB. + /// 3. forall P in preds(MBB): LiveOut[P] == LiveOut[MBB] + /// The live-out value passses through MBB. All predecessors must carry + /// the same value. + /// + /// The domtree node may be null, it can be computed. + /// + /// The map can be shared by multiple live ranges as long as no two are + /// live-out of the same block. + LiveOutMap Map; + + void reset(unsigned NumBlocks) { + Seen.clear(); + Seen.resize(NumBlocks); + Map.resize(NumBlocks); + } + + void setLiveOutValue(MachineBasicBlock *MBB, VNInfo *VNI) { + Seen.set(MBB->getNumber()); + Map[MBB] = LiveOutPair(VNI, nullptr); + } + }; + LiveOutData MainLiveOutData; /// LiveInBlock - Information about a basic block where a live range is known /// to be live-in, but the value has not yet been determined. @@ -112,17 +126,19 @@ class LiveRangeCalc { /// /// PhysReg, when set, is used to verify live-in lists on basic blocks. bool findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, - SlotIndex Kill, unsigned PhysReg); + SlotIndex Kill, unsigned PhysReg, + LiveOutData &LiveOuts); /// updateSSA - Compute the values that will be live in to all requested /// blocks in LiveIn. Create PHI-def values as required to preserve SSA form. /// /// Every live-in block must be jointly dominated by the added live-out /// blocks. No values are read from the live ranges. - void updateSSA(); + void updateSSA(LiveOutData &LiveOuts); - /// Add liveness as specified in the LiveIn vector. - void updateLiveIns(); + /// Transfer information from the LiveIn vector to the live ranges and update + /// the given @p LiveOuts. + void updateFromLiveIns(LiveOutData &LiveOuts); public: LiveRangeCalc() : MF(nullptr), MRI(nullptr), Indexes(nullptr), @@ -160,17 +176,25 @@ public: /// single existing value, Alloc may be null. /// /// PhysReg, when set, is used to verify live-in lists on basic blocks. - void extend(LiveRange &LR, SlotIndex Kill, unsigned PhysReg = 0); + void extend(LiveRange &LR, SlotIndex Kill, unsigned PhysReg, + LiveOutData &LiveOuts); + + void extend(LiveRange &LR, SlotIndex Kill) { + extend(LR, Kill, 0, MainLiveOutData); + } /// createDeadDefs - Create a dead def in LI for every def operand of Reg. /// Each instruction defining Reg gets a new VNInfo with a corresponding /// minimal live range. void createDeadDefs(LiveRange &LR, unsigned Reg); - /// createDeadDefs - Create a dead def in LI for every def of LI->reg. - void createDeadDefs(LiveInterval &LI) { - createDeadDefs(LI, LI.reg); - } + /// Subregister aware version of createDeadDefs(LiveRange &LR, unsigned Reg). + /// If subregister liveness tracking is enabled new subranges are created as + /// necessary when subregister defs are found. As with + /// createDeadDefs(LiveRange &LR, unsigned Reg) new short live segments are + /// created for every def of LI.reg. The new segments start and end at the + /// defining instruction (hence the name "DeadDef"). + void createDeadDefs(LiveInterval &LI); /// extendToUses - Extend the live range of LI to reach all uses of Reg. /// @@ -178,10 +202,13 @@ public: /// inserted as needed to preserve SSA form. void extendToUses(LiveRange &LR, unsigned Reg); - /// extendToUses - Extend the live range of LI to reach all uses of LI->reg. - void extendToUses(LiveInterval &LI) { - extendToUses(LI, LI.reg); - } + /// Subregister aware version of extendToUses(LiveRange &LR, unsigned Reg). + /// If subregister liveness tracking is enabled new subranges are created + /// as necessary when subregister uses are found. As with + /// extendToUses(LiveRange &LR, unsigned Reg) the segments existing at the + /// defs are extend until they reach all uses. New value numbers are created + /// at CFG joins as necessary (SSA construction). + void extendToUses(LiveInterval &LI); //===--------------------------------------------------------------------===// // Low-level interface. @@ -203,8 +230,7 @@ public: /// VNI may be null only if MBB is a live-through block also passed to /// addLiveInBlock(). void setLiveOutValue(MachineBasicBlock *MBB, VNInfo *VNI) { - Seen.set(MBB->getNumber()); - LiveOut[MBB] = LiveOutPair(VNI, nullptr); + MainLiveOutData.setLiveOutValue(MBB, VNI); } /// addLiveInBlock - Add a block with an unknown live-in value. This @@ -229,7 +255,11 @@ public: /// /// Every predecessor of a live-in block must have been given a value with /// setLiveOutValue, the value may be null for live-trough blocks. - void calculateValues(); + void calculateValues(LiveOutData &LiveOuts); + + void calculateValues() { + calculateValues(MainLiveOutData); + } }; } // end namespace llvm