diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index bb5b23b00c3..cd704c1c53b 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -21,6 +21,7 @@ #include "llvm/ADT/ilist.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/DebugLoc.h" #include "llvm/Support/Recycler.h" @@ -105,6 +106,9 @@ class MachineFunction { // Allocation management for instructions in function. Recycler InstructionRecycler; + // Allocation management for operand arrays on instructions. + ArrayRecycler OperandRecycler; + // Allocation management for basic blocks in function. Recycler BasicBlockRecycler; @@ -394,6 +398,21 @@ public: MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, int64_t Offset, uint64_t Size); + typedef ArrayRecycler::Capacity OperandCapacity; + + /// Allocate an array of MachineOperands. This is only intended for use by + /// internal MachineInstr functions. + MachineOperand *allocateOperandArray(OperandCapacity Cap) { + return OperandRecycler.allocate(Cap, Allocator); + } + + /// Dellocate an array of MachineOperands and recycle the memory. This is + /// only intended for use by internal MachineInstr functions. + /// Cap must be the same capacity that was used to allocate the array. + void deallocateOperandArray(OperandCapacity Cap, MachineOperand *Array) { + OperandRecycler.deallocate(Cap, Array); + } + /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand /// pointers. This array is owned by the MachineFunction. MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num); diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index 4af48f74d7b..3c7b213533f 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" +#include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/DebugLoc.h" #include "llvm/Target/TargetOpcodes.h" #include @@ -63,6 +64,13 @@ public: }; private: const MCInstrDesc *MCID; // Instruction descriptor. + MachineBasicBlock *Parent; // Pointer to the owning basic block. + + // Operands are allocated by an ArrayRecycler. + MachineOperand *Operands; // Pointer to the first operand. + unsigned NumOperands; // Number of operands on instruction. + typedef ArrayRecycler::Capacity OperandCapacity; + OperandCapacity CapOperands; // Capacity of the Operands array. uint8_t Flags; // Various bits of additional // information about machine @@ -78,8 +86,6 @@ private: uint16_t NumMemRefs; // information on memory references mmo_iterator MemRefs; - std::vector Operands; // the operands - MachineBasicBlock *Parent; // Pointer to the owning basic block. DebugLoc debugLoc; // Source line information. MachineInstr(const MachineInstr&) LLVM_DELETED_FUNCTION; @@ -252,7 +258,7 @@ public: /// Access to explicit operands of the instruction. /// - unsigned getNumOperands() const { return (unsigned)Operands.size(); } + unsigned getNumOperands() const { return NumOperands; } const MachineOperand& getOperand(unsigned i) const { assert(i < getNumOperands() && "getOperand() out of range!"); @@ -268,14 +274,14 @@ public: unsigned getNumExplicitOperands() const; /// iterator/begin/end - Iterate over all operands of a machine instruction. - typedef std::vector::iterator mop_iterator; - typedef std::vector::const_iterator const_mop_iterator; + typedef MachineOperand *mop_iterator; + typedef const MachineOperand *const_mop_iterator; - mop_iterator operands_begin() { return Operands.begin(); } - mop_iterator operands_end() { return Operands.end(); } + mop_iterator operands_begin() { return Operands; } + mop_iterator operands_end() { return Operands + NumOperands; } - const_mop_iterator operands_begin() const { return Operands.begin(); } - const_mop_iterator operands_end() const { return Operands.end(); } + const_mop_iterator operands_begin() const { return Operands; } + const_mop_iterator operands_end() const { return Operands + NumOperands; } /// Access to memory operands of the instruction mmo_iterator memoperands_begin() const { return MemRefs; } diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h index 606833cd408..66a56ed464e 100644 --- a/include/llvm/CodeGen/MachineOperand.h +++ b/include/llvm/CodeGen/MachineOperand.h @@ -35,6 +35,11 @@ class MCSymbol; /// MachineOperand class - Representation of each machine instruction operand. /// +/// This class isn't a POD type because it has a private constructor, but its +/// destructor must be trivial. Functions like MachineInstr::addOperand(), +/// MachineRegisterInfo::moveOperands(), and MF::DeleteMachineInstr() depend on +/// not having to call the MachineOperand destructor. +/// class MachineOperand { public: enum MachineOperandType { diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index ab8f1f43fb1..08d116afead 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -78,6 +78,7 @@ MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM, MachineFunction::~MachineFunction() { BasicBlocks.clear(); InstructionRecycler.clear(Allocator); + OperandRecycler.clear(Allocator); BasicBlockRecycler.clear(Allocator); if (RegInfo) { RegInfo->~MachineRegisterInfo(); @@ -177,6 +178,12 @@ MachineFunction::CloneMachineInstr(const MachineInstr *Orig) { /// void MachineFunction::DeleteMachineInstr(MachineInstr *MI) { + // Strip it for parts. The operand array and the MI object itself are + // independently recyclable. + if (MI->Operands) + deallocateOperandArray(MI->CapOperands, MI->Operands); + MI->Operands = 0; + MI->NumOperands = 0; MI->~MachineInstr(); InstructionRecycler.Deallocate(Allocator, MI); } diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 5239d43d7e8..d3342987c05 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -532,12 +532,16 @@ void MachineInstr::addImplicitDefUseOperands(MachineFunction &MF) { /// the MCInstrDesc. MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &tid, const DebugLoc dl, bool NoImp) - : MCID(&tid), Flags(0), AsmPrinterFlags(0), - NumMemRefs(0), MemRefs(0), Parent(0), debugLoc(dl) { - unsigned NumImplicitOps = 0; - if (!NoImp) - NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + MCID->getNumOperands()); + : MCID(&tid), Parent(0), Operands(0), NumOperands(0), + Flags(0), AsmPrinterFlags(0), + NumMemRefs(0), MemRefs(0), debugLoc(dl) { + // Reserve space for the expected number of operands. + if (unsigned NumOps = MCID->getNumOperands() + + MCID->getNumImplicitDefs() + MCID->getNumImplicitUses()) { + CapOperands = OperandCapacity::get(NumOps); + Operands = MF.allocateOperandArray(CapOperands); + } + if (!NoImp) addImplicitDefUseOperands(MF); // Make sure that we get added to a machine basicblock @@ -547,10 +551,12 @@ MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &tid, /// MachineInstr ctor - Copies MachineInstr arg exactly /// MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) - : MCID(&MI.getDesc()), Flags(0), AsmPrinterFlags(0), + : MCID(&MI.getDesc()), Parent(0), Operands(0), NumOperands(0), + Flags(0), AsmPrinterFlags(0), NumMemRefs(MI.NumMemRefs), MemRefs(MI.MemRefs), - Parent(0), debugLoc(MI.getDebugLoc()) { - Operands.reserve(MI.getNumOperands()); + debugLoc(MI.getDebugLoc()) { + CapOperands = OperandCapacity::get(MI.getNumOperands()); + Operands = MF.allocateOperandArray(CapOperands); // Add operands for (unsigned i = 0; i != MI.getNumOperands(); ++i) @@ -611,35 +617,52 @@ void MachineInstr::addOperand(const MachineOperand &Op) { addOperand(*MF, Op); } +/// Move NumOps MachineOperands from Src to Dst, with support for overlapping +/// ranges. If MRI is non-null also update use-def chains. +static void moveOperands(MachineOperand *Dst, MachineOperand *Src, + unsigned NumOps, MachineRegisterInfo *MRI) { + if (MRI) + return MRI->moveOperands(Dst, Src, NumOps); + + // Here it would be convenient to call memmove, so that isn't allowed because + // MachineOperand has a constructor and so isn't a POD type. + if (Dst < Src) + for (unsigned i = 0; i != NumOps; ++i) + new (Dst + i) MachineOperand(Src[i]); + else + for (unsigned i = NumOps; i ; --i) + new (Dst + i - 1) MachineOperand(Src[i - 1]); +} + /// addOperand - Add the specified operand to the instruction. If it is an /// implicit operand, it is added to the end of the operand list. If it is /// an explicit operand it is added at the end of the explicit operand list /// (before the first implicit operand). void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) { assert(MCID && "Cannot add operands before providing an instr descriptor"); - bool isImpReg = Op.isReg() && Op.isImplicit(); - MachineRegisterInfo *RegInfo = getRegInfo(); - // If the Operands backing store is reallocated, all register operands must - // be removed and re-added to RegInfo. It is storing pointers to operands. - bool Reallocate = RegInfo && - !Operands.empty() && getNumOperands() == Operands.capacity(); + // Check if we're adding one of our existing operands. + if (&Op >= Operands && &Op < Operands + NumOperands) { + // This is unusual: MI->addOperand(MI->getOperand(i)). + // If adding Op requires reallocating or moving existing operands around, + // the Op reference could go stale. Support it by copying Op. + MachineOperand CopyOp(Op); + return addOperand(MF, CopyOp); + } // Find the insert location for the new operand. Implicit registers go at - // the end, everything goes before the implicit regs. - unsigned OpNo = getNumOperands(); - - // Remove all the implicit operands from RegInfo if they need to be shifted. + // the end, everything else goes before the implicit regs. + // // FIXME: Allow mixed explicit and implicit operands on inline asm. // InstrEmitter::EmitSpecialNode() is marking inline asm clobbers as // implicit-defs, but they must not be moved around. See the FIXME in // InstrEmitter.cpp. + unsigned OpNo = getNumOperands(); + bool isImpReg = Op.isReg() && Op.isImplicit(); if (!isImpReg && !isInlineAsm()) { while (OpNo && Operands[OpNo-1].isReg() && Operands[OpNo-1].isImplicit()) { --OpNo; assert(!Operands[OpNo].isTied() && "Cannot move tied operands"); - if (RegInfo) - RegInfo->removeRegOperandFromUseList(&Operands[OpNo]); } } @@ -650,55 +673,56 @@ void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) { OpNo < MCID->getNumOperands()) && "Trying to add an operand to a machine instr that is already done!"); - // All operands from OpNo have been removed from RegInfo. If the Operands - // backing store needs to be reallocated, we also need to remove any other - // register operands. - if (Reallocate) - for (unsigned i = 0; i != OpNo; ++i) - if (Operands[i].isReg()) - RegInfo->removeRegOperandFromUseList(&Operands[i]); + MachineRegisterInfo *MRI = getRegInfo(); - // Insert the new operand at OpNo. - Operands.insert(Operands.begin() + OpNo, Op); - Operands[OpNo].ParentMI = this; + // Determine if the Operands array needs to be reallocated. + // Save the old capacity and operand array. + OperandCapacity OldCap = CapOperands; + MachineOperand *OldOperands = Operands; + if (!OldOperands || OldCap.getSize() == getNumOperands()) { + CapOperands = OldOperands ? OldCap.getNext() : OldCap.get(1); + Operands = MF.allocateOperandArray(CapOperands); + // Move the operands before the insertion point. + if (OpNo) + moveOperands(Operands, OldOperands, OpNo, MRI); + } - // The Operands backing store has now been reallocated, so we can re-add the - // operands before OpNo. - if (Reallocate) - for (unsigned i = 0; i != OpNo; ++i) - if (Operands[i].isReg()) - RegInfo->addRegOperandToUseList(&Operands[i]); + // Move the operands following the insertion point. + if (OpNo != NumOperands) + moveOperands(Operands + OpNo + 1, OldOperands + OpNo, NumOperands - OpNo, + MRI); + ++NumOperands; - // When adding a register operand, tell RegInfo about it. - if (Operands[OpNo].isReg()) { + // Deallocate the old operand array. + if (OldOperands != Operands && OldOperands) + MF.deallocateOperandArray(OldCap, OldOperands); + + // Copy Op into place. It still needs to be inserted into the MRI use lists. + MachineOperand *NewMO = new (Operands + OpNo) MachineOperand(Op); + NewMO->ParentMI = this; + + // When adding a register operand, tell MRI about it. + if (NewMO->isReg()) { // Ensure isOnRegUseList() returns false, regardless of Op's status. - Operands[OpNo].Contents.Reg.Prev = 0; + NewMO->Contents.Reg.Prev = 0; // Ignore existing ties. This is not a property that can be copied. - Operands[OpNo].TiedTo = 0; - // Add the new operand to RegInfo. - if (RegInfo) - RegInfo->addRegOperandToUseList(&Operands[OpNo]); + NewMO->TiedTo = 0; + // Add the new operand to MRI, but only for instructions in an MBB. + if (MRI) + MRI->addRegOperandToUseList(NewMO); // The MCID operand information isn't accurate until we start adding // explicit operands. The implicit operands are added first, then the // explicits are inserted before them. if (!isImpReg) { // Tie uses to defs as indicated in MCInstrDesc. - if (Operands[OpNo].isUse()) { + if (NewMO->isUse()) { int DefIdx = MCID->getOperandConstraint(OpNo, MCOI::TIED_TO); if (DefIdx != -1) tieOperands(DefIdx, OpNo); } // If the register operand is flagged as early, mark the operand as such. if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) - Operands[OpNo].setIsEarlyClobber(true); - } - } - - // Re-add all the implicit ops. - if (RegInfo) { - for (unsigned i = OpNo + 1, e = getNumOperands(); i != e; ++i) { - assert(Operands[i].isReg() && "Should only be an implicit reg!"); - RegInfo->addRegOperandToUseList(&Operands[i]); + NewMO->setIsEarlyClobber(true); } } } @@ -709,16 +733,6 @@ void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) { void MachineInstr::RemoveOperand(unsigned OpNo) { assert(OpNo < getNumOperands() && "Invalid operand number"); untieRegOperand(OpNo); - MachineRegisterInfo *RegInfo = getRegInfo(); - - // If we have reginfo to update, remove all operands that will be shifted - // down from their reg lists, move everything down, then re-add them. - if (RegInfo) { - for (unsigned i = OpNo, e = getNumOperands(); i != e; ++i) { - if (Operands[i].isReg()) - RegInfo->removeRegOperandFromUseList(&Operands[i]); - } - } #ifndef NDEBUG // Moving tied operands would break the ties. @@ -727,14 +741,17 @@ void MachineInstr::RemoveOperand(unsigned OpNo) { assert(!Operands[i].isTied() && "Cannot move tied operands"); #endif - Operands.erase(Operands.begin()+OpNo); + MachineRegisterInfo *MRI = getRegInfo(); + if (MRI && Operands[OpNo].isReg()) + MRI->removeRegOperandFromUseList(Operands + OpNo); - if (RegInfo) { - for (unsigned i = OpNo, e = getNumOperands(); i != e; ++i) { - if (Operands[i].isReg()) - RegInfo->addRegOperandToUseList(&Operands[i]); - } - } + // Don't call the MachineOperand destructor. A lot of this code depends on + // MachineOperand having a trivial destructor anyway, and adding a call here + // wouldn't make it 'destructor-correct'. + + if (unsigned N = NumOperands - 1 - OpNo) + moveOperands(Operands + OpNo, Operands + OpNo + 1, N, MRI); + --NumOperands; } /// addMemOperand - Add a MachineMemOperand to the machine instruction.