Instead of adding dependence edges between terminator instructions

and every other instruction in their blocks to keep the terminator
instructions at the end, teach the post-RA scheduler how to operate
on ranges of instructions, and exclude terminators from the range
of instructions that get scheduled.

Also, exclude mid-block labels, such as EH_LABEL instructions, and
schedule code before them separately from code after them. This
fixes problems with the post-RA scheduler moving code past
EH_LABELs.

llvm-svn: 62366

include/llvm/CodeGen/ScheduleDAG.h

@@ -418,6 +418,8 @@ namespace llvm {
   public:
     SelectionDAG *DAG;                    // DAG of the current basic block
     MachineBasicBlock *BB;                // Current basic block
+    MachineBasicBlock::iterator Begin;    // The beginning of the range to be scheduled.
+    MachineBasicBlock::iterator End;      // The end of the range to be scheduled.
     const TargetMachine &TM;              // Target processor
     const TargetInstrInfo *TII;           // Target instruction information
     const TargetRegisterInfo *TRI;        // Target processor register info
@@ -440,7 +442,9 @@ namespace llvm {
   
     /// Run - perform scheduling.
     ///
-    void Run(SelectionDAG *DAG, MachineBasicBlock *MBB);
+    void Run(SelectionDAG *DAG, MachineBasicBlock *MBB,
+             MachineBasicBlock::iterator Begin,
+             MachineBasicBlock::iterator End);
 
     /// BuildSchedGraph - Build SUnits and set up their Preds and Succs
     /// to form the scheduling dependency graph.

lib/CodeGen/PostRASchedulerList.cpp

@@ -187,9 +187,17 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
   // Loop over all of the basic blocks
   for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
        MBB != MBBe; ++MBB) {
+    // Schedule each sequence of instructions not interrupted by a label
+    // or anything else that effectively needs to shut down scheduling.
+    MachineBasicBlock::iterator Current = MBB->begin(), End = MBB->end();
+    for (MachineBasicBlock::iterator MI = Current; MI != End; ++MI)
+      if (MI->getDesc().isTerminator() || MI->isLabel()) {
+        Scheduler.Run(0, MBB, Current, MI);
+        Scheduler.EmitSchedule();
+        Current = next(MI);
+      }
 
-    Scheduler.Run(0, MBB);
+    Scheduler.Run(0, MBB, Current, End);
-
     Scheduler.EmitSchedule();
   }
 

lib/CodeGen/ScheduleDAG.cpp

@@ -46,11 +46,18 @@ void ScheduleDAG::dumpSchedule() const {
 
 /// Run - perform scheduling.
 ///
-void ScheduleDAG::Run(SelectionDAG *dag, MachineBasicBlock *bb) {
+void ScheduleDAG::Run(SelectionDAG *dag, MachineBasicBlock *bb,
+                      MachineBasicBlock::iterator begin,
+                      MachineBasicBlock::iterator end) {
+  assert((!dag || begin == end) &&
+         "An instruction range was given for SelectionDAG scheduling!");
+
   SUnits.clear();
   Sequence.clear();
   DAG = dag;
   BB = bb;
+  Begin = begin;
+  End = end;
 
   Schedule();
   

lib/CodeGen/ScheduleDAGEmit.cpp

@@ -33,7 +33,7 @@ void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) {
 }
 
 void ScheduleDAG::EmitNoop() {
-  TII->insertNoop(*BB, BB->end());
+  TII->insertNoop(*BB, End);
 }
 
 void ScheduleDAG::EmitPhysRegCopy(SUnit *SU,
@@ -54,7 +54,7 @@ void ScheduleDAG::EmitPhysRegCopy(SUnit *SU,
           break;
         }
       }
-      TII->copyRegToReg(*BB, BB->end(), Reg, VRI->second,
+      TII->copyRegToReg(*BB, End, Reg, VRI->second,
                         SU->CopyDstRC, SU->CopySrcRC);
     } else {
       // Copy from physical register.
@@ -63,7 +63,7 @@ void ScheduleDAG::EmitPhysRegCopy(SUnit *SU,
       bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)).second;
       isNew = isNew; // Silence compiler warning.
       assert(isNew && "Node emitted out of order - early");
-      TII->copyRegToReg(*BB, BB->end(), VRBase, I->getReg(),
+      TII->copyRegToReg(*BB, End, VRBase, I->getReg(),
                         SU->CopyDstRC, SU->CopySrcRC);
     }
     break;

lib/CodeGen/ScheduleDAGInstrs.cpp

@@ -112,10 +112,16 @@ void ScheduleDAGInstrs::BuildSchedGraph() {
   std::map<const Value *, SUnit *> MemDefs;
   std::map<const Value *, std::vector<SUnit *> > MemUses;
 
-  // Terminators can perform control transfers, we we need to make sure that
+  // If we have an SUnit which is representing a terminator instruction, we
-  // all the work of the block is done before the terminator.
+  // can use it as a place-holder successor for inter-block dependencies.
   SUnit *Terminator = 0;
 
+  // Terminators can perform control transfers, we we need to make sure that
+  // all the work of the block is done before the terminator. Labels can
+  // mark points of interest for various types of meta-data (eg. EH data),
+  // and we need to make sure nothing is scheduled around them.
+  SUnit *SchedulingBarrier = 0;
+
   LoopDependencies LoopRegs(MLI, MDT);
 
   // Track which regs are live into a loop, to help guide back-edge-aware
@@ -137,7 +143,7 @@ void ScheduleDAGInstrs::BuildSchedGraph() {
   unsigned SpecialAddressLatency =
     TM.getSubtarget<TargetSubtarget>().getSpecialAddressLatency();
 
-  for (MachineBasicBlock::iterator MII = BB->end(), MIE = BB->begin();
+  for (MachineBasicBlock::iterator MII = End, MIE = Begin;
        MII != MIE; --MII) {
     MachineInstr *MI = prior(MII);
     const TargetInstrDesc &TID = MI->getDesc();
@@ -368,11 +374,26 @@ void ScheduleDAGInstrs::BuildSchedGraph() {
       }
     }
 
-    // Add chain edges from the terminator to ensure that all the work of the
+    // Add chain edges from terminators and labels to ensure that no
-    // block is completed before any control transfers.
+    // instructions are scheduled past them.
-    if (Terminator && SU->Succs.empty())
+    if (SchedulingBarrier && SU->Succs.empty())
-      Terminator->addPred(SDep(SU, SDep::Order, SU->Latency));
+      SchedulingBarrier->addPred(SDep(SU, SDep::Order, SU->Latency));
+    // If we encounter a mid-block label, we need to go back and add
+    // dependencies on SUnits we've already processed to prevent the
+    // label from moving downward.
+    if (MI->isLabel())
+      for (SUnit *I = SU; I != &SUnits[0]; --I) {
+        SUnit *SuccSU = SU-1;
+        SuccSU->addPred(SDep(SU, SDep::Order, SU->Latency));
+        MachineInstr *SuccMI = SuccSU->getInstr();
+        if (SuccMI->getDesc().isTerminator() || SuccMI->isLabel())
+          break;
+      }
+    // If this instruction obstructs all scheduling, remember it.
     if (TID.isTerminator() || MI->isLabel())
+      SchedulingBarrier = SU;
+    // If this instruction is a terminator, remember it.
+    if (TID.isTerminator())
       Terminator = SU;
   }
 
@@ -413,8 +434,11 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const {
 MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() {
   // For MachineInstr-based scheduling, we're rescheduling the instructions in
   // the block, so start by removing them from the block.
-  while (!BB->empty())
+  while (Begin != End) {
-    BB->remove(BB->begin());
+    MachineBasicBlock::iterator I = Begin;
+    ++Begin;
+    BB->remove(I);
+  }
 
   // Then re-insert them according to the given schedule.
   for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
@@ -425,7 +449,7 @@ MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() {
       continue;
     }
 
-    BB->push_back(SU->getInstr());
+    BB->insert(End, SU->getInstr());
   }
 
   return BB;

lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp

@@ -127,7 +127,7 @@ void ScheduleDAGSDNodes::EmitCopyFromReg(SDNode *Node, unsigned ResNo,
     // Create the reg, emit the copy.
     VRBase = MRI.createVirtualRegister(DstRC);
     bool Emitted =
-      TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC);
+      TII->copyRegToReg(*BB, End, VRBase, SrcReg, DstRC, SrcRC);
     Emitted = Emitted; // Silence compiler warning.
     assert(Emitted && "Unable to issue a copy instruction!");
   }
@@ -410,7 +410,7 @@ void ScheduleDAGSDNodes::EmitSubregNode(SDNode *Node,
     MI->addOperand(MachineOperand::CreateReg(VRBase, true));
     AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap);
     MI->addOperand(MachineOperand::CreateImm(SubIdx));
-    BB->push_back(MI);    
+    BB->insert(End, MI);
   } else if (Opc == TargetInstrInfo::INSERT_SUBREG ||
              Opc == TargetInstrInfo::SUBREG_TO_REG) {
     SDValue N0 = Node->getOperand(0);
@@ -445,7 +445,7 @@ void ScheduleDAGSDNodes::EmitSubregNode(SDNode *Node,
     // Add the subregster being inserted
     AddOperand(MI, N1, 0, 0, VRBaseMap);
     MI->addOperand(MachineOperand::CreateImm(SubIdx));
-    BB->push_back(MI);
+    BB->insert(End, MI);
   } else
     assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg");
      
@@ -505,12 +505,13 @@ void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
     for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i)
       AddMemOperand(MI, cast<MemOperandSDNode>(Node->getOperand(i))->MO);
 
-    if (II.usesCustomDAGSchedInsertionHook())
+    if (II.usesCustomDAGSchedInsertionHook()) {
       // Insert this instruction into the basic block using a target
       // specific inserter which may returns a new basic block.
       BB = TLI->EmitInstrWithCustomInserter(MI, BB);
-    else
+      Begin = End = BB->end();
-      BB->push_back(MI);
+    } else
+      BB->insert(End, MI);
 
     // Additional results must be an physical register def.
     if (HasPhysRegOuts) {
@@ -559,7 +560,7 @@ void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
     else
       DstTRC = TRI->getPhysicalRegisterRegClass(DestReg,
                                             Node->getOperand(1).getValueType());
-    TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC);
+    TII->copyRegToReg(*BB, End, DestReg, SrcReg, DstTRC, SrcTRC);
     break;
   }
   case ISD::CopyFromReg: {
@@ -614,7 +615,7 @@ void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
         break;
       }
     }
-    BB->push_back(MI);
+    BB->insert(End, MI);
     break;
   }
   }

lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

@@ -1074,7 +1074,7 @@ ScheduleDAG *SelectionDAGISel::Schedule() {
   }
   
   ScheduleDAG *Scheduler = Ctor(this, Fast);
-  Scheduler->Run(CurDAG, BB);
+  Scheduler->Run(CurDAG, BB, BB->end(), BB->end());
 
   return Scheduler;
 }