Create HandlePHINodesInSuccessorBlocksFast, a version of

HandlePHINodesInSuccessorBlocks that works FastISel-style. This
allows PHI nodes to be updated correctly while using FastISel.

This also involves some code reorganization; ValueMap and
MBBMap are now members of the FastISel class, so they needn't
be passed around explicitly anymore. Also, SelectInstructions
is changed to SelectInstruction, and only does one instruction
at a time.

llvm-svn: 55746

include/llvm/CodeGen/FastISel.h

@@ -36,6 +36,8 @@ class TargetRegisterClass;
 class FastISel {
 protected:
   MachineBasicBlock *MBB;
+  DenseMap<const Value *, unsigned> &ValueMap;
+  DenseMap<const BasicBlock *, MachineBasicBlock *> &MBBMap;
   MachineFunction &MF;
   MachineRegisterInfo &MRI;
   const TargetMachine &TM;
@@ -44,17 +46,18 @@ protected:
   const TargetLowering &TLI;
 
 public:
-  /// SelectInstructions - Do "fast" instruction selection over the
+  /// setCurrentBlock - Set the current block, to which generated
-  /// LLVM IR instructions in the range [Begin, N) where N is either
+  /// machine instructions will be appended.
-  /// End or the first unsupported instruction. Return N.
+  ///
-  /// ValueMap is filled in with a mapping of LLVM IR Values to
+  void setCurrentBlock(MachineBasicBlock *mbb) {
-  /// virtual register numbers. MBB is a block to which to append
+    MBB = mbb;
-  /// the generated MachineInstrs.
+  }
-  BasicBlock::iterator
+
-  SelectInstructions(BasicBlock::iterator Begin, BasicBlock::iterator End,
+  /// SelectInstruction - Do "fast" instruction selection for the given
-                     DenseMap<const Value *, unsigned> &ValueMap,
+  /// LLVM IR instruction, and append generated machine instructions to
-                     DenseMap<const BasicBlock *, MachineBasicBlock *> &MBBMap,
+  /// the current block. Return true if selection was successful.
-                     MachineBasicBlock *MBB);
+  ///
+  bool SelectInstruction(Instruction *I);
 
   /// TargetSelectInstruction - This method is called by target-independent
   /// code when the normal FastISel process fails to select an instruction.
@@ -62,15 +65,18 @@ public:
   /// fit into FastISel's framework. It returns true if it was successful.
   ///
   virtual bool
-  TargetSelectInstruction(Instruction *I,
+  TargetSelectInstruction(Instruction *I) = 0;
-                          DenseMap<const Value *, unsigned> &ValueMap,
+
-                      DenseMap<const BasicBlock *, MachineBasicBlock *> &MBBMap,
+  /// getRegForValue - Create a virtual register and arrange for it to
-                          MachineBasicBlock *MBB) = 0;
+  /// be assigned the value for the given LLVM value.
+  unsigned getRegForValue(Value *V);
 
   virtual ~FastISel();
 
 protected:
-  explicit FastISel(MachineFunction &mf);
+  FastISel(MachineFunction &mf,
+           DenseMap<const Value *, unsigned> &vm,
+           DenseMap<const BasicBlock *, MachineBasicBlock *> &bm);
 
   /// FastEmit_r - This method is called by target-independent code
   /// to request that an instruction with the given type and opcode
@@ -208,26 +214,18 @@ protected:
   /// from a specified index of a superregister.
   unsigned FastEmitInst_extractsubreg(unsigned Op0, uint32_t Idx);
 
-  unsigned getRegForValue(Value *V,
+  void UpdateValueMap(Instruction* I, unsigned Reg);
-                          DenseMap<const Value*, unsigned> &ValueMap);
-
-  void UpdateValueMap(Instruction* I, unsigned Reg, 
-                      DenseMap<const Value*, unsigned> &ValueMap);
 
   unsigned createResultReg(const TargetRegisterClass *RC);
 
 private:
-  bool SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode,
+  bool SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode);
-                      DenseMap<const Value*, unsigned> &ValueMap);
 
-  bool SelectGetElementPtr(Instruction *I,
+  bool SelectGetElementPtr(Instruction *I);
-                           DenseMap<const Value*, unsigned> &ValueMap);
 
-  bool SelectBitCast(Instruction *I,
+  bool SelectBitCast(Instruction *I);
-                     DenseMap<const Value*, unsigned> &ValueMap);
   
-  bool SelectCast(Instruction *I, ISD::NodeType Opcode,
+  bool SelectCast(Instruction *I, ISD::NodeType Opcode);
-                  DenseMap<const Value*, unsigned> &ValueMap);                  
 };
 
 }

include/llvm/CodeGen/SelectionDAGISel.h

@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/SelectionDAG.h"
 
 namespace llvm {
+  class FastISel;
   class SelectionDAGLowering;
   class SDValue;
   class MachineRegisterInfo;
@@ -118,6 +119,8 @@ private:
 
   void HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB);
 
+  bool HandlePHINodesInSuccessorBlocksFast(BasicBlock *LLVMBB, FastISel *F);
+
   /// Pick a safe ordering for instructions for each target node in the
   /// graph.
   ScheduleDAG *Schedule();

include/llvm/Target/TargetLowering.h

@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include <map>
 #include <vector>
@@ -1116,7 +1117,12 @@ public:
 
   /// createFastISel - This method returns a target specific FastISel object,
   /// or null if the target does not support "fast" ISel.
-  virtual FastISel *createFastISel(MachineFunction &) { return 0; }
+  virtual FastISel *
+  createFastISel(MachineFunction &,
+                 DenseMap<const Value *, unsigned> &,
+                 DenseMap<const BasicBlock *, MachineBasicBlock *> &) {
+    return 0;
+  }
 
   //===--------------------------------------------------------------------===//
   // Inline Asm Support hooks

lib/CodeGen/SelectionDAG/FastISel.cpp

@@ -25,8 +25,7 @@ using namespace llvm;
 // tracking what uses they dominate.  Non-constants, however, already
 // have the SSA def-doms-use requirement enforced, so we can cache their
 // computations.
-unsigned FastISel::getRegForValue(Value *V,
+unsigned FastISel::getRegForValue(Value *V) {
-                                  DenseMap<const Value*, unsigned> &ValueMap) {
   if (ValueMap.count(V))
     return ValueMap[V];
 
@@ -78,8 +77,7 @@ unsigned FastISel::getRegForValue(Value *V,
 /// NOTE: This is only necessary because we might select a block that uses
 /// a value before we select the block that defines the value.  It might be
 /// possible to fix this by selecting blocks in reverse postorder.
-void FastISel::UpdateValueMap(Instruction* I, unsigned Reg, 
+void FastISel::UpdateValueMap(Instruction* I, unsigned Reg) {
-                              DenseMap<const Value*, unsigned> &ValueMap) {
   if (!ValueMap.count(I))
     ValueMap[I] = Reg;
   else
@@ -90,8 +88,7 @@ void FastISel::UpdateValueMap(Instruction* I, unsigned Reg,
 /// SelectBinaryOp - Select and emit code for a binary operator instruction,
 /// which has an opcode which directly corresponds to the given ISD opcode.
 ///
-bool FastISel::SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode,
+bool FastISel::SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode) {
-                              DenseMap<const Value*, unsigned> &ValueMap) {
   MVT VT = MVT::getMVT(I->getType(), /*HandleUnknown=*/true);
   if (VT == MVT::Other || !VT.isSimple())
     // Unhandled type. Halt "fast" selection and bail.
@@ -103,7 +100,7 @@ bool FastISel::SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode,
   if (!TLI.isTypeLegal(VT))
     return false;
 
-  unsigned Op0 = getRegForValue(I->getOperand(0), ValueMap);
+  unsigned Op0 = getRegForValue(I->getOperand(0));
   if (Op0 == 0)
     // Unhandled operand. Halt "fast" selection and bail.
     return false;
@@ -114,7 +111,7 @@ bool FastISel::SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode,
                                      ISDOpcode, Op0, CI->getZExtValue());
     if (ResultReg != 0) {
       // We successfully emitted code for the given LLVM Instruction.
-      UpdateValueMap(I, ResultReg, ValueMap);
+      UpdateValueMap(I, ResultReg);
       return true;
     }
   }
@@ -125,12 +122,12 @@ bool FastISel::SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode,
                                      ISDOpcode, Op0, CF);
     if (ResultReg != 0) {
       // We successfully emitted code for the given LLVM Instruction.
-      UpdateValueMap(I, ResultReg, ValueMap);
+      UpdateValueMap(I, ResultReg);
       return true;
     }
   }
 
-  unsigned Op1 = getRegForValue(I->getOperand(1), ValueMap);
+  unsigned Op1 = getRegForValue(I->getOperand(1));
   if (Op1 == 0)
     // Unhandled operand. Halt "fast" selection and bail.
     return false;
@@ -144,13 +141,12 @@ bool FastISel::SelectBinaryOp(Instruction *I, ISD::NodeType ISDOpcode,
     return false;
 
   // We successfully emitted code for the given LLVM Instruction.
-  UpdateValueMap(I, ResultReg, ValueMap);
+  UpdateValueMap(I, ResultReg);
   return true;
 }
 
-bool FastISel::SelectGetElementPtr(Instruction *I,
+bool FastISel::SelectGetElementPtr(Instruction *I) {
-                                   DenseMap<const Value*, unsigned> &ValueMap) {
+  unsigned N = getRegForValue(I->getOperand(0));
-  unsigned N = getRegForValue(I->getOperand(0), ValueMap);
   if (N == 0)
     // Unhandled operand. Halt "fast" selection and bail.
     return false;
@@ -190,7 +186,7 @@ bool FastISel::SelectGetElementPtr(Instruction *I,
       
       // N = N + Idx * ElementSize;
       uint64_t ElementSize = TD.getABITypeSize(Ty);
-      unsigned IdxN = getRegForValue(Idx, ValueMap);
+      unsigned IdxN = getRegForValue(Idx);
       if (IdxN == 0)
         // Unhandled operand. Halt "fast" selection and bail.
         return false;
@@ -220,12 +216,11 @@ bool FastISel::SelectGetElementPtr(Instruction *I,
   }
 
   // We successfully emitted code for the given LLVM Instruction.
-  UpdateValueMap(I, N, ValueMap);
+  UpdateValueMap(I, N);
   return true;
 }
 
-bool FastISel::SelectCast(Instruction *I, ISD::NodeType Opcode,
+bool FastISel::SelectCast(Instruction *I, ISD::NodeType Opcode) {
-                          DenseMap<const Value*, unsigned> &ValueMap) {
   MVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
   MVT DstVT = TLI.getValueType(I->getType());
     
@@ -235,7 +230,7 @@ bool FastISel::SelectCast(Instruction *I, ISD::NodeType Opcode,
     // Unhandled type. Halt "fast" selection and bail.
     return false;
     
-  unsigned InputReg = getRegForValue(I->getOperand(0), ValueMap);
+  unsigned InputReg = getRegForValue(I->getOperand(0));
   if (!InputReg)
     // Unhandled operand.  Halt "fast" selection and bail.
     return false;
@@ -247,18 +242,17 @@ bool FastISel::SelectCast(Instruction *I, ISD::NodeType Opcode,
   if (!ResultReg)
     return false;
     
-  UpdateValueMap(I, ResultReg, ValueMap);
+  UpdateValueMap(I, ResultReg);
   return true;
 }
 
-bool FastISel::SelectBitCast(Instruction *I,
+bool FastISel::SelectBitCast(Instruction *I) {
-                             DenseMap<const Value*, unsigned> &ValueMap) {
   // If the bitcast doesn't change the type, just use the operand value.
   if (I->getType() == I->getOperand(0)->getType()) {
-    unsigned Reg = getRegForValue(I->getOperand(0), ValueMap);
+    unsigned Reg = getRegForValue(I->getOperand(0));
     if (Reg == 0)
       return false;
-    UpdateValueMap(I, Reg, ValueMap);
+    UpdateValueMap(I, Reg);
     return true;
   }
 
@@ -272,7 +266,7 @@ bool FastISel::SelectBitCast(Instruction *I,
     // Unhandled type. Halt "fast" selection and bail.
     return false;
   
-  unsigned Op0 = getRegForValue(I->getOperand(0), ValueMap);
+  unsigned Op0 = getRegForValue(I->getOperand(0));
   if (Op0 == 0)
     // Unhandled operand. Halt "fast" selection and bail.
     return false;
@@ -298,143 +292,124 @@ bool FastISel::SelectBitCast(Instruction *I,
   if (!ResultReg)
     return false;
   
-  UpdateValueMap(I, ResultReg, ValueMap);
+  UpdateValueMap(I, ResultReg);
   return true;
 }
 
-BasicBlock::iterator
+bool
-FastISel::SelectInstructions(BasicBlock::iterator Begin,
+FastISel::SelectInstruction(Instruction *I) {
-                             BasicBlock::iterator End,
+  switch (I->getOpcode()) {
-                             DenseMap<const Value*, unsigned> &ValueMap,
+  case Instruction::Add: {
-                             DenseMap<const BasicBlock*,
+    ISD::NodeType Opc = I->getType()->isFPOrFPVector() ? ISD::FADD : ISD::ADD;
-                                      MachineBasicBlock *> &MBBMap,
+    return SelectBinaryOp(I, Opc);
-                             MachineBasicBlock *mbb) {
+  }
-  MBB = mbb;
+  case Instruction::Sub: {
-  BasicBlock::iterator I = Begin;
+    ISD::NodeType Opc = I->getType()->isFPOrFPVector() ? ISD::FSUB : ISD::SUB;
+    return SelectBinaryOp(I, Opc);
+  }
+  case Instruction::Mul: {
+    ISD::NodeType Opc = I->getType()->isFPOrFPVector() ? ISD::FMUL : ISD::MUL;
+    return SelectBinaryOp(I, Opc);
+  }
+  case Instruction::SDiv:
+    return SelectBinaryOp(I, ISD::SDIV);
+  case Instruction::UDiv:
+    return SelectBinaryOp(I, ISD::UDIV);
+  case Instruction::FDiv:
+    return SelectBinaryOp(I, ISD::FDIV);
+  case Instruction::SRem:
+    return SelectBinaryOp(I, ISD::SREM);
+  case Instruction::URem:
+    return SelectBinaryOp(I, ISD::UREM);
+  case Instruction::FRem:
+    return SelectBinaryOp(I, ISD::FREM);
+  case Instruction::Shl:
+    return SelectBinaryOp(I, ISD::SHL);
+  case Instruction::LShr:
+    return SelectBinaryOp(I, ISD::SRL);
+  case Instruction::AShr:
+    return SelectBinaryOp(I, ISD::SRA);
+  case Instruction::And:
+    return SelectBinaryOp(I, ISD::AND);
+  case Instruction::Or:
+    return SelectBinaryOp(I, ISD::OR);
+  case Instruction::Xor:
+    return SelectBinaryOp(I, ISD::XOR);
 
-  for (; I != End; ++I) {
+  case Instruction::GetElementPtr:
-    switch (I->getOpcode()) {
+    return SelectGetElementPtr(I);
-    case Instruction::Add: {
-      ISD::NodeType Opc = I->getType()->isFPOrFPVector() ? ISD::FADD : ISD::ADD;
-      if (!SelectBinaryOp(I, Opc, ValueMap))  return I; break;
-    }
-    case Instruction::Sub: {
-      ISD::NodeType Opc = I->getType()->isFPOrFPVector() ? ISD::FSUB : ISD::SUB;
-      if (!SelectBinaryOp(I, Opc, ValueMap))  return I; break;
-    }
-    case Instruction::Mul: {
-      ISD::NodeType Opc = I->getType()->isFPOrFPVector() ? ISD::FMUL : ISD::MUL;
-      if (!SelectBinaryOp(I, Opc, ValueMap))  return I; break;
-    }
-    case Instruction::SDiv:
-      if (!SelectBinaryOp(I, ISD::SDIV, ValueMap)) return I; break;
-    case Instruction::UDiv:
-      if (!SelectBinaryOp(I, ISD::UDIV, ValueMap)) return I; break;
-    case Instruction::FDiv:
-      if (!SelectBinaryOp(I, ISD::FDIV, ValueMap)) return I; break;
-    case Instruction::SRem:
-      if (!SelectBinaryOp(I, ISD::SREM, ValueMap)) return I; break;
-    case Instruction::URem:
-      if (!SelectBinaryOp(I, ISD::UREM, ValueMap)) return I; break;
-    case Instruction::FRem:
-      if (!SelectBinaryOp(I, ISD::FREM, ValueMap)) return I; break;
-    case Instruction::Shl:
-      if (!SelectBinaryOp(I, ISD::SHL, ValueMap)) return I; break;
-    case Instruction::LShr:
-      if (!SelectBinaryOp(I, ISD::SRL, ValueMap)) return I; break;
-    case Instruction::AShr:
-      if (!SelectBinaryOp(I, ISD::SRA, ValueMap)) return I; break;
-    case Instruction::And:
-      if (!SelectBinaryOp(I, ISD::AND, ValueMap)) return I; break;
-    case Instruction::Or:
-      if (!SelectBinaryOp(I, ISD::OR, ValueMap)) return I; break;
-    case Instruction::Xor:
-      if (!SelectBinaryOp(I, ISD::XOR, ValueMap)) return I; break;
 
-    case Instruction::GetElementPtr:
+  case Instruction::Br: {
-      if (!SelectGetElementPtr(I, ValueMap)) return I;
+    BranchInst *BI = cast<BranchInst>(I);
-      break;
 
-    case Instruction::Br: {
+    if (BI->isUnconditional()) {
-      BranchInst *BI = cast<BranchInst>(I);
+      MachineFunction::iterator NextMBB =
+         next(MachineFunction::iterator(MBB));
+      BasicBlock *LLVMSucc = BI->getSuccessor(0);
+      MachineBasicBlock *MSucc = MBBMap[LLVMSucc];
 
-      if (BI->isUnconditional()) {
+      if (NextMBB != MF.end() && MSucc == NextMBB) {
-        MachineFunction::iterator NextMBB =
+        // The unconditional fall-through case, which needs no instructions.
-           next(MachineFunction::iterator(MBB));
-        BasicBlock *LLVMSucc = BI->getSuccessor(0);
-        MachineBasicBlock *MSucc = MBBMap[LLVMSucc];
-
-        if (NextMBB != MF.end() && MSucc == NextMBB) {
-          // The unconditional fall-through case, which needs no instructions.
-        } else {
-          // The unconditional branch case.
-          TII.InsertBranch(*MBB, MSucc, NULL, SmallVector<MachineOperand, 0>());
-        }
-        MBB->addSuccessor(MSucc);
-        break;
-      }
-
-      // Conditional branches are not handed yet.
-      // Halt "fast" selection and bail.
-      return I;
-    }
-
-    case Instruction::PHI:
-      // PHI nodes are already emitted.
-      break;
-      
-    case Instruction::BitCast:
-      if (!SelectBitCast(I, ValueMap)) return I; break;
-
-    case Instruction::FPToSI:
-      if (!SelectCast(I, ISD::FP_TO_SINT, ValueMap)) return I; 
-      break;
-    case Instruction::ZExt:
-      if (!SelectCast(I, ISD::ZERO_EXTEND, ValueMap)) return I;
-      break;
-    case Instruction::SExt:
-      if (!SelectCast(I, ISD::SIGN_EXTEND, ValueMap)) return I;
-      break;
-    case Instruction::Trunc:
-      if (!SelectCast(I, ISD::TRUNCATE, ValueMap)) return I;
-      break;
-    case Instruction::SIToFP:
-      if (!SelectCast(I, ISD::SINT_TO_FP, ValueMap)) return I;
-      break;
-
-    case Instruction::IntToPtr: // Deliberate fall-through.
-    case Instruction::PtrToInt: {
-      MVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
-      MVT DstVT = TLI.getValueType(I->getType());
-      if (SrcVT.getSimpleVT() == DstVT.getSimpleVT()) {
-        if (ValueMap[I->getOperand(0)]) {
-          UpdateValueMap(I, ValueMap[I->getOperand(0)], ValueMap);
-          break;
-        } else
-          // Unhandled operand
-          return I;
-      } else if (DstVT.bitsGT(SrcVT)) {
-        if (!SelectCast(I, ISD::ZERO_EXTEND, ValueMap)) return I;
-        break;
       } else {
-        // TODO: Handle SrcVT > DstVT, where truncation is needed.
+        // The unconditional branch case.
-        return I;
+        TII.InsertBranch(*MBB, MSucc, NULL, SmallVector<MachineOperand, 0>());
       }
+      MBB->addSuccessor(MSucc);
+      return true;
     }
-    
+
-    default:
+    // Conditional branches are not handed yet.
-      // Unhandled instruction. Halt "fast" selection and bail.
+    // Halt "fast" selection and bail.
-      return I;
+    return false;
-    }
   }
 
-  return I;
+  case Instruction::PHI:
+    // PHI nodes are already emitted.
+    return true;
+    
+  case Instruction::BitCast:
+    return SelectBitCast(I);
+
+  case Instruction::FPToSI:
+    return SelectCast(I, ISD::FP_TO_SINT);
+  case Instruction::ZExt:
+    return SelectCast(I, ISD::ZERO_EXTEND);
+  case Instruction::SExt:
+    return SelectCast(I, ISD::SIGN_EXTEND);
+  case Instruction::Trunc:
+    return SelectCast(I, ISD::TRUNCATE);
+  case Instruction::SIToFP:
+    return SelectCast(I, ISD::SINT_TO_FP);
+
+  case Instruction::IntToPtr: // Deliberate fall-through.
+  case Instruction::PtrToInt: {
+    MVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
+    MVT DstVT = TLI.getValueType(I->getType());
+    if (DstVT.bitsGT(SrcVT))
+      return SelectCast(I, ISD::ZERO_EXTEND);
+    if (DstVT.bitsLT(SrcVT))
+      return SelectCast(I, ISD::TRUNCATE);
+    unsigned Reg = getRegForValue(I->getOperand(0));
+    if (Reg == 0) return false;
+    UpdateValueMap(I, Reg);
+    return true;
+  }
+  
+  default:
+    // Unhandled instruction. Halt "fast" selection and bail.
+    return false;
+  }
 }
 
-FastISel::FastISel(MachineFunction &mf)
+FastISel::FastISel(MachineFunction &mf,
-  : MF(mf),
+                   DenseMap<const Value *, unsigned> &vm,
-    MRI(mf.getRegInfo()),
+                   DenseMap<const BasicBlock *, MachineBasicBlock *> &bm)
-    TM(mf.getTarget()),
+  : MBB(0),
+    ValueMap(vm),
+    MBBMap(bm),
+    MF(mf),
+    MRI(MF.getRegInfo()),
+    TM(MF.getTarget()),
     TD(*TM.getTargetData()),
     TII(*TM.getInstrInfo()),
     TLI(*TM.getTargetLowering()) {

lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp

@@ -4757,9 +4757,64 @@ SelectionDAGISel::HandlePHINodesInSuccessorBlocks(BasicBlock *LLVMBB) {
     }
   }
   SDL->ConstantsOut.clear();
-
-  // Lower the terminator after the copies are emitted.
-  SDL->visit(*LLVMBB->getTerminator());
 }
 
+/// This is the Fast-ISel version of HandlePHINodesInSuccessorBlocks. It only
+/// supports legal types, and it emits MachineInstrs directly instead of
+/// creating SelectionDAG nodes.
+///
+bool
+SelectionDAGISel::HandlePHINodesInSuccessorBlocksFast(BasicBlock *LLVMBB,
+                                                      FastISel *F) {
+  TerminatorInst *TI = LLVMBB->getTerminator();
 
+  SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
+  unsigned OrigNumPHINodesToUpdate = SDL->PHINodesToUpdate.size();
+
+  // Check successor nodes' PHI nodes that expect a constant to be available
+  // from this block.
+  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
+    BasicBlock *SuccBB = TI->getSuccessor(succ);
+    if (!isa<PHINode>(SuccBB->begin())) continue;
+    MachineBasicBlock *SuccMBB = FuncInfo->MBBMap[SuccBB];
+    
+    // If this terminator has multiple identical successors (common for
+    // switches), only handle each succ once.
+    if (!SuccsHandled.insert(SuccMBB)) continue;
+    
+    MachineBasicBlock::iterator MBBI = SuccMBB->begin();
+    PHINode *PN;
+
+    // At this point we know that there is a 1-1 correspondence between LLVM PHI
+    // nodes and Machine PHI nodes, but the incoming operands have not been
+    // emitted yet.
+    for (BasicBlock::iterator I = SuccBB->begin();
+         (PN = dyn_cast<PHINode>(I)); ++I) {
+      // Ignore dead phi's.
+      if (PN->use_empty()) continue;
+
+      // Only handle legal types. Two interesting things to note here. First,
+      // by bailing out early, we may leave behind some dead instructions,
+      // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
+      // own moves. Second, this check is necessary becuase FastISel doesn't
+      // use CreateRegForValue to create registers, so it always creates
+      // exactly one register for each non-void instruction.
+      MVT VT = TLI.getValueType(PN->getType(), /*AllowUnknown=*/true);
+      if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
+        SDL->PHINodesToUpdate.resize(OrigNumPHINodesToUpdate);
+        return false;
+      }
+
+      Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB);
+
+      unsigned Reg = F->getRegForValue(PHIOp);
+      if (Reg == 0) {
+        SDL->PHINodesToUpdate.resize(OrigNumPHINodesToUpdate);
+        return false;
+      }
+      SDL->PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg));
+    }
+  }
+
+  return true;
+}

lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

@@ -408,8 +408,12 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB,
     }
 
   // Handle PHI nodes in successor blocks.
-  if (End == LLVMBB->end())
+  if (End == LLVMBB->end()) {
     HandlePHINodesInSuccessorBlocks(LLVMBB);
+
+    // Lower the terminator after the copies are emitted.
+    SDL->visit(*LLVMBB->getTerminator());
+  }
     
   // Make sure the root of the DAG is up-to-date.
   CurDAG->setRoot(SDL->getControlRoot());
@@ -606,8 +610,9 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF) {
     BasicBlock *LLVMBB = &*I;
     BB = FuncInfo->MBBMap[LLVMBB];
 
-    BasicBlock::iterator Begin = LLVMBB->begin();
+    BasicBlock::iterator const Begin = LLVMBB->begin();
-    BasicBlock::iterator End = LLVMBB->end();
+    BasicBlock::iterator const End = LLVMBB->end();
+    BasicBlock::iterator I = Begin;
 
     // Lower any arguments needed in this block if this is the entry block.
     if (LLVMBB == &Fn.getEntryBlock())
@@ -616,7 +621,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF) {
     // Before doing SelectionDAG ISel, see if FastISel has been requested.
     // FastISel doesn't support EH landing pads, which require special handling.
     if (EnableFastISel && !BB->isLandingPad()) {
-      if (FastISel *F = TLI.createFastISel(*FuncInfo->MF)) {
+      if (FastISel *F = TLI.createFastISel(*FuncInfo->MF, FuncInfo->ValueMap,
+                                           FuncInfo->MBBMap)) {
         // Emit code for any incoming arguments. This must happen before
         // beginning FastISel on the entry block.
         if (LLVMBB == &Fn.getEntryBlock()) {
@@ -624,45 +630,51 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF) {
           CodeGenAndEmitDAG();
           SDL->clear();
         }
+        F->setCurrentBlock(BB);
         // Do FastISel on as many instructions as possible.
-        while (Begin != End) {
+        for (; I != End; ++I) {
-          Begin = F->SelectInstructions(Begin, End, FuncInfo->ValueMap,
+          // Just before the terminator instruction, insert instructions to
-                                        FuncInfo->MBBMap, BB);
+          // feed PHI nodes in successor blocks.
-
+          if (isa<TerminatorInst>(I))
-          // If the "fast" selector selected the entire block, we're done.
+            if (!HandlePHINodesInSuccessorBlocksFast(LLVMBB, F)) {
-          if (Begin == End)
+              if (DisableFastISelAbort)
-            break;
+                break;
-
+#ifndef NDEBUG
-          // Next, try calling the target to attempt to handle the instruction.
+              I->dump();
-          if (F->TargetSelectInstruction(Begin, FuncInfo->ValueMap,
+#endif  
-                                         FuncInfo->MBBMap, BB)) {
+              assert(0 && "FastISel didn't handle a PHI in a successor");
-            ++Begin;
-            continue;
-          }
-
-          // Handle certain instructions as single-LLVM-Instruction blocks.
-          if (isa<CallInst>(Begin) || isa<LoadInst>(Begin) ||
-              isa<StoreInst>(Begin)) {
-            if (Begin->getType() != Type::VoidTy) {
-              unsigned &R = FuncInfo->ValueMap[Begin];
-              if (!R)
-                R = FuncInfo->CreateRegForValue(Begin);
             }
 
-            SelectBasicBlock(LLVMBB, Begin, next(Begin));
+          // First try normal tablegen-generated "fast" selection.
-            ++Begin;
+          if (F->SelectInstruction(I))
+            continue;
+
+          // Next, try calling the target to attempt to handle the instruction.
+          if (F->TargetSelectInstruction(I))
+            continue;
+
+          // Then handle certain instructions as single-LLVM-Instruction blocks.
+          if (isa<CallInst>(I) || isa<LoadInst>(I) ||
+              isa<StoreInst>(I)) {
+            if (I->getType() != Type::VoidTy) {
+              unsigned &R = FuncInfo->ValueMap[I];
+              if (!R)
+                R = FuncInfo->CreateRegForValue(I);
+            }
+
+            SelectBasicBlock(LLVMBB, I, next(I));
             continue;
           }
 
           if (!DisableFastISelAbort &&
               // For now, don't abort on non-conditional-branch terminators.
-              (!isa<TerminatorInst>(Begin) ||
+              (!isa<TerminatorInst>(I) ||
-               (isa<BranchInst>(Begin) &&
+               (isa<BranchInst>(I) &&
-                cast<BranchInst>(Begin)->isUnconditional()))) {
+                cast<BranchInst>(I)->isUnconditional()))) {
             // The "fast" selector couldn't handle something and bailed.
             // For the purpose of debugging, just abort.
 #ifndef NDEBUG
-            Begin->dump();
+            I->dump();
 #endif
             assert(0 && "FastISel didn't select the entire block");
           }
@@ -674,12 +686,9 @@ void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF) {
 
     // Run SelectionDAG instruction selection on the remainder of the block
     // not handled by FastISel. If FastISel is not run, this is the entire
-    // block. If FastISel is run and happens to handle all of the
+    // block.
-    // LLVM Instructions in the block, [Begin,End) will be an empty range,
+    if (I != End)
-    // but we still need to run this so that
+      SelectBasicBlock(LLVMBB, I, End);
-    // HandlePHINodesInSuccessorBlocks is called and any resulting code
-    // is emitted.
-    SelectBasicBlock(LLVMBB, Begin, End);
 
     FinishBasicBlock();
   }

lib/Target/X86/X86FastISel.cpp

@@ -30,34 +30,27 @@ class X86FastISel : public FastISel {
   const X86Subtarget *Subtarget;
     
 public:
-  explicit X86FastISel(MachineFunction &mf) : FastISel(mf) {
+  explicit X86FastISel(MachineFunction &mf,
+                       DenseMap<const Value *, unsigned> &vm,
+                       DenseMap<const BasicBlock *, MachineBasicBlock *> &bm)
+    : FastISel(mf, vm, bm) {
     Subtarget = &TM.getSubtarget<X86Subtarget>();
   }
 
-  virtual bool
+  virtual bool TargetSelectInstruction(Instruction *I);
-    TargetSelectInstruction(Instruction *I,
-                            DenseMap<const Value *, unsigned> &ValueMap,
-                      DenseMap<const BasicBlock *, MachineBasicBlock *> &MBBMap,
-                            MachineBasicBlock *MBB);
 
 #include "X86GenFastISel.inc"
 
 private:
-  bool X86SelectConstAddr(Value *V,
+  bool X86SelectConstAddr(Value *V, unsigned &Op0);
-                          DenseMap<const Value *, unsigned> &ValueMap,
-                          MachineBasicBlock *MBB, unsigned &Op0);
 
-  bool X86SelectLoad(Instruction *I,
+  bool X86SelectLoad(Instruction *I);
-                     DenseMap<const Value *, unsigned> &ValueMap,
-                     MachineBasicBlock *MBB);
 };
 
 /// X86SelectConstAddr - Select and emit code to materialize constant address.
 /// 
 bool X86FastISel::X86SelectConstAddr(Value *V,
-                                    DenseMap<const Value *, unsigned> &ValueMap,
+                                     unsigned &Op0) {
-                                    MachineBasicBlock *MBB,
-                                    unsigned &Op0) {
   // FIXME: Only GlobalAddress for now.
   GlobalValue *GV = dyn_cast<GlobalValue>(V);
   if (!GV)
@@ -84,9 +77,7 @@ bool X86FastISel::X86SelectConstAddr(Value *V,
 
 /// X86SelectLoad - Select and emit code to implement load instructions.
 ///
-bool X86FastISel::X86SelectLoad(Instruction *I,
+bool X86FastISel::X86SelectLoad(Instruction *I)  {
-                                DenseMap<const Value *, unsigned> &ValueMap,
-                                MachineBasicBlock *MBB)  {
   MVT VT = MVT::getMVT(I->getType(), /*HandleUnknown=*/true);
   if (VT == MVT::Other || !VT.isSimple())
     // Unhandled type. Halt "fast" selection and bail.
@@ -102,10 +93,10 @@ bool X86FastISel::X86SelectLoad(Instruction *I,
     return false;
 
   Value *V = I->getOperand(0);
-  unsigned Op0 = getRegForValue(V, ValueMap);
+  unsigned Op0 = getRegForValue(V);
   if (Op0 == 0) {
     // Handle constant load address.
-    if (!isa<Constant>(V) || !X86SelectConstAddr(V, ValueMap, MBB, Op0))
+    if (!isa<Constant>(V) || !X86SelectConstAddr(V, Op0))
       // Unhandled operand. Halt "fast" selection and bail.
       return false;    
   }
@@ -164,27 +155,26 @@ bool X86FastISel::X86SelectLoad(Instruction *I,
   else
     AM.GV = cast<GlobalValue>(V);
   addFullAddress(BuildMI(MBB, TII.get(Opc), ResultReg), AM);
-  UpdateValueMap(I, ResultReg, ValueMap);
+  UpdateValueMap(I, ResultReg);
   return true;
 }
 
 
 bool
-X86FastISel::TargetSelectInstruction(Instruction *I,
+X86FastISel::TargetSelectInstruction(Instruction *I)  {
-                                  DenseMap<const Value *, unsigned> &ValueMap,
-                      DenseMap<const BasicBlock *, MachineBasicBlock *> &MBBMap,
-                                  MachineBasicBlock *MBB)  {
   switch (I->getOpcode()) {
   default: break;
   case Instruction::Load:
-    return X86SelectLoad(I, ValueMap, MBB);
+    return X86SelectLoad(I);
   }
 
   return false;
 }
 
 namespace llvm {
-  llvm::FastISel *X86::createFastISel(MachineFunction &mf) {
+  llvm::FastISel *X86::createFastISel(MachineFunction &mf,
-    return new X86FastISel(mf);
+                        DenseMap<const Value *, unsigned> &vm,
+                        DenseMap<const BasicBlock *, MachineBasicBlock *> &bm) {
+    return new X86FastISel(mf, vm, bm);
   }
 }

lib/Target/X86/X86ISelLowering.cpp

@@ -1879,8 +1879,12 @@ bool X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Call,
   return false;
 }
 
-FastISel *X86TargetLowering::createFastISel(MachineFunction &mf) {
+FastISel *
-  return X86::createFastISel(mf);
+X86TargetLowering::createFastISel(MachineFunction &mf,
+                                  DenseMap<const Value *, unsigned> &vm,
+                                  DenseMap<const BasicBlock *,
+                                           MachineBasicBlock *> &bm) {
+  return X86::createFastISel(mf, vm, bm);
 }
 
 

lib/Target/X86/X86ISelLowering.h

@@ -470,7 +470,10 @@ namespace llvm {
 
     /// createFastISel - This method returns a target specific FastISel object,
     /// or null if the target does not support "fast" ISel.
-    virtual FastISel *createFastISel(MachineFunction &mf);
+    virtual FastISel *
+    createFastISel(MachineFunction &mf,
+                   DenseMap<const Value *, unsigned> &,
+                   DenseMap<const BasicBlock *, MachineBasicBlock *> &);
     
   private:
     /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
@@ -598,7 +601,9 @@ namespace llvm {
   };
 
   namespace X86 {
-    FastISel *createFastISel(MachineFunction &mf);
+    FastISel *createFastISel(MachineFunction &mf,
+                           DenseMap<const Value *, unsigned> &,
+                           DenseMap<const BasicBlock *, MachineBasicBlock *> &);
   }
 }