[SelectionDAGBuilder] Pass fast math flags to getNode calls rather than trying to set them after the fact.:

This removes the after the fact FMF handling from D46854 in favor of passing fast math flags to getNode. This should be a superset of D87130.

This required adding a SDNodeFlags to SelectionDAG::getSetCC.

Now we manage to contant fold some stuff undefs during the
initial getNode that we don't do in later DAG combines.

Differential Revision: https://reviews.llvm.org/D87200

include/llvm/CodeGen/SelectionDAG.h

@@ -1049,8 +1049,8 @@ public:
   /// Helper function to make it easier to build SetCC's if you just have an
   /// ISD::CondCode instead of an SDValue.
   SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
-                   ISD::CondCode Cond, SDValue Chain = SDValue(),
+                   ISD::CondCode Cond, SDNodeFlags Flags = SDNodeFlags(),
-                   bool IsSignaling = false) {
+                   SDValue Chain = SDValue(), bool IsSignaling = false) {
     assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&
            "Cannot compare scalars to vectors");
     assert(LHS.getValueType().isVector() == VT.isVector() &&
@@ -1060,7 +1060,7 @@ public:
     if (Chain)
       return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL,
                      {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)});
-    return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
+    return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond), Flags);
   }
 
   /// Helper function to make it easier to build Select's if you just have

include/llvm/CodeGen/SelectionDAGNodes.h

@@ -357,10 +357,6 @@ template<> struct simplify_type<SDUse> {
 /// the backend.
 struct SDNodeFlags {
 private:
-  // This bit is used to determine if the flags are in a defined state. It is
-  // only used by SelectionDAGBuilder.
-  bool AnyDefined : 1;
-
   bool NoUnsignedWrap : 1;
   bool NoSignedWrap : 1;
   bool Exact : 1;
@@ -382,9 +378,8 @@ private:
 public:
   /// Default constructor turns off all optimization flags.
   SDNodeFlags()
-      : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
+      : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
-        Exact(false), NoNaNs(false), NoInfs(false),
+        NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
-        NoSignedZeros(false), AllowReciprocal(false),
         AllowContract(false), ApproximateFuncs(false),
         AllowReassociation(false), NoFPExcept(false) {}
 
@@ -399,56 +394,18 @@ public:
     setAllowReassociation(FPMO.hasAllowReassoc());
   }
 
-  /// Sets the state of the flags to the defined state.
-  void setDefined() { AnyDefined = true; }
-  /// Returns true if the flags are in a defined state.
-  bool isDefined() const { return AnyDefined; }
-
   // These are mutators for each flag.
-  void setNoUnsignedWrap(bool b) {
+  void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
-    setDefined();
+  void setNoSignedWrap(bool b) { NoSignedWrap = b; }
-    NoUnsignedWrap = b;
+  void setExact(bool b) { Exact = b; }
-  }
+  void setNoNaNs(bool b) { NoNaNs = b; }
-  void setNoSignedWrap(bool b) {
+  void setNoInfs(bool b) { NoInfs = b; }
-    setDefined();
+  void setNoSignedZeros(bool b) { NoSignedZeros = b; }
-    NoSignedWrap = b;
+  void setAllowReciprocal(bool b) { AllowReciprocal = b; }
-  }
+  void setAllowContract(bool b) { AllowContract = b; }
-  void setExact(bool b) {
+  void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
-    setDefined();
+  void setAllowReassociation(bool b) { AllowReassociation = b; }
-    Exact = b;
+  void setNoFPExcept(bool b) { NoFPExcept = b; }
-  }
-  void setNoNaNs(bool b) {
-    setDefined();
-    NoNaNs = b;
-  }
-  void setNoInfs(bool b) {
-    setDefined();
-    NoInfs = b;
-  }
-  void setNoSignedZeros(bool b) {
-    setDefined();
-    NoSignedZeros = b;
-  }
-  void setAllowReciprocal(bool b) {
-    setDefined();
-    AllowReciprocal = b;
-  }
-  void setAllowContract(bool b) {
-    setDefined();
-    AllowContract = b;
-  }
-  void setApproximateFuncs(bool b) {
-    setDefined();
-    ApproximateFuncs = b;
-  }
-  void setAllowReassociation(bool b) {
-    setDefined();
-    AllowReassociation = b;
-  }
-  void setNoFPExcept(bool b) {
-    setDefined();
-    NoFPExcept = b;
-  }
 
   // These are accessors for each flag.
   bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

@@ -7398,9 +7398,9 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
         if (N0.hasOneUse()) {
           // FIXME Can we handle multiple uses? Could we token factor the chain
           // results from the new/old setcc?
-          SDValue SetCC = DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC,
+          SDValue SetCC =
-                                       N0.getOperand(0),
+              DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, SDNodeFlags(),
-                                       N0Opcode == ISD::STRICT_FSETCCS);
+                           N0.getOperand(0), N0Opcode == ISD::STRICT_FSETCCS);
           CombineTo(N, SetCC);
           DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), SetCC.getValue(1));
           recursivelyDeleteUnusedNodes(N0.getNode());

lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

@@ -1735,12 +1735,16 @@ bool SelectionDAGLegalize::LegalizeSetCCCondCode(
     if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
       // If we aren't the ordered or unorder operation,
       // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
-      SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
+      SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, SDNodeFlags(), Chain,
-      SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
+                            IsSignaling);
+      SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, SDNodeFlags(), Chain,
+                            IsSignaling);
     } else {
       // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
-      SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
+      SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, SDNodeFlags(), Chain,
-      SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
+                            IsSignaling);
+      SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, SDNodeFlags(), Chain,
+                            IsSignaling);
     }
     if (Chain)
       Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),

lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

@@ -1777,17 +1777,18 @@ void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
   // The following can be improved, but not that much.
   SDValue Tmp1, Tmp2, Tmp3, OutputChain;
   Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
-                      RHSHi, ISD::SETOEQ, Chain, IsSignaling);
+                      RHSHi, ISD::SETOEQ, SDNodeFlags(), Chain, IsSignaling);
   OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
   Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
-                      RHSLo, CCCode, OutputChain, IsSignaling);
+                      RHSLo, CCCode, SDNodeFlags(), OutputChain, IsSignaling);
   OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
   Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
-  Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
+  Tmp1 =
-                      RHSHi, ISD::SETUNE, OutputChain, IsSignaling);
+      DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
+                   ISD::SETUNE, SDNodeFlags(), OutputChain, IsSignaling);
   OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
   Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
-                      RHSHi, CCCode, OutputChain, IsSignaling);
+                      RHSHi, CCCode, SDNodeFlags(), OutputChain, IsSignaling);
   OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
   Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
   NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

@@ -1120,27 +1120,6 @@ void SelectionDAGBuilder::visit(const Instruction &I) {
 
   visit(I.getOpcode(), I);
 
-  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
-    // ConstrainedFPIntrinsics handle their own FMF.
-    if (!isa<ConstrainedFPIntrinsic>(&I)) {
-      // Propagate the fast-math-flags of this IR instruction to the DAG node that
-      // maps to this instruction.
-      // TODO: We could handle all flags (nsw, etc) here.
-      // TODO: If an IR instruction maps to >1 node, only the final node will have
-      //       flags set.
-      // TODO: The handling of flags should be improved, see
-      //       https://reviews.llvm.org/D86871
-      if (SDNode *Node = getNodeForIRValue(&I)) {
-        SDNodeFlags IncomingFlags;
-        IncomingFlags.copyFMF(*FPMO);
-        if (!Node->getFlags().isDefined())
-          Node->setFlags(IncomingFlags);
-        else
-          Node->intersectFlagsWith(IncomingFlags);
-      }
-    }
-  }
-
   if (!I.isTerminator() && !HasTailCall &&
       !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
     CopyToExportRegsIfNeeded(&I);
@@ -3023,9 +3002,10 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
     Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
     Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
   }
-  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
+  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
     Flags.setExact(ExactOp->isExact());
-  }
+  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
+    Flags.copyFMF(*FPOp);
 
   SDValue Op1 = getValue(I.getOperand(0));
   SDValue Op2 = getValue(I.getOperand(1));
@@ -3135,13 +3115,16 @@ void SelectionDAGBuilder::visitFCmp(const User &I) {
   SDValue Op2 = getValue(I.getOperand(1));
 
   ISD::CondCode Condition = getFCmpCondCode(predicate);
-  auto *FPMO = dyn_cast<FPMathOperator>(&I);
+  auto *FPMO = cast<FPMathOperator>(&I);
-  if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
+  if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
     Condition = getFCmpCodeWithoutNaN(Condition);
 
+  SDNodeFlags Flags;
+  Flags.copyFMF(*FPMO);
+
   EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                         I.getType());
-  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
+  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition, Flags));
 }
 
 // Check if the condition of the select has one use or two users that are both
@@ -3169,6 +3152,10 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
 
   bool IsUnaryAbs = false;
 
+  SDNodeFlags Flags;
+  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
+    Flags.copyFMF(*FPOp);
+
   // Min/max matching is only viable if all output VTs are the same.
   if (is_splat(ValueVTs)) {
     EVT VT = ValueVTs[0];
@@ -3272,7 +3259,7 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
       Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
       Values[i] = DAG.getNode(
           OpCode, getCurSDLoc(),
-          LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops);
+          LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
     }
   }
 
@@ -4876,7 +4863,7 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
 /// expandExp - Lower an exp intrinsic. Handles the special sequences for
 /// limited-precision mode.
 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
-                         const TargetLowering &TLI) {
+                         const TargetLowering &TLI, SDNodeFlags Flags) {
   if (Op.getValueType() == MVT::f32 &&
       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
 
@@ -4892,13 +4879,13 @@ static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
   }
 
   // No special expansion.
-  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
+  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
 }
 
 /// expandLog - Lower a log intrinsic. Handles the special sequences for
 /// limited-precision mode.
 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
-                         const TargetLowering &TLI) {
+                         const TargetLowering &TLI, SDNodeFlags Flags) {
   // TODO: What fast-math-flags should be set on the floating-point nodes?
 
   if (Op.getValueType() == MVT::f32 &&
@@ -4991,13 +4978,13 @@ static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
   }
 
   // No special expansion.
-  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
+  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
 }
 
 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
 /// limited-precision mode.
 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
-                          const TargetLowering &TLI) {
+                          const TargetLowering &TLI, SDNodeFlags Flags) {
   // TODO: What fast-math-flags should be set on the floating-point nodes?
 
   if (Op.getValueType() == MVT::f32 &&
@@ -5088,13 +5075,13 @@ static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
   }
 
   // No special expansion.
-  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
+  return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
 }
 
 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
 /// limited-precision mode.
 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
-                           const TargetLowering &TLI) {
+                           const TargetLowering &TLI, SDNodeFlags Flags) {
   // TODO: What fast-math-flags should be set on the floating-point nodes?
 
   if (Op.getValueType() == MVT::f32 &&
@@ -5178,25 +5165,26 @@ static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
   }
 
   // No special expansion.
-  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
+  return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
 }
 
 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
 /// limited-precision mode.
 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
-                          const TargetLowering &TLI) {
+                          const TargetLowering &TLI, SDNodeFlags Flags) {
   if (Op.getValueType() == MVT::f32 &&
       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
     return getLimitedPrecisionExp2(Op, dl, DAG);
 
   // No special expansion.
-  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
+  return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
 }
 
 /// visitPow - Lower a pow intrinsic. Handles the special sequences for
 /// limited-precision mode with x == 10.0f.
 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
-                         SelectionDAG &DAG, const TargetLowering &TLI) {
+                         SelectionDAG &DAG, const TargetLowering &TLI,
+                         SDNodeFlags Flags) {
   bool IsExp10 = false;
   if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
       LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
@@ -5219,7 +5207,7 @@ static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
   }
 
   // No special expansion.
-  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
+  return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
 }
 
 /// ExpandPowI - Expand a llvm.powi intrinsic.
@@ -5640,6 +5628,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   DebugLoc dl = getCurDebugLoc();
   SDValue Res;
 
+  SDNodeFlags Flags;
+  if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
+    Flags.copyFMF(*FPOp);
+
   switch (Intrinsic) {
   default:
     // By default, turn this into a target intrinsic node.
@@ -6054,23 +6046,26 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                             getValue(I.getArgOperand(1)), DAG));
     return;
   case Intrinsic::log:
-    setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
+    setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
     return;
   case Intrinsic::log2:
-    setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
+    setValue(&I,
+             expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
     return;
   case Intrinsic::log10:
-    setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
+    setValue(&I,
+             expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
     return;
   case Intrinsic::exp:
-    setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
+    setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
     return;
   case Intrinsic::exp2:
-    setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
+    setValue(&I,
+             expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
     return;
   case Intrinsic::pow:
     setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
-                           getValue(I.getArgOperand(1)), DAG, TLI));
+                           getValue(I.getArgOperand(1)), DAG, TLI, Flags));
     return;
   case Intrinsic::sqrt:
   case Intrinsic::fabs:
@@ -6103,7 +6098,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
 
     setValue(&I, DAG.getNode(Opcode, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
-                             getValue(I.getArgOperand(0))));
+                             getValue(I.getArgOperand(0)), Flags));
     return;
   }
   case Intrinsic::lround:
@@ -6128,38 +6123,37 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
                              getValue(I.getArgOperand(0)),
-                             getValue(I.getArgOperand(1))));
+                             getValue(I.getArgOperand(1)), Flags));
     return;
   case Intrinsic::maxnum:
     setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
                              getValue(I.getArgOperand(0)),
-                             getValue(I.getArgOperand(1))));
+                             getValue(I.getArgOperand(1)), Flags));
     return;
   case Intrinsic::minimum:
     setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
                              getValue(I.getArgOperand(0)),
-                             getValue(I.getArgOperand(1))));
+                             getValue(I.getArgOperand(1)), Flags));
     return;
   case Intrinsic::maximum:
     setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
                              getValue(I.getArgOperand(0)),
-                             getValue(I.getArgOperand(1))));
+                             getValue(I.getArgOperand(1)), Flags));
     return;
   case Intrinsic::copysign:
     setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
                              getValue(I.getArgOperand(0)),
-                             getValue(I.getArgOperand(1))));
+                             getValue(I.getArgOperand(1)), Flags));
     return;
   case Intrinsic::fma:
-    setValue(&I, DAG.getNode(ISD::FMA, sdl,
+    setValue(&I, DAG.getNode(
-                             getValue(I.getArgOperand(0)).getValueType(),
+                     ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
-                             getValue(I.getArgOperand(0)),
+                     getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
-                             getValue(I.getArgOperand(1)),
+                     getValue(I.getArgOperand(2)), Flags));
-                             getValue(I.getArgOperand(2))));
     return;
 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
   case Intrinsic::INTRINSIC:
@@ -6174,17 +6168,15 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                                getValue(I.getArgOperand(0)).getValueType(),
                                getValue(I.getArgOperand(0)),
                                getValue(I.getArgOperand(1)),
-                               getValue(I.getArgOperand(2))));
+                               getValue(I.getArgOperand(2)), Flags));
     } else {
       // TODO: Intrinsic calls should have fast-math-flags.
-      SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
+      SDValue Mul = DAG.getNode(
-                                getValue(I.getArgOperand(0)).getValueType(),
+          ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
-                                getValue(I.getArgOperand(0)),
+          getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
-                                getValue(I.getArgOperand(1)));
       SDValue Add = DAG.getNode(ISD::FADD, sdl,
                                 getValue(I.getArgOperand(0)).getValueType(),
-                                Mul,
+                                Mul, getValue(I.getArgOperand(2)), Flags);
-                                getValue(I.getArgOperand(2)));
       setValue(&I, Add);
     }
     return;
@@ -7532,8 +7524,12 @@ bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
   if (!I.onlyReadsMemory())
     return false;
 
+  SDNodeFlags Flags;
+  Flags.copyFMF(cast<FPMathOperator>(I));
+
   SDValue Tmp = getValue(I.getArgOperand(0));
-  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
+  setValue(&I,
+           DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
   return true;
 }
 
@@ -7548,10 +7544,13 @@ bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
   if (!I.onlyReadsMemory())
     return false;
 
+  SDNodeFlags Flags;
+  Flags.copyFMF(cast<FPMathOperator>(I));
+
   SDValue Tmp0 = getValue(I.getArgOperand(0));
   SDValue Tmp1 = getValue(I.getArgOperand(1));
   EVT VT = Tmp0.getValueType();
-  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
+  setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
   return true;
 }
 
@@ -8952,23 +8951,28 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
   SDValue Res;
   FastMathFlags FMF;
-  if (isa<FPMathOperator>(I))
+  SDNodeFlags SDFlags;
-    FMF = I.getFastMathFlags();
+  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
+    FMF = FPMO->getFastMathFlags();
+    SDFlags.copyFMF(*FPMO);
+  }
 
   switch (Intrinsic) {
   case Intrinsic::experimental_vector_reduce_v2_fadd:
     if (FMF.allowReassoc())
       Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
-                        DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2));
+                        DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
+                        SDFlags);
     else
-      Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2);
+      Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2, SDFlags);
     break;
   case Intrinsic::experimental_vector_reduce_v2_fmul:
     if (FMF.allowReassoc())
       Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
-                        DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2));
+                        DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
+                        SDFlags);
     else
-      Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2);
+      Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2, SDFlags);
     break;
   case Intrinsic::experimental_vector_reduce_add:
     Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
@@ -8998,10 +9002,10 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
     Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
     break;
   case Intrinsic::experimental_vector_reduce_fmax:
-    Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1);
+    Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
     break;
   case Intrinsic::experimental_vector_reduce_fmin:
-    Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1);
+    Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
     break;
   default:
     llvm_unreachable("Unhandled vector reduce intrinsic");

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

@@ -518,13 +518,6 @@ public:
 
   SDValue getValue(const Value *V);
 
-  /// Return the SDNode for the specified IR value if it exists.
-  SDNode *getNodeForIRValue(const Value *V) {
-    if (NodeMap.find(V) == NodeMap.end())
-      return nullptr;
-    return NodeMap[V].getNode();
-  }
-
   SDValue getNonRegisterValue(const Value *V);
   SDValue getValueImpl(const Value *V);
 

lib/CodeGen/SelectionDAG/TargetLowering.cpp

@@ -6409,7 +6409,7 @@ bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
   SDValue Sel;
 
   if (Node->isStrictFPOpcode()) {
-    Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
+    Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, SDNodeFlags(),
                        Node->getOperand(0), /*IsSignaling*/ true);
     Chain = Sel.getValue(1);
   } else {

lib/Target/PowerPC/PPCISelLowering.cpp

@@ -8219,8 +8219,8 @@ SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
               getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
           EVT DstSetCCVT =
               getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
-          SDValue Sel =
+          SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
-              DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, Chain, true);
+                                     SDNodeFlags(), Chain, true);
           Chain = Sel.getValue(1);
 
           SDValue FltOfs = DAG.getSelect(

lib/Target/X86/X86ISelLowering.cpp

@@ -20345,7 +20345,7 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
                                    *DAG.getContext(), TheVT);
     SDValue Cmp;
     if (IsStrict) {
-      Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT,
+      Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT, SDNodeFlags(),
                          Chain, /*IsSignaling*/ true);
       Chain = Cmp.getValue(1);
     } else {

test/CodeGen/AArch64/fp-const-fold.ll

@@ -161,49 +161,33 @@ define double @fmul_nnan_inf_op1(double %x) {
   ret double %r
 }
 
-; TODO: Should simplify to undef
-
 define double @fdiv_nnan_undef_op0(double %x) {
 ; CHECK-LABEL: fdiv_nnan_undef_op0:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #9221120237041090560
-; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ret
   %r = fdiv nnan double undef, %x
   ret double %r
 }
 
-; TODO: Should simplify to undef
-
 define double @fdiv_nnan_undef_op1(double %x) {
 ; CHECK-LABEL: fdiv_nnan_undef_op1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #9221120237041090560
-; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ret
   %r = fdiv nnan double %x, undef
   ret double %r
 }
 
-; TODO: Should simplify to undef
-
 define double @fdiv_ninf_undef_op0(double %x) {
 ; CHECK-LABEL: fdiv_ninf_undef_op0:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #9221120237041090560
-; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ret
   %r = fdiv ninf double undef, %x
   ret double %r
 }
 
-; TODO: Should simplify to undef
-
 define double @fdiv_ninf_undef_op1(double %x) {
 ; CHECK-LABEL: fdiv_ninf_undef_op1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #9221120237041090560
-; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    ret
   %r = fdiv ninf double %x, undef
   ret double %r

test/CodeGen/PowerPC/fmf-propagation.ll

@@ -557,13 +557,13 @@ define double @fcmp_nnan(double %a, double %y, double %z) {
 ; FMFDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'log2_approx:'
 ; FMFDEBUG:         ch,glue = PPCISD::CALL_NOP t11, TargetGlobalAddress:i64<double (double)* @log2>
 ; FMFDEBUG:         ch,glue = callseq_end t15, TargetConstant:i64<32>, TargetConstant:i64<0>, t15:1
-; FMFDEBUG:         f64,ch,glue = CopyFromReg afn t16, Register:f64 $f1, t16:1
+; FMFDEBUG:         f64,ch,glue = CopyFromReg t16, Register:f64 $f1, t16:1
 ; FMFDEBUG:       Type-legalized selection DAG: %bb.0 'log2_approx:'
 
 ; GLOBALDEBUG-LABEL: Optimized lowered selection DAG: %bb.0 'log2_approx:'
 ; GLOBALDEBUG:         ch,glue = PPCISD::CALL_NOP t11, TargetGlobalAddress:i64<double (double)* @log2>
 ; GLOBALDEBUG:         ch,glue = callseq_end t15, TargetConstant:i64<32>, TargetConstant:i64<0>, t15:1
-; GLOBALDEBUG:         f64,ch,glue = CopyFromReg afn t16, Register:f64 $f1, t16:1
+; GLOBALDEBUG:         f64,ch,glue = CopyFromReg t16, Register:f64 $f1, t16:1
 ; GLOBALDEBUG:       Type-legalized selection DAG: %bb.0 'log2_approx:'
 
 declare double @log2(double)

test/CodeGen/SystemZ/fp-mul-14.ll

@@ -2,9 +2,6 @@
 ;
 ; Check that a multiply-and-add results.
 
-; FIXME: This test is xfailed temporarily
-; XFAIL: *
-
 define void @f1(float %arg, float* %Dst) {
 ; CHECK-LABEL: f1:
 ; CHECK: maeb

test/CodeGen/Thumb2/mve-vecreduce-fadd.ll

@@ -3,30 +3,11 @@
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve,+fullfp16,+fp64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP
 
 define arm_aapcs_vfpcc float @fadd_v2f32(<2 x float> %x, float %y) {
-; CHECK-FP-LABEL: fadd_v2f32:
+; CHECK-LABEL: fadd_v2f32:
-; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK:       @ %bb.0: @ %entry
-; CHECK-FP-NEXT:    vadd.f32 s0, s0, s1
+; CHECK-NEXT:    vadd.f32 s0, s0, s1
-; CHECK-FP-NEXT:    vldr s2, .LCPI0_0
+; CHECK-NEXT:    vadd.f32 s0, s4, s0
-; CHECK-FP-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    bx lr
-; CHECK-FP-NEXT:    vadd.f32 s0, s4, s0
-; CHECK-FP-NEXT:    bx lr
-; CHECK-FP-NEXT:    .p2align 2
-; CHECK-FP-NEXT:  @ %bb.1:
-; CHECK-FP-NEXT:  .LCPI0_0:
-; CHECK-FP-NEXT:    .long 0x00000000 @ float 0
-;
-; CHECK-NOFP-LABEL: fadd_v2f32:
-; CHECK-NOFP:       @ %bb.0: @ %entry
-; CHECK-NOFP-NEXT:    vadd.f32 s0, s0, s1
-; CHECK-NOFP-NEXT:    vldr s2, .LCPI0_0
-; CHECK-NOFP-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f32 s0, s4, s0
-; CHECK-NOFP-NEXT:    bx lr
-; CHECK-NOFP-NEXT:    .p2align 2
-; CHECK-NOFP-NEXT:  @ %bb.1:
-; CHECK-NOFP-NEXT:  .LCPI0_0:
-; CHECK-NOFP-NEXT:    .long 0x00000000 @ float 0
 entry:
   %z = call fast float @llvm.experimental.vector.reduce.v2.fadd.f32.v2f32(float %y, <2 x float> %x)
   ret float %z
@@ -80,34 +61,14 @@ entry:
 }
 
 define arm_aapcs_vfpcc void @fadd_v2f16(<2 x half> %x, half* %yy) {
-; CHECK-FP-LABEL: fadd_v2f16:
+; CHECK-LABEL: fadd_v2f16:
-; CHECK-FP:       @ %bb.0: @ %entry
+; CHECK:       @ %bb.0: @ %entry
-; CHECK-FP-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP-NEXT:    vadd.f16 s0, s0, s4
+; CHECK-NEXT:    vadd.f16 s0, s0, s4
-; CHECK-FP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NEXT:    vldr.16 s2, [r0]
-; CHECK-FP-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NEXT:    vadd.f16 s0, s2, s0
-; CHECK-FP-NEXT:    vstr.16 s0, [r0]
+; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-FP-NEXT:    bx lr
+; CHECK-NEXT:    bx lr
-;
-; CHECK-NOFP-LABEL: fadd_v2f16:
-; CHECK-NOFP:       @ %bb.0: @ %entry
-; CHECK-NOFP-NEXT:    vmovx.f16 s4, s0
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s4
-; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI3_0
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s2, s0
-; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
-; CHECK-NOFP-NEXT:    bx lr
-; CHECK-NOFP-NEXT:    .p2align 1
-; CHECK-NOFP-NEXT:  @ %bb.1:
-; CHECK-NOFP-NEXT:  .LCPI3_0:
-; CHECK-NOFP-NEXT:    .short 0x0000 @ half 0
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.v2.fadd.f16.v2f16(half %y, <2 x half> %x)
@@ -134,20 +95,11 @@ define arm_aapcs_vfpcc void @fadd_v4f16(<4 x half> %x, half* %yy) {
 ; CHECK-NOFP-NEXT:    vadd.f16 s4, s0, s4
 ; CHECK-NOFP-NEXT:    vmovx.f16 s0, s1
 ; CHECK-NOFP-NEXT:    vadd.f16 s4, s4, s1
-; CHECK-NOFP-NEXT:    vldr.16 s2, .LCPI4_0
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s4, s0
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
-; CHECK-NOFP-NEXT:    vadd.f16 s0, s0, s2
 ; CHECK-NOFP-NEXT:    vldr.16 s2, [r0]
+; CHECK-NOFP-NEXT:    vadd.f16 s0, s4, s0
 ; CHECK-NOFP-NEXT:    vadd.f16 s0, s2, s0
 ; CHECK-NOFP-NEXT:    vstr.16 s0, [r0]
 ; CHECK-NOFP-NEXT:    bx lr
-; CHECK-NOFP-NEXT:    .p2align 1
-; CHECK-NOFP-NEXT:  @ %bb.1:
-; CHECK-NOFP-NEXT:  .LCPI4_0:
-; CHECK-NOFP-NEXT:    .short 0x0000 @ half 0
 entry:
   %y = load half, half* %yy
   %z = call fast half @llvm.experimental.vector.reduce.v2.fadd.f16.v4f16(half %y, <4 x half> %x)

test/CodeGen/X86/fp-undef.ll

@@ -100,7 +100,6 @@ define float @frem_undef_op1(float %x) {
 define float @fadd_undef_op0_nnan(float %x) {
 ; ANY-LABEL: fadd_undef_op0_nnan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fadd nnan float undef, %x
   ret float %r
@@ -109,7 +108,6 @@ define float @fadd_undef_op0_nnan(float %x) {
 define float @fadd_undef_op1_fast(float %x) {
 ; ANY-LABEL: fadd_undef_op1_fast:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fadd fast float %x, undef
   ret float %r
@@ -118,7 +116,6 @@ define float @fadd_undef_op1_fast(float %x) {
 define float @fsub_undef_op0_fast(float %x) {
 ; ANY-LABEL: fsub_undef_op0_fast:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fsub fast float undef, %x
   ret float %r
@@ -127,7 +124,6 @@ define float @fsub_undef_op0_fast(float %x) {
 define float @fsub_undef_op1_nnan(float %x) {
 ; ANY-LABEL: fsub_undef_op1_nnan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fsub nnan float %x, undef
   ret float %r
@@ -136,7 +132,6 @@ define float @fsub_undef_op1_nnan(float %x) {
 define float @fmul_undef_op0_nnan(float %x) {
 ; ANY-LABEL: fmul_undef_op0_nnan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fmul nnan float undef, %x
   ret float %r
@@ -145,7 +140,6 @@ define float @fmul_undef_op0_nnan(float %x) {
 define float @fmul_undef_op1_fast(float %x) {
 ; ANY-LABEL: fmul_undef_op1_fast:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fmul fast float %x, undef
   ret float %r
@@ -154,7 +148,6 @@ define float @fmul_undef_op1_fast(float %x) {
 define float @fdiv_undef_op0_fast(float %x) {
 ; ANY-LABEL: fdiv_undef_op0_fast:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fdiv fast float undef, %x
   ret float %r
@@ -163,7 +156,6 @@ define float @fdiv_undef_op0_fast(float %x) {
 define float @fdiv_undef_op1_nnan(float %x) {
 ; ANY-LABEL: fdiv_undef_op1_nnan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fdiv nnan float %x, undef
   ret float %r
@@ -172,7 +164,6 @@ define float @fdiv_undef_op1_nnan(float %x) {
 define float @frem_undef_op0_nnan(float %x) {
 ; ANY-LABEL: frem_undef_op0_nnan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = frem nnan float undef, %x
   ret float %r
@@ -181,7 +172,6 @@ define float @frem_undef_op0_nnan(float %x) {
 define float @frem_undef_op1_fast(float %x) {
 ; ANY-LABEL: frem_undef_op1_fast:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = frem fast float %x, undef
   ret float %r
@@ -234,7 +224,6 @@ define double @frem_undef_undef(double %x) {
 define float @fadd_undef_op0_nnan_constant(float %x) {
 ; ANY-LABEL: fadd_undef_op0_nnan_constant:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fadd nnan float undef, 1.0
   ret float %r
@@ -252,7 +241,6 @@ define float @fadd_undef_op1_constant(float %x) {
 define float @fsub_undef_op0_fast_constant(float %x) {
 ; ANY-LABEL: fsub_undef_op0_fast_constant:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fsub fast float undef, 3.0
   ret float %r
@@ -270,7 +258,6 @@ define float @fsub_undef_op1_constant(float %x) {
 define float @fmul_undef_op0_nnan_constant(float %x) {
 ; ANY-LABEL: fmul_undef_op0_nnan_constant:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fmul nnan float undef, 5.0
   ret float %r
@@ -288,7 +275,6 @@ define float @fmul_undef_op1_constant(float %x) {
 define float @fdiv_undef_op0_fast_constant(float %x) {
 ; ANY-LABEL: fdiv_undef_op0_fast_constant:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = fdiv fast float undef, 7.0
   ret float %r
@@ -306,7 +292,6 @@ define float @fdiv_undef_op1_constant(float %x) {
 define float @frem_undef_op0_nnan_constant(float %x) {
 ; ANY-LABEL: frem_undef_op0_nnan_constant:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ANY-NEXT:    retq
   %r = frem nnan float undef, 9.0
   ret float %r
@@ -335,7 +320,6 @@ define double @fadd_undef_op0_constant_nan(double %x) {
 define double @fadd_undef_op1_fast_constant_nan(double %x) {
 ; ANY-LABEL: fadd_undef_op1_fast_constant_nan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fadd fast double 0xFFF0000000000001, undef
   ret double %r
@@ -353,7 +337,6 @@ define double @fsub_undef_op0_constant_nan(double %x) {
 define double @fsub_undef_op1_nnan_constant_nan(double %x) {
 ; ANY-LABEL: fsub_undef_op1_nnan_constant_nan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fsub nnan double 0x7FF0000000000011, undef
   ret double %r
@@ -371,7 +354,6 @@ define double @fmul_undef_op0_constant_nan(double %x) {
 define double @fmul_undef_op1_fast_constant_nan(double %x) {
 ; ANY-LABEL: fmul_undef_op1_fast_constant_nan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fmul fast double 0xFFF0000000000101, undef
   ret double %r
@@ -389,7 +371,6 @@ define double @fdiv_undef_op0_constant_nan(double %x) {
 define double @fdiv_undef_op1_nnan_constant_nan(double %x) {
 ; ANY-LABEL: fdiv_undef_op1_nnan_constant_nan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fdiv nnan double 0x7FF0000000000111, undef
   ret double %r
@@ -407,7 +388,6 @@ define double @frem_undef_op0_constant_nan(double %x) {
 define double @frem_undef_op1_fast_constant_nan(double %x) {
 ; ANY-LABEL: frem_undef_op1_fast_constant_nan:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = frem fast double 0xFFF0000000001001, undef
   ret double %r
@@ -427,7 +407,6 @@ define double @fadd_undef_op0_constant_inf(double %x) {
 define double @fadd_undef_op1_fast_constant_inf(double %x) {
 ; ANY-LABEL: fadd_undef_op1_fast_constant_inf:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fadd fast double 0xFFF0000000000000, undef
   ret double %r
@@ -445,7 +424,6 @@ define double @fsub_undef_op0_constant_inf(double %x) {
 define double @fsub_undef_op1_ninf_constant_inf(double %x) {
 ; ANY-LABEL: fsub_undef_op1_ninf_constant_inf:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fsub ninf double 0x7FF0000000000000, undef
   ret double %r
@@ -463,7 +441,6 @@ define double @fmul_undef_op0_constant_inf(double %x) {
 define double @fmul_undef_op1_fast_constant_inf(double %x) {
 ; ANY-LABEL: fmul_undef_op1_fast_constant_inf:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fmul fast double 0xFFF0000000000000, undef
   ret double %r
@@ -481,7 +458,6 @@ define double @fdiv_undef_op0_constant_inf(double %x) {
 define double @fdiv_undef_op1_ninf_constant_inf(double %x) {
 ; ANY-LABEL: fdiv_undef_op1_ninf_constant_inf:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = fdiv ninf double 0x7FF0000000000000, undef
   ret double %r
@@ -499,7 +475,6 @@ define double @frem_undef_op0_constant_inf(double %x) {
 define double @frem_undef_op1_fast_constant_inf(double %x) {
 ; ANY-LABEL: frem_undef_op1_fast_constant_inf:
 ; ANY:       # %bb.0:
-; ANY-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; ANY-NEXT:    retq
   %r = frem fast double 0xFFF0000000000000, undef
   ret double %r