[InstCombine] fold fdiv with non-splat divisor to fmul: X/C --> X * (1/C)

llvm-svn: 325590

include/llvm/IR/Constant.h

@@ -79,6 +79,10 @@ public:
   /// scalar constant or a vector constant with all normal elements.
   bool isNormalFP() const;
 
+  /// Return true if this scalar has an exact multiplicative inverse or this
+  /// vector has an exact multiplicative inverse for each element in the vector.
+  bool hasExactInverseFP() const;
+
   /// Return true if evaluation of this constant could trap. This is true for
   /// things like constant expressions that could divide by zero.
   bool canTrap() const;

lib/IR/Constants.cpp

@@ -228,6 +228,19 @@ bool Constant::isNormalFP() const {
   return true;
 }
 
+bool Constant::hasExactInverseFP() const {
+  if (auto *CFP = dyn_cast<ConstantFP>(this))
+    return CFP->getValueAPF().getExactInverse(nullptr);
+  if (!getType()->isVectorTy())
+    return false;
+  for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) {
+    auto *CFP = dyn_cast_or_null<ConstantFP>(this->getAggregateElement(i));
+    if (!CFP || !CFP->getValueAPF().getExactInverse(nullptr))
+      return false;
+  }
+  return true;
+}
+
 /// Constructor to create a '0' constant of arbitrary type.
 Constant *Constant::getNullValue(Type *Ty) {
   switch (Ty->getTypeID()) {

lib/Transforms/InstCombine/InstCombineMulDivRem.cpp

@@ -1289,32 +1289,27 @@ Instruction *InstCombiner::visitSDiv(BinaryOperator &I) {
 }
 
 /// Try to convert X/C into X * (1/C).
-static Instruction *foldFDivConstantDivisor(BinaryOperator &FDiv) {
-  // TODO: Handle non-splat vector constants.
-  const APFloat *C;
-  if (!match(FDiv.getOperand(1), m_APFloat(C)))
+static Instruction *foldFDivConstantDivisor(BinaryOperator &I) {
+  Constant *C;
+  if (!match(I.getOperand(1), m_Constant(C)))
+    return nullptr;
+
+  // If the constant divisor has an exact inverse, this is always safe. If not,
+  // then we can still create a reciprocal if fast-math-flags allow it and the
+  // constant is a regular number (not zero, infinite, or denormal).
+  if (!(C->hasExactInverseFP() || (I.hasAllowReciprocal() && C->isNormalFP())))
     return nullptr;
 
-  // This returns false if the inverse would be a denormal.
-  APFloat Reciprocal(C->getSemantics());
-  bool HasRecip = C->getExactInverse(&Reciprocal);
-  // If the inverse is not exact, we may still be able to convert if we are
-  // not operating with strict math.
-  if (!HasRecip && FDiv.hasAllowReciprocal() && C->isFiniteNonZero()) {
-    Reciprocal = APFloat(C->getSemantics(), 1.0f);
-    Reciprocal.divide(*C, APFloat::rmNearestTiesToEven);
   // Disallow denormal constants because we don't know what would happen
   // on all targets.
-    // TODO: Function attributes can tell us that denorms are flushed?
-    HasRecip = !Reciprocal.isDenormal();
-  }
-
-  if (!HasRecip)
+  // TODO: Use Intrinsic::canonicalize or let function attributes tell us that
+  // denorms are flushed?
+  auto *RecipC = ConstantExpr::getFDiv(ConstantFP::get(I.getType(), 1.0), C);
+  if (!RecipC->isNormalFP())
     return nullptr;
 
-  auto *RecipCFP = ConstantFP::get(FDiv.getType(), Reciprocal);
-  return BinaryOperator::CreateWithCopiedFlags(Instruction::FMul, RecipCFP,
-                                               FDiv.getOperand(0), &FDiv);
+  return BinaryOperator::CreateWithCopiedFlags(
+      Instruction::FMul, I.getOperand(0), RecipC, &I);
 }
 
 /// Try to reassociate C / X expressions where X includes another constant.

test/Transforms/InstCombine/fast-math.ll

@@ -229,6 +229,9 @@ define float @fmul_distribute1(float %f1) {
 }
 
 ; (X/C1 + C2) * C3 => X/(C1/C3) + C2*C3
+; TODO: We don't convert the fast fdiv to fmul because that would be multiplication
+; by a denormal, but we could do better when we know that denormals are not a problem.
+
 define double @fmul_distribute2(double %f1, double %f2) {
 ; CHECK-LABEL: @fmul_distribute2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fdiv fast double [[F1:%.*]], 0x7FE8000000000000
@@ -345,7 +348,9 @@ define float @fmul4(float %f1, float %f2) {
 
 ; X / C1 * C2 => X / (C2/C1) if  C1/C2 is either a special value of a denormal,
 ;  and C2/C1 is a normal value.
-;
+; TODO: We don't convert the fast fdiv to fmul because that would be multiplication
+; by a denormal, but we could do better when we know that denormals are not a problem.
+
 define float @fmul5(float %f1, float %f2) {
 ; CHECK-LABEL: @fmul5(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fdiv fast float [[F1:%.*]], 0x47E8000000000000

test/Transforms/InstCombine/fdiv.ll

@@ -86,11 +86,9 @@ define <2 x float> @not_exact_but_allow_recip_splat(<2 x float> %x) {
   ret <2 x float> %div
 }
 
-; FIXME: Vector neglect.
-
 define <2 x float> @exact_inverse_vec(<2 x float> %x) {
 ; CHECK-LABEL: @exact_inverse_vec(
-; CHECK-NEXT:    [[DIV:%.*]] = fdiv <2 x float> [[X:%.*]], <float 4.000000e+00, float 8.000000e+00>
+; CHECK-NEXT:    [[DIV:%.*]] = fmul <2 x float> [[X:%.*]], <float 2.500000e-01, float 1.250000e-01>
 ; CHECK-NEXT:    ret <2 x float> [[DIV]]
 ;
   %div = fdiv <2 x float> %x, <float 4.0, float 8.0>
@@ -115,6 +113,15 @@ define <2 x float> @not_exact_inverse_vec(<2 x float> %x) {
   ret <2 x float> %div
 }
 
+define <2 x float> @not_exact_inverse_vec_arcp(<2 x float> %x) {
+; CHECK-LABEL: @not_exact_inverse_vec_arcp(
+; CHECK-NEXT:    [[DIV:%.*]] = fmul arcp <2 x float> [[X:%.*]], <float 2.500000e-01, float 0x3FD5555560000000>
+; CHECK-NEXT:    ret <2 x float> [[DIV]]
+;
+  %div = fdiv arcp <2 x float> %x, <float 4.0, float 3.0>
+  ret <2 x float> %div
+}
+
 ; (X / Y) / Z --> X / (Y * Z)
 
 define float @div_with_div_numerator(float %x, float %y, float %z) {