RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 20:51:52 +01:00
Code Issues Projects Releases Wiki Activity

[DAGCombine] Support FMF contract in fused multiple-and-sub too

Browse Source 
This is a follow-on to r299096 which added support for fmadd.

Subtract does not have the case where with two multiply operands we commute in
order to fuse with the multiply with the fewer uses.

llvm-svn: 299572
This commit is contained in:
Adam Nemet 2017-04-05 17:58:48 +00:00
parent 33bb79be94
commit f9970cbc84
2 changed files with 60 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lib/CodeGen/SelectionDAG/DAGCombiner.cpp
View File

@ -8909,23 +8909,26 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
SDLoc SL(N); SDLoc SL(N);
const TargetOptions &Options = DAG.getTarget().Options; const TargetOptions &Options = DAG.getTarget().Options;
bool AllowFusion =
(Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
// Floating-point multiply-add with intermediate rounding. // Floating-point multiply-add with intermediate rounding.
bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)); bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT));
// Floating-point multiply-add without intermediate rounding. // Floating-point multiply-add without intermediate rounding.
bool HasFMA = bool HasFMA =
AllowFusion && TLI.isFMAFasterThanFMulAndFAdd(VT) && TLI.isFMAFasterThanFMulAndFAdd(VT) &&
(!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT));
// No valid opcode, do not combine. // No valid opcode, do not combine.
if (!HasFMAD && !HasFMA) if (!HasFMAD && !HasFMA)
return SDValue(); return SDValue();
bool AllowFusionGlobally = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
Options.UnsafeFPMath || HasFMAD);
// If the subtraction is not contractable, do not combine.
if (!AllowFusionGlobally && !isContractable(N))
return SDValue();
const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo(); const SelectionDAGTargetInfo *STI = DAG.getSubtarget().getSelectionDAGInfo();
if (AllowFusion && STI && STI->generateFMAsInMachineCombiner(OptLevel)) if (STI && STI->generateFMAsInMachineCombiner(OptLevel))
return SDValue(); return SDValue();
// Always prefer FMAD to FMA for precision. // Always prefer FMAD to FMA for precision.
@ -8933,9 +8936,16 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
bool Aggressive = TLI.enableAggressiveFMAFusion(VT); bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
bool LookThroughFPExt = TLI.isFPExtFree(VT); bool LookThroughFPExt = TLI.isFPExtFree(VT);
// Is the node an FMUL and contractable either due to global flags or
// SDNodeFlags.
auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
if (N.getOpcode() != ISD::FMUL)
return false;
return AllowFusionGlobally || isContractable(N.getNode());
};
// fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z)) // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
if (N0.getOpcode() == ISD::FMUL && if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) {
(Aggressive || N0->hasOneUse())) {
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(0), N0.getOperand(1), N0.getOperand(0), N0.getOperand(1),
DAG.getNode(ISD::FNEG, SL, VT, N1)); DAG.getNode(ISD::FNEG, SL, VT, N1));
@ -8943,16 +8953,14 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
// fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x) // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
// Note: Commutes FSUB operands. // Note: Commutes FSUB operands.
if (N1.getOpcode() == ISD::FMUL && if (isContractableFMUL(N1) && (Aggressive || N1->hasOneUse()))
(Aggressive || N1->hasOneUse()))
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
DAG.getNode(ISD::FNEG, SL, VT, DAG.getNode(ISD::FNEG, SL, VT,
N1.getOperand(0)), N1.getOperand(0)),
N1.getOperand(1), N0); N1.getOperand(1), N0);
// fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
if (N0.getOpcode() == ISD::FNEG && if (N0.getOpcode() == ISD::FNEG && isContractableFMUL(N0.getOperand(0)) &&
N0.getOperand(0).getOpcode() == ISD::FMUL &&
(Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) { (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) {
SDValue N00 = N0.getOperand(0).getOperand(0); SDValue N00 = N0.getOperand(0).getOperand(0);
SDValue N01 = N0.getOperand(0).getOperand(1); SDValue N01 = N0.getOperand(0).getOperand(1);
@ -8962,12 +8970,12 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
} }
// Look through FP_EXTEND nodes to do more combining. // Look through FP_EXTEND nodes to do more combining.
if (AllowFusion && LookThroughFPExt) { if (LookThroughFPExt) {
// fold (fsub (fpext (fmul x, y)), z) // fold (fsub (fpext (fmul x, y)), z)
// -> (fma (fpext x), (fpext y), (fneg z)) // -> (fma (fpext x), (fpext y), (fneg z))
if (N0.getOpcode() == ISD::FP_EXTEND) { if (N0.getOpcode() == ISD::FP_EXTEND) {
SDValue N00 = N0.getOperand(0); SDValue N00 = N0.getOperand(0);
if (N00.getOpcode() == ISD::FMUL) if (isContractableFMUL(N00))
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
DAG.getNode(ISD::FP_EXTEND, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT,
N00.getOperand(0)), N00.getOperand(0)),
@ -8981,7 +8989,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
// Note: Commutes FSUB operands. // Note: Commutes FSUB operands.
if (N1.getOpcode() == ISD::FP_EXTEND) { if (N1.getOpcode() == ISD::FP_EXTEND) {
SDValue N10 = N1.getOperand(0); SDValue N10 = N1.getOperand(0);
if (N10.getOpcode() == ISD::FMUL) if (isContractableFMUL(N10))
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
DAG.getNode(ISD::FNEG, SL, VT, DAG.getNode(ISD::FNEG, SL, VT,
DAG.getNode(ISD::FP_EXTEND, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT,
@ -9001,7 +9009,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
SDValue N00 = N0.getOperand(0); SDValue N00 = N0.getOperand(0);
if (N00.getOpcode() == ISD::FNEG) { if (N00.getOpcode() == ISD::FNEG) {
SDValue N000 = N00.getOperand(0); SDValue N000 = N00.getOperand(0);
if (N000.getOpcode() == ISD::FMUL) { if (isContractableFMUL(N000)) {
return DAG.getNode(ISD::FNEG, SL, VT, return DAG.getNode(ISD::FNEG, SL, VT,
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
DAG.getNode(ISD::FP_EXTEND, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT,
@ -9023,7 +9031,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
SDValue N00 = N0.getOperand(0); SDValue N00 = N0.getOperand(0);
if (N00.getOpcode() == ISD::FP_EXTEND) { if (N00.getOpcode() == ISD::FP_EXTEND) {
SDValue N000 = N00.getOperand(0); SDValue N000 = N00.getOperand(0);
if (N000.getOpcode() == ISD::FMUL) { if (isContractableFMUL(N000)) {
return DAG.getNode(ISD::FNEG, SL, VT, return DAG.getNode(ISD::FNEG, SL, VT,
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
DAG.getNode(ISD::FP_EXTEND, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT,
@ -9043,10 +9051,9 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
// -> (fma x, y (fma u, v, (fneg z))) // -> (fma x, y (fma u, v, (fneg z)))
// FIXME: The UnsafeAlgebra flag should be propagated to FMA/FMAD, but FMF // FIXME: The UnsafeAlgebra flag should be propagated to FMA/FMAD, but FMF
// are currently only supported on binary nodes. // are currently only supported on binary nodes.
if (Options.UnsafeFPMath && if (Options.UnsafeFPMath && N0.getOpcode() == PreferredFusedOpcode &&
N0.getOpcode() == PreferredFusedOpcode && isContractableFMUL(N0.getOperand(2)) && N0->hasOneUse() &&
N0.getOperand(2).getOpcode() == ISD::FMUL && N0.getOperand(2)->hasOneUse()) {
N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) {
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(0), N0.getOperand(1), N0.getOperand(0), N0.getOperand(1),
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
@ -9060,9 +9067,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
// -> (fma (fneg y), z, (fma (fneg u), v, x)) // -> (fma (fneg y), z, (fma (fneg u), v, x))
// FIXME: The UnsafeAlgebra flag should be propagated to FMA/FMAD, but FMF // FIXME: The UnsafeAlgebra flag should be propagated to FMA/FMAD, but FMF
// are currently only supported on binary nodes. // are currently only supported on binary nodes.
if (Options.UnsafeFPMath && if (Options.UnsafeFPMath && N1.getOpcode() == PreferredFusedOpcode &&
N1.getOpcode() == PreferredFusedOpcode && isContractableFMUL(N1.getOperand(2))) {
N1.getOperand(2).getOpcode() == ISD::FMUL) {
SDValue N20 = N1.getOperand(2).getOperand(0); SDValue N20 = N1.getOperand(2).getOperand(0);
SDValue N21 = N1.getOperand(2).getOperand(1); SDValue N21 = N1.getOperand(2).getOperand(1);
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
@ -9075,14 +9081,14 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
N21, N0)); N21, N0));
} }
if (AllowFusion && LookThroughFPExt) { if (LookThroughFPExt) {
// fold (fsub (fma x, y, (fpext (fmul u, v))), z) // fold (fsub (fma x, y, (fpext (fmul u, v))), z)
// -> (fma x, y (fma (fpext u), (fpext v), (fneg z))) // -> (fma x, y (fma (fpext u), (fpext v), (fneg z)))
if (N0.getOpcode() == PreferredFusedOpcode) { if (N0.getOpcode() == PreferredFusedOpcode) {
SDValue N02 = N0.getOperand(2); SDValue N02 = N0.getOperand(2);
if (N02.getOpcode() == ISD::FP_EXTEND) { if (N02.getOpcode() == ISD::FP_EXTEND) {
SDValue N020 = N02.getOperand(0); SDValue N020 = N02.getOperand(0);
if (N020.getOpcode() == ISD::FMUL) if (isContractableFMUL(N020))
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
N0.getOperand(0), N0.getOperand(1), N0.getOperand(0), N0.getOperand(1),
DAG.getNode(PreferredFusedOpcode, SL, VT, DAG.getNode(PreferredFusedOpcode, SL, VT,
@ -9105,7 +9111,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
SDValue N00 = N0.getOperand(0); SDValue N00 = N0.getOperand(0);
if (N00.getOpcode() == PreferredFusedOpcode) { if (N00.getOpcode() == PreferredFusedOpcode) {
SDValue N002 = N00.getOperand(2); SDValue N002 = N00.getOperand(2);
if (N002.getOpcode() == ISD::FMUL) if (isContractableFMUL(N002))
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
DAG.getNode(ISD::FP_EXTEND, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT,
N00.getOperand(0)), N00.getOperand(0)),
@ -9126,7 +9132,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
if (N1.getOpcode() == PreferredFusedOpcode && if (N1.getOpcode() == PreferredFusedOpcode &&
N1.getOperand(2).getOpcode() == ISD::FP_EXTEND) { N1.getOperand(2).getOpcode() == ISD::FP_EXTEND) {
SDValue N120 = N1.getOperand(2).getOperand(0); SDValue N120 = N1.getOperand(2).getOperand(0);
if (N120.getOpcode() == ISD::FMUL) { if (isContractableFMUL(N120)) {
SDValue N1200 = N120.getOperand(0); SDValue N1200 = N120.getOperand(0);
SDValue N1201 = N120.getOperand(1); SDValue N1201 = N120.getOperand(1);
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,
@ -9153,7 +9159,7 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
SDValue N100 = N1.getOperand(0).getOperand(0); SDValue N100 = N1.getOperand(0).getOperand(0);
SDValue N101 = N1.getOperand(0).getOperand(1); SDValue N101 = N1.getOperand(0).getOperand(1);
SDValue N102 = N1.getOperand(0).getOperand(2); SDValue N102 = N1.getOperand(0).getOperand(2);
if (N102.getOpcode() == ISD::FMUL) { if (isContractableFMUL(N102)) {
SDValue N1020 = N102.getOperand(0); SDValue N1020 = N102.getOperand(0);
SDValue N1021 = N102.getOperand(1); SDValue N1021 = N102.getOperand(1);
return DAG.getNode(PreferredFusedOpcode, SL, VT, return DAG.getNode(PreferredFusedOpcode, SL, VT,

26
test/CodeGen/AArch64/neon-fma-FMF.ll
View File

@ -25,3 +25,29 @@ define <2 x float> @no_fma_2(<2 x float> %A, <2 x float> %B, <2 x float> %C) {
%tmp2 = fadd contract <2 x float> %C, %tmp1; %tmp2 = fadd contract <2 x float> %C, %tmp1;
ret <2 x float> %tmp2 ret <2 x float> %tmp2
} }
define <2 x float> @fma_sub(<2 x float> %A, <2 x float> %B, <2 x float> %C) {
; CHECK-LABEL: fma_sub:
; CHECK: fmls {{v[0-9]+}}.2s, {{v[0-9]+}}.2s, {{v[0-9]+}}.2s
%tmp1 = fmul contract <2 x float> %A, %B;
%tmp2 = fsub contract <2 x float> %C, %tmp1;
ret <2 x float> %tmp2
}
define <2 x float> @no_fma_sub_1(<2 x float> %A, <2 x float> %B, <2 x float> %C) {
; CHECK-LABEL: no_fma_sub_1:
; CHECK: fmul
; CHECK: fsub
%tmp1 = fmul contract <2 x float> %A, %B;
%tmp2 = fsub <2 x float> %C, %tmp1;
ret <2 x float> %tmp2
}
define <2 x float> @no_fma_sub_2(<2 x float> %A, <2 x float> %B, <2 x float> %C) {
; CHECK-LABEL: no_fma_sub_2:
; CHECK: fmul
; CHECK: fsub
%tmp1 = fmul <2 x float> %A, %B;
%tmp2 = fsub contract <2 x float> %C, %tmp1;
ret <2 x float> %tmp2
}