diff --git a/test/Transforms/LoopVectorize/float-induction.ll b/test/Transforms/LoopVectorize/float-induction.ll index 837f41d753f..3a9d1bbf68b 100644 --- a/test/Transforms/LoopVectorize/float-induction.ll +++ b/test/Transforms/LoopVectorize/float-induction.ll @@ -3,43 +3,6 @@ ; RUN: opt < %s -loop-vectorize -force-vector-interleave=2 -force-vector-width=1 -dce -instcombine -S | FileCheck --check-prefix VEC1_INTERL2 %s ; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -dce -simplifycfg -instcombine -S | FileCheck --check-prefix VEC2_INTERL1_PRED_STORE %s -; VEC4_INTERL1-LABEL: @fp_iv_loop1( -; VEC4_INTERL1: %[[FP_INC:.*]] = load float, float* @fp_inc -; VEC4_INTERL1: vector.body: -; VEC4_INTERL1: %[[FP_INDEX:.*]] = sitofp i64 {{.*}} to float -; VEC4_INTERL1: %[[VEC_INCR:.*]] = fmul fast float {{.*}}, %[[FP_INDEX]] -; VEC4_INTERL1: %[[FP_OFFSET_IDX:.*]] = fsub fast float %init, %[[VEC_INCR]] -; VEC4_INTERL1: %[[BRCT_INSERT:.*]] = insertelement <4 x float> undef, float %[[FP_OFFSET_IDX]], i32 0 -; VEC4_INTERL1-NEXT: %[[BRCT_SPLAT:.*]] = shufflevector <4 x float> %[[BRCT_INSERT]], <4 x float> undef, <4 x i32> zeroinitializer -; VEC4_INTERL1: %[[BRCT_INSERT:.*]] = insertelement {{.*}} %[[FP_INC]] -; VEC4_INTERL1-NEXT: %[[FP_INC_BCST:.*]] = shufflevector <4 x float> %[[BRCT_INSERT]], {{.*}} zeroinitializer -; VEC4_INTERL1: %[[VSTEP:.*]] = fmul fast <4 x float> %[[FP_INC_BCST]], -; VEC4_INTERL1-NEXT: %[[VEC_INDUCTION:.*]] = fsub fast <4 x float> %[[BRCT_SPLAT]], %[[VSTEP]] -; VEC4_INTERL1: store <4 x float> %[[VEC_INDUCTION]] - -; VEC4_INTERL2-LABEL: @fp_iv_loop1( -; VEC4_INTERL2: %[[FP_INC:.*]] = load float, float* @fp_inc -; VEC4_INTERL2: vector.body: -; VEC4_INTERL2: %[[INDEX:.*]] = sitofp i64 {{.*}} to float -; VEC4_INTERL2: %[[VEC_INCR:.*]] = fmul fast float %{{.*}}, %[[INDEX]] -; VEC4_INTERL2: fsub fast float %init, %[[VEC_INCR]] -; VEC4_INTERL2: %[[VSTEP1:.*]] = fmul fast <4 x float> %{{.*}}, -; VEC4_INTERL2-NEXT: %[[VEC_INDUCTION1:.*]] = fsub fast <4 x float> {{.*}}, %[[VSTEP1]] -; VEC4_INTERL2: %[[VSTEP2:.*]] = fmul fast <4 x float> %{{.*}}, -; VEC4_INTERL2-NEXT: %[[VEC_INDUCTION2:.*]] = fsub fast <4 x float> {{.*}}, %[[VSTEP2]] -; VEC4_INTERL2: store <4 x float> %[[VEC_INDUCTION1]] -; VEC4_INTERL2: store <4 x float> %[[VEC_INDUCTION2]] - -; VEC1_INTERL2-LABEL: @fp_iv_loop1( -; VEC1_INTERL2: %[[FP_INC:.*]] = load float, float* @fp_inc -; VEC1_INTERL2: vector.body: -; VEC1_INTERL2: %[[INDEX:.*]] = sitofp i64 {{.*}} to float -; VEC1_INTERL2: %[[STEP:.*]] = fmul fast float %{{.*}}, %[[INDEX]] -; VEC1_INTERL2: %[[FP_OFFSET_IDX:.*]] = fsub fast float %init, %[[STEP]] -; VEC1_INTERL2: %[[SCALAR_INDUCTION2:.*]] = fsub fast float %[[FP_OFFSET_IDX]], %[[FP_INC]] -; VEC1_INTERL2: store float %[[FP_OFFSET_IDX]] -; VEC1_INTERL2: store float %[[SCALAR_INDUCTION2]] - @fp_inc = common global float 0.000000e+00, align 4 ;void fp_iv_loop1(float init, float * __restrict__ A, int N) { @@ -50,6 +13,68 @@ ; } ;} +; VEC4_INTERL1-LABEL: @fp_iv_loop1( +; VEC4_INTERL1: vector.ph: +; VEC4_INTERL1-NEXT: br label %vector.body +; VEC4_INTERL1: vector.body: +; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = sitofp i64 [[INDEX]] to float +; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fmul fast float %fpinc, [[TMP5]] +; VEC4_INTERL1-NEXT: [[FP_OFFSET_IDX:%.*]] = fsub fast float %init, [[TMP6]] +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> undef, float [[FP_OFFSET_IDX]], i32 0 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float %fpinc, i32 0 +; VEC4_INTERL1-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[DOTSPLAT]], +; VEC4_INTERL1-NEXT: [[INDUCTION5:%.*]] = fsub fast <4 x float> [[BROADCAST_SPLAT4]], [[TMP7]] +; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]] +; VEC4_INTERL1-NEXT: [[TMP9:%.*]] = bitcast float* [[TMP8]] to <4 x float>* +; VEC4_INTERL1-NEXT: store <4 x float> [[INDUCTION5]], <4 x float>* [[TMP9]], align 4 +; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; VEC4_INTERL1: br i1 {{.*}}, label %middle.block, label %vector.body + +; VEC4_INTERL2-LABEL: @fp_iv_loop1( +; VEC4_INTERL2: vector.ph: +; VEC4_INTERL2-NEXT: br label %vector.body +; VEC4_INTERL2: vector.body: +; VEC4_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; VEC4_INTERL2-NEXT: [[TMP5:%.*]] = sitofp i64 [[INDEX]] to float +; VEC4_INTERL2-NEXT: [[TMP6:%.*]] = fmul fast float %fpinc, [[TMP5]] +; VEC4_INTERL2-NEXT: [[FP_OFFSET_IDX:%.*]] = fsub fast float %init, [[TMP6]] +; VEC4_INTERL2-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x float> undef, float [[FP_OFFSET_IDX]], i32 0 +; VEC4_INTERL2-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT4]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL2-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float %fpinc, i32 0 +; VEC4_INTERL2-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[DOTSPLAT]], +; VEC4_INTERL2-NEXT: [[INDUCTION6:%.*]] = fsub fast <4 x float> [[BROADCAST_SPLAT5]], [[TMP7]] +; VEC4_INTERL2-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[DOTSPLAT]], +; VEC4_INTERL2-NEXT: [[INDUCTION9:%.*]] = fsub fast <4 x float> [[BROADCAST_SPLAT5]], [[TMP8]] +; VEC4_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]] +; VEC4_INTERL2-NEXT: [[TMP10:%.*]] = bitcast float* [[TMP9]] to <4 x float>* +; VEC4_INTERL2-NEXT: store <4 x float> [[INDUCTION6]], <4 x float>* [[TMP10]], align 4 +; VEC4_INTERL2-NEXT: [[TMP11:%.*]] = getelementptr float, float* [[TMP9]], i64 4 +; VEC4_INTERL2-NEXT: [[TMP12:%.*]] = bitcast float* [[TMP11]] to <4 x float>* +; VEC4_INTERL2-NEXT: store <4 x float> [[INDUCTION9]], <4 x float>* [[TMP12]], align 4 +; VEC4_INTERL2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8 +; VEC4_INTERL2: br i1 {{.*}}, label %middle.block, label %vector.body + +; VEC1_INTERL2-LABEL: @fp_iv_loop1( +; VEC1_INTERL2: vector.ph: +; VEC1_INTERL2-NEXT: br label %vector.body +; VEC1_INTERL2: vector.body: +; VEC1_INTERL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; VEC1_INTERL2-NEXT: [[INDUCTION2:%.*]] = or i64 [[INDEX]], 1 +; VEC1_INTERL2-NEXT: [[TMP6:%.*]] = sitofp i64 [[INDEX]] to float +; VEC1_INTERL2-NEXT: [[TMP7:%.*]] = fmul fast float %fpinc, [[TMP6]] +; VEC1_INTERL2-NEXT: [[FP_OFFSET_IDX:%.*]] = fsub fast float %init, [[TMP7]] +; VEC1_INTERL2-NEXT: [[TMP8:%.*]] = fsub fast float [[FP_OFFSET_IDX]], %fpinc +; VEC1_INTERL2-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]] +; VEC1_INTERL2-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDUCTION2]] +; VEC1_INTERL2-NEXT: store float [[FP_OFFSET_IDX]], float* [[TMP9]], align 4 +; VEC1_INTERL2-NEXT: store float [[TMP8]], float* [[TMP10]], align 4 +; VEC1_INTERL2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 2 +; VEC1_INTERL2: br i1 {{.*}}, label %middle.block, label %vector.body + define void @fp_iv_loop1(float %init, float* noalias nocapture %A, i32 %N) #1 { entry: %cmp4 = icmp sgt i32 %N, 0 @@ -86,15 +111,21 @@ for.end: ; preds = %for.end.loopexit, % ;} ; VEC4_INTERL1-LABEL: @fp_iv_loop2( -; VEC4_INTERL1: vector.body -; VEC4_INTERL1: %[[index:.*]] = phi i64 [ 0, %vector.ph ] -; VEC4_INTERL1: sitofp i64 %[[index]] to float -; VEC4_INTERL1: %[[VAR1:.*]] = fmul fast float {{.*}}, 5.000000e-01 -; VEC4_INTERL1: %[[VAR2:.*]] = fadd fast float %[[VAR1]] -; VEC4_INTERL1: insertelement <4 x float> undef, float %[[VAR2]], i32 0 -; VEC4_INTERL1: shufflevector <4 x float> {{.*}}, <4 x float> undef, <4 x i32> zeroinitializer -; VEC4_INTERL1: fadd fast <4 x float> {{.*}}, -; VEC4_INTERL1: store <4 x float> +; VEC4_INTERL1: vector.ph: +; VEC4_INTERL1-NEXT: br label %vector.body +; VEC4_INTERL1: vector.body: +; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = sitofp i64 [[INDEX]] to float +; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP5]], 5.000000e-01 +; VEC4_INTERL1-NEXT: [[FP_OFFSET_IDX:%.*]] = fadd fast float [[TMP6]], %init +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> undef, float [[FP_OFFSET_IDX]], i32 0 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[INDUCTION5:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT4]], +; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]] +; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = bitcast float* [[TMP7]] to <4 x float>* +; VEC4_INTERL1-NEXT: store <4 x float> [[INDUCTION5]], <4 x float>* [[TMP8]], align 4 +; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; VEC4_INTERL1: br i1 {{.*}}, label %middle.block, label %vector.body define void @fp_iv_loop2(float %init, float* noalias nocapture %A, i32 %N) #0 { entry: @@ -134,14 +165,44 @@ for.end: ; preds = %for.end.loopexit, % ; C[i] = y; ; } ;} + ; VEC4_INTERL1-LABEL: @fp_iv_loop3( -; VEC4_INTERL1: vector.body -; VEC4_INTERL1: %[[index:.*]] = phi i64 [ 0, %vector.ph ] -; VEC4_INTERL1: sitofp i64 %[[index]] to float -; VEC4_INTERL1: %[[VAR1:.*]] = fmul fast float {{.*}}, -5.000000e-01 -; VEC4_INTERL1: fadd fast float %[[VAR1]] -; VEC4_INTERL1: fadd fast <4 x float> {{.*}}, -; VEC4_INTERL1: store <4 x float> +; VEC4_INTERL1: for.body.lr.ph: +; VEC4_INTERL1: [[TMP0:%.*]] = load float, float* @fp_inc, align 4 +; VEC4_INTERL1: vector.ph: +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT14:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i32 0 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT15:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT14]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: br label %vector.body +; VEC4_INTERL1: vector.body: +; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = sitofp i64 [[INDEX]] to float +; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = fmul fast float [[TMP7]], -5.000000e-01 +; VEC4_INTERL1-NEXT: [[FP_OFFSET_IDX:%.*]] = fadd fast float [[TMP8]], 0x3FB99999A0000000 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x float> undef, float [[FP_OFFSET_IDX]], i32 0 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT6]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[TMP9:%.*]] = sitofp i64 [[INDEX]] to float +; VEC4_INTERL1-NEXT: [[TMP10:%.*]] = fmul fast float [[TMP0]], [[TMP9]] +; VEC4_INTERL1-NEXT: [[FP_OFFSET_IDX10:%.*]] = fadd fast float [[TMP10]], %init +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x float> undef, float [[FP_OFFSET_IDX10]], i32 0 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT11]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> undef, float [[TMP0]], i32 0 +; VEC4_INTERL1-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[TMP11:%.*]] = fmul fast <4 x float> [[DOTSPLAT]], +; VEC4_INTERL1-NEXT: [[INDUCTION13:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT12]], [[TMP11]] +; VEC4_INTERL1-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]] +; VEC4_INTERL1-NEXT: [[TMP13:%.*]] = bitcast float* [[TMP12]] to <4 x float>* +; VEC4_INTERL1-NEXT: store <4 x float> [[INDUCTION13]], <4 x float>* [[TMP13]], align 4 +; VEC4_INTERL1-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[INDUCTION13]], [[BROADCAST_SPLAT15]] +; VEC4_INTERL1-NEXT: [[TMP15:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT7]], +; VEC4_INTERL1-NEXT: [[TMP16:%.*]] = fadd fast <4 x float> [[TMP15]], [[TMP14]] +; VEC4_INTERL1-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, float* %B, i64 [[INDEX]] +; VEC4_INTERL1-NEXT: [[TMP18:%.*]] = bitcast float* [[TMP17]] to <4 x float>* +; VEC4_INTERL1-NEXT: store <4 x float> [[TMP16]], <4 x float>* [[TMP18]], align 4 +; VEC4_INTERL1-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, float* %C, i64 [[INDEX]] +; VEC4_INTERL1-NEXT: [[TMP20:%.*]] = bitcast float* [[TMP19]] to <4 x float>* +; VEC4_INTERL1-NEXT: store <4 x float> [[TMP15]], <4 x float>* [[TMP20]], align 4 +; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; VEC4_INTERL1: br i1 {{.*}}, label %middle.block, label %vector.body define void @fp_iv_loop3(float %init, float* noalias nocapture %A, float* noalias nocapture %B, float* noalias nocapture %C, i32 %N) #1 { entry: @@ -187,10 +248,21 @@ for.end: ;} ; VEC4_INTERL1-LABEL: @fp_iv_loop4( -; VEC4_INTERL1: vector.body -; VEC4_INTERL1-NOT: fmul fast <4 x float> -; VEC4_INTERL1: %[[induction:.*]] = fadd fast <4 x float> %{{.*}}, -; VEC4_INTERL1: store <4 x float> %[[induction]] +; VEC4_INTERL1: vector.ph: +; VEC4_INTERL1-NEXT: br label %vector.body +; VEC4_INTERL1: vector.body: +; VEC4_INTERL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] +; VEC4_INTERL1-NEXT: [[TMP5:%.*]] = sitofp i64 [[INDEX]] to float +; VEC4_INTERL1-NEXT: [[TMP6:%.*]] = fmul fast float [[TMP5]], 5.000000e-01 +; VEC4_INTERL1-NEXT: [[FP_OFFSET_IDX:%.*]] = fadd fast float [[TMP6]], 1.000000e+00 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x float> undef, float [[FP_OFFSET_IDX]], i32 0 +; VEC4_INTERL1-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT3]], <4 x float> undef, <4 x i32> zeroinitializer +; VEC4_INTERL1-NEXT: [[INDUCTION5:%.*]] = fadd fast <4 x float> [[BROADCAST_SPLAT4]], +; VEC4_INTERL1-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, float* %A, i64 [[INDEX]] +; VEC4_INTERL1-NEXT: [[TMP8:%.*]] = bitcast float* [[TMP7]] to <4 x float>* +; VEC4_INTERL1-NEXT: store <4 x float> [[INDUCTION5]], <4 x float>* [[TMP8]], align 4 +; VEC4_INTERL1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 +; VEC4_INTERL1: br i1 {{.*}}, label %middle.block, label %vector.body define void @fp_iv_loop4(float* noalias nocapture %A, i32 %N) { entry: