diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td index 9f209e8102b..68af4ae8257 100644 --- a/include/llvm/IR/IntrinsicsARM.td +++ b/include/llvm/IR/IntrinsicsARM.td @@ -1168,4 +1168,14 @@ defm int_arm_mve_vbrsr: MVEMXPredicated< [llvm_anyvector_ty], [], [LLVMMatchType<0>, llvm_i32_ty], LLVMMatchType<0>, llvm_anyvector_ty>; +def int_arm_mve_vqdmull: Intrinsic< + [llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty], + [IntrNoMem]>; +def int_arm_mve_vqdmull_predicated: Intrinsic< + [llvm_anyvector_ty], + [llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty, llvm_anyvector_ty, + LLVMMatchType<0>], + [IntrNoMem]>; + } // end TargetPrefix diff --git a/lib/Target/ARM/ARMInstrMVE.td b/lib/Target/ARM/ARMInstrMVE.td index 735fc0e497c..9a7886d6ecc 100644 --- a/lib/Target/ARM/ARMInstrMVE.td +++ b/lib/Target/ARM/ARMInstrMVE.td @@ -4577,13 +4577,34 @@ class MVE_VQDMULL { - def bh : MVE_VQDMULL<"vqdmullb", suffix, size, 0b0, cstr>; - def th : MVE_VQDMULL<"vqdmullt", suffix, size, 0b1, cstr>; +multiclass MVE_VQDMULL_m { + def "" : MVE_VQDMULL; + defvar Inst = !cast(NAME); + + let Predicates = [HasMVEInt] in { + // Unpredicated saturating multiply + def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm), + (VTI.Vec MQPR:$Qn), (i32 T))), + (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>; + // Predicated saturating multiply + def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated + (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), + (i32 T), (VTI.DblPred VCCR:$mask), + (VTI.DblVec MQPR:$inactive))), + (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), + ARMVCCThen, (VTI.DblPred VCCR:$mask), + (VTI.DblVec MQPR:$inactive)))>; + } } -defm MVE_VQDMULLs16 : MVE_VQDMULL_halves<"s16", 0b0>; -defm MVE_VQDMULLs32 : MVE_VQDMULL_halves<"s32", 0b1, "@earlyclobber $Qd">; +multiclass MVE_VQDMULL_halves { + defm bh : MVE_VQDMULL_m<"vqdmullb", VTI, size, 0b0, cstr>; + defm th : MVE_VQDMULL_m<"vqdmullt", VTI, size, 0b1, cstr>; +} + +defm MVE_VQDMULLs16 : MVE_VQDMULL_halves; +defm MVE_VQDMULLs32 : MVE_VQDMULL_halves; // end of mve_qDest_qSrc @@ -4766,13 +4787,37 @@ class MVE_VQDMULL_qr { - def bh : MVE_VQDMULL_qr<"vqdmullb", suffix, size, 0b0, cstr>; - def th : MVE_VQDMULL_qr<"vqdmullt", suffix, size, 0b1, cstr>; +multiclass MVE_VQDMULL_qr_m { + def "" : MVE_VQDMULL_qr; + defvar Inst = !cast(NAME); + + let Predicates = [HasMVEInt] in { + // Unpredicated saturating multiply + def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm), + (VTI.Vec (ARMvdup rGPR:$val)), + (i32 T))), + (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>; + // Predicated saturating multiply + def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated + (VTI.Vec MQPR:$Qm), + (VTI.Vec (ARMvdup rGPR:$val)), + (i32 T), + (VTI.DblPred VCCR:$mask), + (VTI.DblVec MQPR:$inactive))), + (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val), + ARMVCCThen, (VTI.DblPred VCCR:$mask), + (VTI.DblVec MQPR:$inactive)))>; + } } -defm MVE_VQDMULL_qr_s16 : MVE_VQDMULL_qr_halves<"s16", 0b0>; -defm MVE_VQDMULL_qr_s32 : MVE_VQDMULL_qr_halves<"s32", 0b1, "@earlyclobber $Qd">; +multiclass MVE_VQDMULL_qr_halves { + defm bh : MVE_VQDMULL_qr_m<"vqdmullb", VTI, size, 0b0, cstr>; + defm th : MVE_VQDMULL_qr_m<"vqdmullt", VTI, size, 0b1, cstr>; +} + +defm MVE_VQDMULL_qr_s16 : MVE_VQDMULL_qr_halves; +defm MVE_VQDMULL_qr_s32 : MVE_VQDMULL_qr_halves; class MVE_VxADDSUB_qr bits_21_20, bit subtract, diff --git a/test/CodeGen/Thumb2/mve-intrinsics/vqdmull.ll b/test/CodeGen/Thumb2/mve-intrinsics/vqdmull.ll new file mode 100644 index 00000000000..f05ea7fc2b0 --- /dev/null +++ b/test/CodeGen/Thumb2/mve-intrinsics/vqdmull.ll @@ -0,0 +1,221 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -verify-machineinstrs -o - %s | FileCheck %s + +declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32) + +declare <4 x i32> @llvm.arm.mve.vqdmull.v4i32.v8i16(<8 x i16>, <8 x i16>, i32) +declare <2 x i64> @llvm.arm.mve.vqdmull.v2i64.v4i32(<4 x i32>, <4 x i32>, i32) +declare <4 x i32> @llvm.arm.mve.vqdmull.predicated.v4i32.v8i16.v4i1(<8 x i16>, <8 x i16>, i32, <4 x i1>, <4 x i32>) +declare <2 x i64> @llvm.arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1(<4 x i32>, <4 x i32>, i32, <4 x i1>, <2 x i64>) + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmullbq_s16(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: test_vqdmullbq_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullb.s16 q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %0 = call <4 x i32> @llvm.arm.mve.vqdmull.v4i32.v8i16(<8 x i16> %a, <8 x i16> %b, i32 0) + ret <4 x i32> %0 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmullbq_s32(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: test_vqdmullbq_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullb.s32 q2, q0, q1 +; CHECK-NEXT: vmov q0, q2 +; CHECK-NEXT: bx lr +entry: + %0 = call <2 x i64> @llvm.arm.mve.vqdmull.v2i64.v4i32(<4 x i32> %a, <4 x i32> %b, i32 0) + ret <2 x i64> %0 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmullbq_m_s16(<4 x i32> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmullbq_m_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r0 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmullbt.s16 q0, q1, q2 +; CHECK-NEXT: bx lr +entry: + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <4 x i32> @llvm.arm.mve.vqdmull.predicated.v4i32.v8i16.v4i1(<8 x i16> %a, <8 x i16> %b, i32 0, <4 x i1> %1, <4 x i32> %inactive) + ret <4 x i32> %2 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmullbq_m_s32(<2 x i64> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmullbq_m_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r0 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmullbt.s32 q0, q1, q2 +; CHECK-NEXT: bx lr +entry: + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <2 x i64> @llvm.arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 0, <4 x i1> %1, <2 x i64> %inactive) + ret <2 x i64> %2 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmullbq_n_s16(<8 x i16> %a, i16 signext %b) { +; CHECK-LABEL: test_vqdmullbq_n_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullb.s16 q0, q0, r0 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <8 x i16> undef, i16 %b, i32 0 + %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer + %0 = call <4 x i32> @llvm.arm.mve.vqdmull.v4i32.v8i16(<8 x i16> %a, <8 x i16> %.splat, i32 0) + ret <4 x i32> %0 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmullbq_n_s32(<4 x i32> %a, i32 %b) #0 { +; CHECK-LABEL: test_vqdmullbq_n_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullb.s32 q1, q0, r0 +; CHECK-NEXT: vmov q0, q1 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <4 x i32> undef, i32 %b, i32 0 + %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %0 = call <2 x i64> @llvm.arm.mve.vqdmull.v2i64.v4i32(<4 x i32> %a, <4 x i32> %.splat, i32 0) + ret <2 x i64> %0 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmullbq_m_n_s16(<4 x i32> %inactive, <8 x i16> %a, i16 signext %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmullbq_m_n_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r1 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmullbt.s16 q0, q1, r0 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <8 x i16> undef, i16 %b, i32 0 + %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <4 x i32> @llvm.arm.mve.vqdmull.predicated.v4i32.v8i16.v4i1(<8 x i16> %a, <8 x i16> %.splat, i32 0, <4 x i1> %1, <4 x i32> %inactive) + ret <4 x i32> %2 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmullbq_m_n_s32(<2 x i64> %inactive, <4 x i32> %a, i32 %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmullbq_m_n_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r1 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmullbt.s32 q0, q1, r0 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <4 x i32> undef, i32 %b, i32 0 + %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <2 x i64> @llvm.arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %.splat, i32 0, <4 x i1> %1, <2 x i64> %inactive) + ret <2 x i64> %2 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmulltq_s16(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: test_vqdmulltq_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullt.s16 q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %0 = call <4 x i32> @llvm.arm.mve.vqdmull.v4i32.v8i16(<8 x i16> %a, <8 x i16> %b, i32 1) + ret <4 x i32> %0 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmulltq_s32(<4 x i32> %a, <4 x i32> %b) { +; CHECK-LABEL: test_vqdmulltq_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullt.s32 q2, q0, q1 +; CHECK-NEXT: vmov q0, q2 +; CHECK-NEXT: bx lr +entry: + %0 = call <2 x i64> @llvm.arm.mve.vqdmull.v2i64.v4i32(<4 x i32> %a, <4 x i32> %b, i32 1) + ret <2 x i64> %0 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmulltq_m_s16(<4 x i32> %inactive, <8 x i16> %a, <8 x i16> %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmulltq_m_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r0 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmulltt.s16 q0, q1, q2 +; CHECK-NEXT: bx lr +entry: + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <4 x i32> @llvm.arm.mve.vqdmull.predicated.v4i32.v8i16.v4i1(<8 x i16> %a, <8 x i16> %b, i32 1, <4 x i1> %1, <4 x i32> %inactive) + ret <4 x i32> %2 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmulltq_m_s32(<2 x i64> %inactive, <4 x i32> %a, <4 x i32> %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmulltq_m_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r0 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmulltt.s32 q0, q1, q2 +; CHECK-NEXT: bx lr +entry: + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <2 x i64> @llvm.arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %b, i32 1, <4 x i1> %1, <2 x i64> %inactive) + ret <2 x i64> %2 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmulltq_n_s16(<8 x i16> %a, i16 signext %b) { +; CHECK-LABEL: test_vqdmulltq_n_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullt.s16 q0, q0, r0 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <8 x i16> undef, i16 %b, i32 0 + %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer + %0 = call <4 x i32> @llvm.arm.mve.vqdmull.v4i32.v8i16(<8 x i16> %a, <8 x i16> %.splat, i32 1) + ret <4 x i32> %0 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmulltq_n_s32(<4 x i32> %a, i32 %b) { +; CHECK-LABEL: test_vqdmulltq_n_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vqdmullt.s32 q1, q0, r0 +; CHECK-NEXT: vmov q0, q1 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <4 x i32> undef, i32 %b, i32 0 + %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %0 = call <2 x i64> @llvm.arm.mve.vqdmull.v2i64.v4i32(<4 x i32> %a, <4 x i32> %.splat, i32 1) + ret <2 x i64> %0 +} + +define arm_aapcs_vfpcc <4 x i32> @test_vqdmulltq_m_n_s16(<4 x i32> %inactive, <8 x i16> %a, i16 signext %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmulltq_m_n_s16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r1 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmulltt.s16 q0, q1, r0 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <8 x i16> undef, i16 %b, i32 0 + %.splat = shufflevector <8 x i16> %.splatinsert, <8 x i16> undef, <8 x i32> zeroinitializer + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <4 x i32> @llvm.arm.mve.vqdmull.predicated.v4i32.v8i16.v4i1(<8 x i16> %a, <8 x i16> %.splat, i32 1, <4 x i1> %1, <4 x i32> %inactive) + ret <4 x i32> %2 +} + +define arm_aapcs_vfpcc <2 x i64> @test_vqdmulltq_m_n_s32(<2 x i64> %inactive, <4 x i32> %a, i32 %b, i16 zeroext %p) { +; CHECK-LABEL: test_vqdmulltq_m_n_s32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vmsr p0, r1 +; CHECK-NEXT: vpst +; CHECK-NEXT: vqdmulltt.s32 q0, q1, r0 +; CHECK-NEXT: bx lr +entry: + %.splatinsert = insertelement <4 x i32> undef, i32 %b, i32 0 + %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer + %0 = zext i16 %p to i32 + %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0) + %2 = call <2 x i64> @llvm.arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1(<4 x i32> %a, <4 x i32> %.splat, i32 1, <4 x i1> %1, <2 x i64> %inactive) + ret <2 x i64> %2 +}