diff --git a/lib/Target/ARM/ARM.h b/lib/Target/ARM/ARM.h index 9076c191d83..3412813a3ef 100644 --- a/lib/Target/ARM/ARM.h +++ b/lib/Target/ARM/ARM.h @@ -53,6 +53,7 @@ FunctionPass *createThumb2SizeReductionPass( InstructionSelector * createARMInstructionSelector(const ARMBaseTargetMachine &TM, const ARMSubtarget &STI, const ARMRegisterBankInfo &RBI); +Pass *createMVEGatherScatterLoweringPass(); void LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, ARMAsmPrinter &AP); @@ -67,6 +68,7 @@ void initializeThumb2ITBlockPass(PassRegistry &); void initializeMVEVPTBlockPass(PassRegistry &); void initializeARMLowOverheadLoopsPass(PassRegistry &); void initializeMVETailPredicationPass(PassRegistry &); +void initializeMVEGatherScatterLoweringPass(PassRegistry &); } // end namespace llvm diff --git a/lib/Target/ARM/ARMTargetMachine.cpp b/lib/Target/ARM/ARMTargetMachine.cpp index 018ce3903c2..a48f351f37a 100644 --- a/lib/Target/ARM/ARMTargetMachine.cpp +++ b/lib/Target/ARM/ARMTargetMachine.cpp @@ -98,6 +98,7 @@ extern "C" void LLVMInitializeARMTarget() { initializeMVEVPTBlockPass(Registry); initializeMVETailPredicationPass(Registry); initializeARMLowOverheadLoopsPass(Registry); + initializeMVEGatherScatterLoweringPass(Registry); } static std::unique_ptr createTLOF(const Triple &TT) { @@ -404,6 +405,8 @@ void ARMPassConfig::addIRPasses() { return ST.hasAnyDataBarrier() && !ST.isThumb1Only(); })); + addPass(createMVEGatherScatterLoweringPass()); + TargetPassConfig::addIRPasses(); // Run the parallel DSP pass. diff --git a/lib/Target/ARM/ARMTargetTransformInfo.cpp b/lib/Target/ARM/ARMTargetTransformInfo.cpp index e4b77ae56a4..41ad8b0c04d 100644 --- a/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -22,6 +22,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" #include "llvm/MC/SubtargetFeature.h" #include "llvm/Support/Casting.h" @@ -46,6 +47,8 @@ static cl::opt DisableLowOverheadLoops( extern cl::opt DisableTailPredication; +extern cl::opt EnableMaskedGatherScatters; + bool ARMTTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { const TargetMachine &TM = getTLI()->getTargetMachine(); @@ -514,6 +517,27 @@ bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) { (EltWidth == 8); } +bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) { + if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps()) + return false; + + // This method is called in 2 places: + // - from the vectorizer with a scalar type, in which case we need to get + // this as good as we can with the limited info we have (and rely on the cost + // model for the rest). + // - from the masked intrinsic lowering pass with the actual vector type. + // For MVE, we have a custom lowering pass that will already have custom + // legalised any gathers that we can to MVE intrinsics, and want to expand all + // the rest. The pass runs before the masked intrinsic lowering pass, so if we + // are here, we know we want to expand. + if (isa(Ty)) + return false; + + unsigned EltWidth = Ty->getScalarSizeInBits(); + return ((EltWidth == 32 && (!Alignment || Alignment >= 4)) || + (EltWidth == 16 && (!Alignment || Alignment >= 2)) || EltWidth == 8); +} + int ARMTTIImpl::getMemcpyCost(const Instruction *I) { const MemCpyInst *MI = dyn_cast(I); assert(MI && "MemcpyInst expected"); diff --git a/lib/Target/ARM/ARMTargetTransformInfo.h b/lib/Target/ARM/ARMTargetTransformInfo.h index 6888c8924fc..880588adfdf 100644 --- a/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/lib/Target/ARM/ARMTargetTransformInfo.h @@ -159,7 +159,7 @@ public: return isLegalMaskedLoad(DataTy, Alignment); } - bool isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) { return false; } + bool isLegalMaskedGather(Type *Ty, MaybeAlign Alignment); bool isLegalMaskedScatter(Type *Ty, MaybeAlign Alignment) { return false; } diff --git a/lib/Target/ARM/CMakeLists.txt b/lib/Target/ARM/CMakeLists.txt index b94a78ea940..7591701857b 100644 --- a/lib/Target/ARM/CMakeLists.txt +++ b/lib/Target/ARM/CMakeLists.txt @@ -51,6 +51,7 @@ add_llvm_target(ARMCodeGen ARMTargetObjectFile.cpp ARMTargetTransformInfo.cpp MLxExpansionPass.cpp + MVEGatherScatterLowering.cpp MVETailPredication.cpp MVEVPTBlockPass.cpp Thumb1FrameLowering.cpp diff --git a/lib/Target/ARM/MVEGatherScatterLowering.cpp b/lib/Target/ARM/MVEGatherScatterLowering.cpp new file mode 100644 index 00000000000..4657a043dba --- /dev/null +++ b/lib/Target/ARM/MVEGatherScatterLowering.cpp @@ -0,0 +1,177 @@ +//===- ARMGatherScatterLowering.cpp - Gather/Scatter lowering -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// This pass custom lowers llvm.gather and llvm.scatter instructions to +/// arm.mve.gather and arm.mve.scatter intrinsics, optimising the code to +/// produce a better final result as we go. +// +//===----------------------------------------------------------------------===// + +#include "ARM.h" +#include "ARMBaseInstrInfo.h" +#include "ARMSubtarget.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsARM.h" +#include "llvm/IR/PatternMatch.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/Casting.h" +#include +#include + +using namespace llvm; + +#define DEBUG_TYPE "mve-gather-scatter-lowering" + +cl::opt EnableMaskedGatherScatters( + "enable-arm-maskedgatscat", cl::Hidden, cl::init(false), + cl::desc("Enable the generation of masked gathers and scatters")); + +namespace { + +class MVEGatherScatterLowering : public FunctionPass { +public: + static char ID; // Pass identification, replacement for typeid + + explicit MVEGatherScatterLowering() : FunctionPass(ID) { + initializeMVEGatherScatterLoweringPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + + StringRef getPassName() const override { + return "MVE gather/scatter lowering"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + FunctionPass::getAnalysisUsage(AU); + } +}; + +} // end anonymous namespace + +char MVEGatherScatterLowering::ID = 0; + +INITIALIZE_PASS(MVEGatherScatterLowering, DEBUG_TYPE, + "MVE gather/scattering lowering pass", false, false) + +Pass *llvm::createMVEGatherScatterLoweringPass() { + return new MVEGatherScatterLowering(); +} + +static bool isLegalTypeAndAlignment(unsigned NumElements, unsigned ElemSize, + unsigned Alignment) { + // Do only allow non-extending v4i32 gathers for now + return NumElements == 4 && ElemSize == 32 && Alignment >= 4; +} + +static bool LowerGather(IntrinsicInst *I) { + using namespace PatternMatch; + LLVM_DEBUG(dbgs() << "masked gathers: checking transform preconditions\n"); + + // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) + // Attempt to turn the masked gather in I into a MVE intrinsic + // Potentially optimising the addressing modes as we do so. + Type *Ty = I->getType(); + Value *Ptr = I->getArgOperand(0); + unsigned Alignment = cast(I->getArgOperand(1))->getZExtValue(); + Value *Mask = I->getArgOperand(2); + Value *PassThru = I->getArgOperand(3); + + // Check this is a valid gather with correct alignment + if (!isLegalTypeAndAlignment(Ty->getVectorNumElements(), + Ty->getScalarSizeInBits(), Alignment)) { + LLVM_DEBUG(dbgs() << "masked gathers: instruction does not have valid " + << "alignment or vector type \n"); + return false; + } + + IRBuilder<> Builder(I->getContext()); + Builder.SetInsertPoint(I); + Builder.SetCurrentDebugLocation(I->getDebugLoc()); + + Value *Load = nullptr; + // Look through bitcast instruction if #elements is the same + if (auto *BitCast = dyn_cast(Ptr)) { + Type *BCTy = BitCast->getType(); + Type *BCSrcTy = BitCast->getOperand(0)->getType(); + if (BCTy->getVectorNumElements() == BCSrcTy->getVectorNumElements()) { + LLVM_DEBUG(dbgs() << "masked gathers: looking through bitcast\n"); + Ptr = BitCast->getOperand(0); + } + } + assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type"); + + if (Ty->getVectorNumElements() != 4) + // Can't build an intrinsic for this + return false; + if (match(Mask, m_One())) + Load = Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base, + {Ty, Ptr->getType()}, + {Ptr, Builder.getInt32(0)}); + else + Load = + Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base_predicated, + {Ty, Ptr->getType(), Mask->getType()}, + {Ptr, Builder.getInt32(0), Mask}); + + if (!isa(PassThru) && !match(PassThru, m_Zero())) { + LLVM_DEBUG(dbgs() << "masked gathers: found non-trivial passthru - " + << "creating select\n"); + Load = Builder.CreateSelect(Mask, Load, PassThru); + } + + LLVM_DEBUG(dbgs() << "masked gathers: successfully built masked gather\n"); + I->replaceAllUsesWith(Load); + I->eraseFromParent(); + return true; +} + +bool MVEGatherScatterLowering::runOnFunction(Function &F) { + if (!EnableMaskedGatherScatters) + return false; + auto &TPC = getAnalysis(); + auto &TM = TPC.getTM(); + auto *ST = &TM.getSubtarget(F); + if (!ST->hasMVEIntegerOps()) + return false; + SmallVector Gathers; + for (BasicBlock &BB : F) { + for (Instruction &I : BB) { + IntrinsicInst *II = dyn_cast(&I); + if (II && II->getIntrinsicID() == Intrinsic::masked_gather) + Gathers.push_back(II); + } + } + + if (Gathers.empty()) + return false; + + for (IntrinsicInst *I : Gathers) + LowerGather(I); + + return true; +} diff --git a/test/CodeGen/ARM/O3-pipeline.ll b/test/CodeGen/ARM/O3-pipeline.ll index 3e9431403af..42bad7c7487 100644 --- a/test/CodeGen/ARM/O3-pipeline.ll +++ b/test/CodeGen/ARM/O3-pipeline.ll @@ -7,6 +7,7 @@ ; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Expand Atomic instructions ; CHECK-NEXT: Simplify the CFG +; CHECK-NEXT: MVE gather/scatter lowering ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) ; CHECK-NEXT: Module Verifier diff --git a/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll b/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll new file mode 100644 index 00000000000..f75379160d7 --- /dev/null +++ b/test/CodeGen/Thumb2/mve-gather-ind32-scaled.ll @@ -0,0 +1,461 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck %s + +define arm_aapcs_vfpcc <4 x i32> @zext_scaled_i16_i32(i16* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: zext_scaled_i16_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_scaled_i16_i32(i16* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: sext_scaled_i16_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @scaled_i32_i32(i32* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: scaled_i32_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +; TODO: scaled_f16_i32 + +define arm_aapcs_vfpcc <4 x float> @scaled_f32_i32(i32* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: scaled_f32_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs + %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @unsigned_scaled_b_i32_i16(i32* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: unsigned_scaled_b_i32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @signed_scaled_i32_i16(i32* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: signed_scaled_i32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x float> @a_unsigned_scaled_f32_i16(i32* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: a_unsigned_scaled_f32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x float> @b_signed_scaled_f32_i16(i32* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: b_signed_scaled_f32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @zext_signed_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: zext_signed_scaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_signed_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: sext_signed_scaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: zext_unsigned_scaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_scaled_i16_i16(i16* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: sext_unsigned_scaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @unsigned_scaled_b_i32_i8(i32* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: unsigned_scaled_b_i32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @signed_scaled_i32_i8(i32* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: signed_scaled_i32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x float> @a_unsigned_scaled_f32_i8(i32* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: a_unsigned_scaled_f32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x float> @b_signed_scaled_f32_i8(i32* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: b_signed_scaled_f32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #2 +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %i32_ptrs = getelementptr inbounds i32, i32* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i32*> %i32_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @zext_signed_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: zext_signed_scaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_signed_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: sext_signed_scaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.sext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: zext_unsigned_scaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_scaled_i16_i8(i16* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: sext_unsigned_scaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vshl.i32 q0, q0, #1 +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i16, i16* %base, <4 x i32> %offs.zext + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>) +declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>) +declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>) +declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>) +declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>) diff --git a/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll b/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll new file mode 100644 index 00000000000..fcb7dd8f87d --- /dev/null +++ b/test/CodeGen/Thumb2/mve-gather-ind32-unscaled.ll @@ -0,0 +1,742 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck %s + +define arm_aapcs_vfpcc <4 x i32> @zext_unscaled_i8_i32(i8* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: zext_unscaled_i8_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r0 +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.zext = zext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unscaled_i8_i32(i8* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: sext_unscaled_i8_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r0, s0 +; CHECK-NEXT: vmov r3, s1 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[0], r0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[1], r3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q0[2], r1 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.sext = sext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unscaled_i16_i32(i8* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: zext_unscaled_i16_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unscaled_i16_i32(i8* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: sext_unscaled_i16_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @unscaled_i32_i32(i8* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: unscaled_i32_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*> + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x float> @unscaled_f32_i32(i8* %base, <4 x i32>* %offptr) { +; CHECK-LABEL: unscaled_f32_i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @unsigned_unscaled_b_i32_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: unsigned_unscaled_b_i32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*> + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @signed_unscaled_i32_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: signed_unscaled_i32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*> + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x float> @a_unsigned_unscaled_f32_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: a_unsigned_unscaled_f32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x float> @b_signed_unscaled_f32_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: b_signed_unscaled_f32_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: zext_signed_unscaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: sext_signed_unscaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: zext_unsigned_unscaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i16_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: sext_unsigned_unscaled_i16_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: zext_signed_unscaled_i8_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r0 +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.zext = zext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: sext_signed_unscaled_i8_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r0, s0 +; CHECK-NEXT: vmov r3, s1 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[0], r0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[1], r3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q0[2], r1 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.sext = sext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.sext = sext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: zext_unsigned_unscaled_i8_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r0 +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.zext = zext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i8_i16(i8* %base, <4 x i16>* %offptr) { +; CHECK-LABEL: sext_unsigned_unscaled_i8_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrh.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r0, s0 +; CHECK-NEXT: vmov r3, s1 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[0], r0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[1], r3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q0[2], r1 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16>, <4 x i16>* %offptr, align 2 + %offs.zext = zext <4 x i16> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.sext = sext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @unsigned_unscaled_b_i32_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: unsigned_unscaled_b_i32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*> + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @signed_unscaled_i32_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: signed_unscaled_i32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*> + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x float> @a_unsigned_unscaled_f32_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: a_unsigned_unscaled_f32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x float> @b_signed_unscaled_f32_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: b_signed_unscaled_f32_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q1, q0, r0 +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x float*> + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %ptrs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: zext_signed_unscaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: sext_signed_unscaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: zext_unsigned_unscaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.zext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i16_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: sext_unsigned_unscaled_i16_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i16*> + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %ptrs, i32 2, <4 x i1> , <4 x i16> undef) + %gather.sext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_signed_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: zext_signed_unscaled_i8_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r0 +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.zext = zext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_signed_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: sext_signed_unscaled_i8_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.s32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r0, s0 +; CHECK-NEXT: vmov r3, s1 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[0], r0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[1], r3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q0[2], r1 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.sext = sext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.sext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.sext = sext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +define arm_aapcs_vfpcc <4 x i32> @zext_unsigned_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: zext_unsigned_unscaled_i8_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r0 +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.zext = zext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.zext +} + +define arm_aapcs_vfpcc <4 x i32> @sext_unsigned_unscaled_i8_i8(i8* %base, <4 x i8>* %offptr) { +; CHECK-LABEL: sext_unsigned_unscaled_i8_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrb.u32 q0, [r1] +; CHECK-NEXT: vadd.i32 q0, q0, r0 +; CHECK-NEXT: vmov r0, s0 +; CHECK-NEXT: vmov r3, s1 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[0], r0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[1], r3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q0[2], r1 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8>, <4 x i8>* %offptr, align 1 + %offs.zext = zext <4 x i8> %offs to <4 x i32> + %ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs.zext + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %ptrs, i32 1, <4 x i1> , <4 x i8> undef) + %gather.sext = sext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %gather.sext +} + +declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>) +declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>) +declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>) +declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>) +declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>) diff --git a/test/CodeGen/Thumb2/mve-gather-ptrs.ll b/test/CodeGen/Thumb2/mve-gather-ptrs.ll new file mode 100644 index 00000000000..b595a20bf49 --- /dev/null +++ b/test/CodeGen/Thumb2/mve-gather-ptrs.ll @@ -0,0 +1,850 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -enable-arm-maskedldst -enable-arm-maskedgatscat %s -o - | FileCheck %s + +; i32 + +define arm_aapcs_vfpcc <2 x i32> @ptr_v2i32(<2 x i32*>* %offptr) { +; CHECK-LABEL: ptr_v2i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: ldrd r1, r0, [r0] +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: ldr r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r1 +; CHECK-NEXT: vmov.32 q0[2], r0 +; CHECK-NEXT: bx lr +entry: + %offs = load <2 x i32*>, <2 x i32*>* %offptr, align 4 + %gather = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %offs, i32 4, <2 x i1> , <2 x i32> undef) + ret <2 x i32> %gather +} + +define arm_aapcs_vfpcc <4 x i32> @ptr_v4i32(<4 x i32*>* %offptr) { +; CHECK-LABEL: ptr_v4i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q1, [r0] +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i32*>, <4 x i32*>* %offptr, align 4 + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %offs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +define arm_aapcs_vfpcc <8 x i32> @ptr_v8i32(<8 x i32*>* %offptr) { +; CHECK-LABEL: ptr_v8i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r3, s0 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r5, s1 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldr.w r12, [r1] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: ldr.w lr, [r2] +; CHECK-NEXT: vmov r2, s6 +; CHECK-NEXT: ldr r3, [r3] +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: ldr r5, [r5] +; CHECK-NEXT: vmov.32 q0[0], r3 +; CHECK-NEXT: vmov.32 q1[0], r0 +; CHECK-NEXT: ldr r4, [r4] +; CHECK-NEXT: vmov.32 q0[1], r5 +; CHECK-NEXT: vmov.32 q0[2], r12 +; CHECK-NEXT: vmov.32 q0[3], lr +; CHECK-NEXT: ldr r1, [r1] +; CHECK-NEXT: ldr r2, [r2] +; CHECK-NEXT: vmov.32 q1[1], r1 +; CHECK-NEXT: vmov.32 q1[2], r2 +; CHECK-NEXT: vmov.32 q1[3], r4 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i32*>, <8 x i32*>* %offptr, align 4 + %gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> %offs, i32 4, <8 x i1> , <8 x i32> undef) + ret <8 x i32> %gather +} + +define arm_aapcs_vfpcc <16 x i32> @ptr_v16i32(<16 x i32*>* %offptr) { +; CHECK-LABEL: ptr_v16i32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r6, lr} +; CHECK-NEXT: push {r4, r5, r6, lr} +; CHECK-NEXT: vldrw.u32 q2, [r0, #32] +; CHECK-NEXT: vldrw.u32 q0, [r0, #48] +; CHECK-NEXT: vldrw.u32 q1, [r0] +; CHECK-NEXT: vldrw.u32 q3, [r0, #16] +; CHECK-NEXT: vmov r1, s10 +; CHECK-NEXT: vmov r5, s4 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r0, s12 +; CHECK-NEXT: vmov r6, s7 +; CHECK-NEXT: vmov r4, s11 +; CHECK-NEXT: ldr.w r12, [r1] +; CHECK-NEXT: vmov r1, s0 +; CHECK-NEXT: ldr r5, [r5] +; CHECK-NEXT: ldr r2, [r2] +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: ldr r6, [r6] +; CHECK-NEXT: ldr r4, [r4] +; CHECK-NEXT: ldr.w lr, [r1] +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: ldr r3, [r1] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov.32 q0[0], r5 +; CHECK-NEXT: vmov r5, s5 +; CHECK-NEXT: ldr r1, [r1] +; CHECK-NEXT: ldr r5, [r5] +; CHECK-NEXT: vmov.32 q0[1], r5 +; CHECK-NEXT: vmov r5, s6 +; CHECK-NEXT: vmov.32 q1[0], r0 +; CHECK-NEXT: vmov r0, s13 +; CHECK-NEXT: ldr r5, [r5] +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r5 +; CHECK-NEXT: vmov r5, s8 +; CHECK-NEXT: vmov.32 q1[1], r0 +; CHECK-NEXT: vmov r0, s14 +; CHECK-NEXT: vmov.32 q0[3], r6 +; CHECK-NEXT: ldr r5, [r5] +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: vmov.32 q1[2], r0 +; CHECK-NEXT: vmov r0, s15 +; CHECK-NEXT: vmov.32 q3[0], lr +; CHECK-NEXT: vmov.32 q3[1], r3 +; CHECK-NEXT: vmov.32 q3[2], r1 +; CHECK-NEXT: vmov.32 q3[3], r2 +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: vmov.32 q1[3], r0 +; CHECK-NEXT: vmov r0, s9 +; CHECK-NEXT: vmov.32 q2[0], r5 +; CHECK-NEXT: ldr r0, [r0] +; CHECK-NEXT: vmov.32 q2[1], r0 +; CHECK-NEXT: vmov.32 q2[2], r12 +; CHECK-NEXT: vmov.32 q2[3], r4 +; CHECK-NEXT: pop {r4, r5, r6, pc} +entry: + %offs = load <16 x i32*>, <16 x i32*>* %offptr, align 4 + %gather = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> %offs, i32 4, <16 x i1> , <16 x i32> undef) + ret <16 x i32> %gather +} + +; f32 + +define arm_aapcs_vfpcc <2 x float> @ptr_v2f32(<2 x float*>* %offptr) { +; CHECK-LABEL: ptr_v2f32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: ldrd r1, r0, [r0] +; CHECK-NEXT: vldr s1, [r0] +; CHECK-NEXT: vldr s0, [r1] +; CHECK-NEXT: bx lr +entry: + %offs = load <2 x float*>, <2 x float*>* %offptr, align 4 + %gather = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> %offs, i32 4, <2 x i1> , <2 x float> undef) + ret <2 x float> %gather +} + +define arm_aapcs_vfpcc <4 x float> @ptr_v4f32(<4 x float*>* %offptr) { +; CHECK-LABEL: ptr_v4f32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q1, [r0] +; CHECK-NEXT: vldrw.u32 q0, [q1] +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x float*>, <4 x float*>* %offptr, align 4 + %gather = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %offs, i32 4, <4 x i1> , <4 x float> undef) + ret <4 x float> %gather +} + +define arm_aapcs_vfpcc <8 x float> @ptr_v8f32(<8 x float*>* %offptr) { +; CHECK-LABEL: ptr_v8f32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q1, [r0] +; CHECK-NEXT: vldrw.u32 q2, [r0, #16] +; CHECK-NEXT: vmov r1, s7 +; CHECK-NEXT: vmov r0, s11 +; CHECK-NEXT: vldr s3, [r1] +; CHECK-NEXT: vmov r1, s6 +; CHECK-NEXT: vldr s2, [r1] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: vldr s1, [r1] +; CHECK-NEXT: vmov r1, s4 +; CHECK-NEXT: vldr s7, [r0] +; CHECK-NEXT: vmov r0, s10 +; CHECK-NEXT: vldr s0, [r1] +; CHECK-NEXT: vldr s6, [r0] +; CHECK-NEXT: vmov r0, s9 +; CHECK-NEXT: vldr s5, [r0] +; CHECK-NEXT: vmov r0, s8 +; CHECK-NEXT: vldr s4, [r0] +; CHECK-NEXT: bx lr +entry: + %offs = load <8 x float*>, <8 x float*>* %offptr, align 4 + %gather = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> %offs, i32 4, <8 x i1> , <8 x float> undef) + ret <8 x float> %gather +} + +; i16 + +define arm_aapcs_vfpcc <8 x i16> @ptr_i16(<8 x i16*>* %offptr) { +; CHECK-LABEL: ptr_i16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r3, s0 +; CHECK-NEXT: vmov r5, s1 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrh.w r12, [r1] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: ldrh.w lr, [r2] +; CHECK-NEXT: vmov r2, s6 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: ldrh r5, [r5] +; CHECK-NEXT: vmov.16 q0[0], r3 +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.16 q0[1], r5 +; CHECK-NEXT: ldrh r4, [r4] +; CHECK-NEXT: vmov.16 q0[2], r12 +; CHECK-NEXT: vmov.16 q0[3], lr +; CHECK-NEXT: vmov.16 q0[4], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: vmov.16 q0[5], r1 +; CHECK-NEXT: vmov.16 q0[6], r2 +; CHECK-NEXT: vmov.16 q0[7], r4 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i16*>, <8 x i16*>* %offptr, align 4 + %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %offs, i32 2, <8 x i1> , <8 x i16> undef) + ret <8 x i16> %gather +} + +define arm_aapcs_vfpcc <2 x i32> @ptr_v2i16_sext(<2 x i16*>* %offptr) { +; CHECK-LABEL: ptr_v2i16_sext: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: ldrd r1, r0, [r0] +; CHECK-NEXT: ldrsh.w r0, [r0] +; CHECK-NEXT: ldrsh.w r1, [r1] +; CHECK-NEXT: asrs r2, r0, #31 +; CHECK-NEXT: vmov.32 q0[0], r1 +; CHECK-NEXT: asrs r1, r1, #31 +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: vmov.32 q0[2], r0 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: bx lr +entry: + %offs = load <2 x i16*>, <2 x i16*>* %offptr, align 4 + %gather = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %offs, i32 2, <2 x i1> , <2 x i16> undef) + %ext = sext <2 x i16> %gather to <2 x i32> + ret <2 x i32> %ext +} + +define arm_aapcs_vfpcc <2 x i32> @ptr_v2i16_zext(<2 x i16*>* %offptr) { +; CHECK-LABEL: ptr_v2i16_zext: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: ldrd r1, r0, [r0] +; CHECK-NEXT: adr r2, .LCPI9_0 +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vldrw.u32 q0, [r2] +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q1[0], r1 +; CHECK-NEXT: vmov.32 q1[2], r0 +; CHECK-NEXT: vand q0, q1, q0 +; CHECK-NEXT: bx lr +; CHECK-NEXT: .p2align 4 +; CHECK-NEXT: @ %bb.1: +; CHECK-NEXT: .LCPI9_0: +; CHECK-NEXT: .long 65535 @ 0xffff +; CHECK-NEXT: .long 0 @ 0x0 +; CHECK-NEXT: .long 65535 @ 0xffff +; CHECK-NEXT: .long 0 @ 0x0 +entry: + %offs = load <2 x i16*>, <2 x i16*>* %offptr, align 4 + %gather = call <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*> %offs, i32 2, <2 x i1> , <2 x i16> undef) + %ext = zext <2 x i16> %gather to <2 x i32> + ret <2 x i32> %ext +} + +define arm_aapcs_vfpcc <4 x i32> @ptr_v4i16_sext(<4 x i16*>* %offptr) { +; CHECK-LABEL: ptr_v4i16_sext: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16*>, <4 x i16*>* %offptr, align 4 + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %offs, i32 2, <4 x i1> , <4 x i16> undef) + %ext = sext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %ext +} + +define arm_aapcs_vfpcc <4 x i32> @ptr_v4i16_zext(<4 x i16*>* %offptr) { +; CHECK-LABEL: ptr_v4i16_zext: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r0, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r1, s3 +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r1 +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i16*>, <4 x i16*>* %offptr, align 4 + %gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*> %offs, i32 2, <4 x i1> , <4 x i16> undef) + %ext = zext <4 x i16> %gather to <4 x i32> + ret <4 x i32> %ext +} + +define arm_aapcs_vfpcc <8 x i32> @ptr_v8i16_sext(<8 x i16*>* %offptr) { +; CHECK-LABEL: ptr_v8i16_sext: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r3, s0 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r5, s1 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrh.w r12, [r1] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: ldrh.w lr, [r2] +; CHECK-NEXT: vmov r2, s6 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: ldrh r5, [r5] +; CHECK-NEXT: vmov.32 q0[0], r3 +; CHECK-NEXT: vmov.32 q1[0], r0 +; CHECK-NEXT: ldrh r4, [r4] +; CHECK-NEXT: vmov.32 q0[1], r5 +; CHECK-NEXT: vmov.32 q0[2], r12 +; CHECK-NEXT: vmov.32 q0[3], lr +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: vmov.32 q1[1], r1 +; CHECK-NEXT: vmov.32 q1[2], r2 +; CHECK-NEXT: vmov.32 q1[3], r4 +; CHECK-NEXT: vmovlb.s16 q1, q1 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i16*>, <8 x i16*>* %offptr, align 4 + %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %offs, i32 2, <8 x i1> , <8 x i16> undef) + %ext = sext <8 x i16> %gather to <8 x i32> + ret <8 x i32> %ext +} + +define arm_aapcs_vfpcc <8 x i32> @ptr_v8i16_zext(<8 x i16*>* %offptr) { +; CHECK-LABEL: ptr_v8i16_zext: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r3, s0 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r5, s1 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrh.w r12, [r1] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: ldrh.w lr, [r2] +; CHECK-NEXT: vmov r2, s6 +; CHECK-NEXT: ldrh r3, [r3] +; CHECK-NEXT: ldrh r0, [r0] +; CHECK-NEXT: ldrh r5, [r5] +; CHECK-NEXT: vmov.32 q0[0], r3 +; CHECK-NEXT: vmov.32 q1[0], r0 +; CHECK-NEXT: ldrh r4, [r4] +; CHECK-NEXT: vmov.32 q0[1], r5 +; CHECK-NEXT: vmov.32 q0[2], r12 +; CHECK-NEXT: vmov.32 q0[3], lr +; CHECK-NEXT: vmovlb.u16 q0, q0 +; CHECK-NEXT: ldrh r1, [r1] +; CHECK-NEXT: ldrh r2, [r2] +; CHECK-NEXT: vmov.32 q1[1], r1 +; CHECK-NEXT: vmov.32 q1[2], r2 +; CHECK-NEXT: vmov.32 q1[3], r4 +; CHECK-NEXT: vmovlb.u16 q1, q1 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i16*>, <8 x i16*>* %offptr, align 4 + %gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*> %offs, i32 2, <8 x i1> , <8 x i16> undef) + %ext = zext <8 x i16> %gather to <8 x i32> + ret <8 x i32> %ext +} + +; f16 + +define arm_aapcs_vfpcc <8 x half> @ptr_f16(<8 x half*>* %offptr) { +; CHECK-LABEL: ptr_f16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q1, [r0] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: vldr.16 s0, [r1] +; CHECK-NEXT: vmov r2, s4 +; CHECK-NEXT: vmov r1, s0 +; CHECK-NEXT: vldr.16 s0, [r2] +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov.16 q0[0], r2 +; CHECK-NEXT: vmov.16 q0[1], r1 +; CHECK-NEXT: vmov r1, s6 +; CHECK-NEXT: vldr.16 s8, [r1] +; CHECK-NEXT: vmov r1, s8 +; CHECK-NEXT: vmov.16 q0[2], r1 +; CHECK-NEXT: vmov r1, s7 +; CHECK-NEXT: vldr.16 s4, [r1] +; CHECK-NEXT: vmov r1, s4 +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov.16 q0[3], r1 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vldr.16 s8, [r0] +; CHECK-NEXT: vmov r0, s8 +; CHECK-NEXT: vmov.16 q0[4], r0 +; CHECK-NEXT: vmov r0, s5 +; CHECK-NEXT: vldr.16 s8, [r0] +; CHECK-NEXT: vmov r0, s8 +; CHECK-NEXT: vmov.16 q0[5], r0 +; CHECK-NEXT: vmov r0, s6 +; CHECK-NEXT: vldr.16 s8, [r0] +; CHECK-NEXT: vmov r0, s8 +; CHECK-NEXT: vmov.16 q0[6], r0 +; CHECK-NEXT: vmov r0, s7 +; CHECK-NEXT: vldr.16 s4, [r0] +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov.16 q0[7], r0 +; CHECK-NEXT: bx lr +entry: + %offs = load <8 x half*>, <8 x half*>* %offptr, align 4 + %gather = call <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*> %offs, i32 2, <8 x i1> , <8 x half> undef) + ret <8 x half> %gather +} + +; i8 + +define arm_aapcs_vfpcc <16 x i8> @ptr_i8(<16 x i8*>* %offptr) { +; CHECK-LABEL: ptr_i8: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r6, lr} +; CHECK-NEXT: push {r4, r5, r6, lr} +; CHECK-NEXT: vldrw.u32 q1, [r0, #32] +; CHECK-NEXT: vldrw.u32 q0, [r0, #48] +; CHECK-NEXT: vldrw.u32 q2, [r0] +; CHECK-NEXT: vmov r1, s6 +; CHECK-NEXT: vmov r5, s8 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r6, s11 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrb.w r12, [r1] +; CHECK-NEXT: vmov r1, s0 +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r6, [r6] +; CHECK-NEXT: ldrb r4, [r4] +; CHECK-NEXT: ldrb.w lr, [r1] +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: ldrb r3, [r1] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov.8 q0[0], r5 +; CHECK-NEXT: vmov r5, s9 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: vmov.8 q0[1], r5 +; CHECK-NEXT: vmov r5, s10 +; CHECK-NEXT: vldrw.u32 q2, [r0, #16] +; CHECK-NEXT: vmov r0, s8 +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: vmov.8 q0[2], r5 +; CHECK-NEXT: vmov r5, s4 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.8 q0[3], r6 +; CHECK-NEXT: vmov.8 q0[4], r0 +; CHECK-NEXT: vmov r0, s9 +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.8 q0[5], r0 +; CHECK-NEXT: vmov r0, s10 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.8 q0[6], r0 +; CHECK-NEXT: vmov r0, s11 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.8 q0[7], r0 +; CHECK-NEXT: vmov r0, s5 +; CHECK-NEXT: vmov.8 q0[8], r5 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.8 q0[9], r0 +; CHECK-NEXT: vmov.8 q0[10], r12 +; CHECK-NEXT: vmov.8 q0[11], r4 +; CHECK-NEXT: vmov.8 q0[12], lr +; CHECK-NEXT: vmov.8 q0[13], r3 +; CHECK-NEXT: vmov.8 q0[14], r1 +; CHECK-NEXT: vmov.8 q0[15], r2 +; CHECK-NEXT: pop {r4, r5, r6, pc} +entry: + %offs = load <16 x i8*>, <16 x i8*>* %offptr, align 4 + %gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*> %offs, i32 2, <16 x i1> , <16 x i8> undef) + ret <16 x i8> %gather +} + +define arm_aapcs_vfpcc <8 x i16> @ptr_v8i8_sext16(<8 x i8*>* %offptr) { +; CHECK-LABEL: ptr_v8i8_sext16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r5, s0 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrb.w r12, [r2] +; CHECK-NEXT: vmov r2, s5 +; CHECK-NEXT: ldrb.w lr, [r1] +; CHECK-NEXT: vmov r1, s6 +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.16 q0[0], r5 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.16 q0[1], lr +; CHECK-NEXT: ldrb r4, [r4] +; CHECK-NEXT: vmov.16 q0[2], r3 +; CHECK-NEXT: vmov.16 q0[3], r12 +; CHECK-NEXT: vmov.16 q0[4], r0 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.16 q0[5], r2 +; CHECK-NEXT: vmov.16 q0[6], r1 +; CHECK-NEXT: vmov.16 q0[7], r4 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4 + %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> , <8 x i8> undef) + %ext = sext <8 x i8> %gather to <8 x i16> + ret <8 x i16> %ext +} + +define arm_aapcs_vfpcc <8 x i16> @ptr_v8i8_zext16(<8 x i8*>* %offptr) { +; CHECK-LABEL: ptr_v8i8_zext16: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r5, s0 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrb.w r12, [r2] +; CHECK-NEXT: vmov r2, s5 +; CHECK-NEXT: ldrb.w lr, [r1] +; CHECK-NEXT: vmov r1, s6 +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.16 q0[0], r5 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.16 q0[1], lr +; CHECK-NEXT: ldrb r4, [r4] +; CHECK-NEXT: vmov.16 q0[2], r3 +; CHECK-NEXT: vmov.16 q0[3], r12 +; CHECK-NEXT: vmov.16 q0[4], r0 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.16 q0[5], r2 +; CHECK-NEXT: vmov.16 q0[6], r1 +; CHECK-NEXT: vmov.16 q0[7], r4 +; CHECK-NEXT: vmovlb.u8 q0, q0 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4 + %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> , <8 x i8> undef) + %ext = zext <8 x i8> %gather to <8 x i16> + ret <8 x i16> %ext +} + +define arm_aapcs_vfpcc <4 x i32> @ptr_v4i8_sext32(<4 x i8*>* %offptr) { +; CHECK-LABEL: ptr_v4i8_sext32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vmov r0, s0 +; CHECK-NEXT: vmov r3, s1 +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[0], r0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[1], r3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q0[2], r1 +; CHECK-NEXT: vmov.32 q0[3], r2 +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8*>, <4 x i8*>* %offptr, align 4 + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %offs, i32 1, <4 x i1> , <4 x i8> undef) + %ext = sext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %ext +} + +define arm_aapcs_vfpcc <4 x i32> @ptr_v4i8_zext32(<4 x i8*>* %offptr) { +; CHECK-LABEL: ptr_v4i8_zext32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: vmov r2, s0 +; CHECK-NEXT: vmov r1, s1 +; CHECK-NEXT: vmov r3, s2 +; CHECK-NEXT: vmov r0, s3 +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: vmov.32 q0[0], r2 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q0[1], r1 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.32 q0[2], r3 +; CHECK-NEXT: vmov.32 q0[3], r0 +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: bx lr +entry: + %offs = load <4 x i8*>, <4 x i8*>* %offptr, align 4 + %gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> %offs, i32 1, <4 x i1> , <4 x i8> undef) + %ext = zext <4 x i8> %gather to <4 x i32> + ret <4 x i32> %ext +} + +define arm_aapcs_vfpcc <8 x i32> @ptr_v8i8_sext32(<8 x i8*>* %offptr) { +; CHECK-LABEL: ptr_v8i8_sext32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r3, s0 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r5, s1 +; CHECK-NEXT: vmov r4, s7 +; CHECK-NEXT: ldrb.w r12, [r1] +; CHECK-NEXT: vmov r1, s5 +; CHECK-NEXT: ldrb.w lr, [r2] +; CHECK-NEXT: vmov r2, s6 +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: vmov.32 q0[0], r3 +; CHECK-NEXT: vmov.32 q1[0], r0 +; CHECK-NEXT: ldrb r4, [r4] +; CHECK-NEXT: vmov.32 q0[1], r5 +; CHECK-NEXT: vmov.32 q0[2], r12 +; CHECK-NEXT: vmov.32 q0[3], lr +; CHECK-NEXT: vmovlb.s8 q0, q0 +; CHECK-NEXT: vmovlb.s16 q0, q0 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q1[1], r1 +; CHECK-NEXT: vmov.32 q1[2], r2 +; CHECK-NEXT: vmov.32 q1[3], r4 +; CHECK-NEXT: vmovlb.s8 q1, q1 +; CHECK-NEXT: vmovlb.s16 q1, q1 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4 + %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> , <8 x i8> undef) + %ext = sext <8 x i8> %gather to <8 x i32> + ret <8 x i32> %ext +} + +define arm_aapcs_vfpcc <8 x i32> @ptr_v8i8_zext32(<8 x i8*>* %offptr) { +; CHECK-LABEL: ptr_v8i8_zext32: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r4, r5, r7, lr} +; CHECK-NEXT: push {r4, r5, r7, lr} +; CHECK-NEXT: vldrw.u32 q0, [r0] +; CHECK-NEXT: vldrw.u32 q1, [r0, #16] +; CHECK-NEXT: vmov r1, s2 +; CHECK-NEXT: vmov r2, s3 +; CHECK-NEXT: vmov r0, s4 +; CHECK-NEXT: vmov r4, s0 +; CHECK-NEXT: vmov r3, s5 +; CHECK-NEXT: vmov r5, s1 +; CHECK-NEXT: ldrb.w r12, [r1] +; CHECK-NEXT: vmov r1, s6 +; CHECK-NEXT: ldrb.w lr, [r2] +; CHECK-NEXT: vmov r2, s7 +; CHECK-NEXT: ldrb r0, [r0] +; CHECK-NEXT: vmov.i32 q1, #0xff +; CHECK-NEXT: ldrb r4, [r4] +; CHECK-NEXT: ldrb r3, [r3] +; CHECK-NEXT: vmov.32 q2[0], r0 +; CHECK-NEXT: ldrb r5, [r5] +; CHECK-NEXT: vmov.32 q0[0], r4 +; CHECK-NEXT: vmov.32 q2[1], r3 +; CHECK-NEXT: vmov.32 q0[1], r5 +; CHECK-NEXT: vmov.32 q0[2], r12 +; CHECK-NEXT: vmov.32 q0[3], lr +; CHECK-NEXT: vand q0, q0, q1 +; CHECK-NEXT: ldrb r1, [r1] +; CHECK-NEXT: ldrb r2, [r2] +; CHECK-NEXT: vmov.32 q2[2], r1 +; CHECK-NEXT: vmov.32 q2[3], r2 +; CHECK-NEXT: vand q1, q2, q1 +; CHECK-NEXT: pop {r4, r5, r7, pc} +entry: + %offs = load <8 x i8*>, <8 x i8*>* %offptr, align 4 + %gather = call <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*> %offs, i32 1, <8 x i1> , <8 x i8> undef) + %ext = zext <8 x i8> %gather to <8 x i32> + ret <8 x i32> %ext +} + +; loops + +define void @foo_ptr_p_int32_t(i32* %dest, i32** %src, i32 %n) { +; CHECK-LABEL: foo_ptr_p_int32_t: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r7, lr} +; CHECK-NEXT: push {r7, lr} +; CHECK-NEXT: bic r2, r2, #15 +; CHECK-NEXT: cmp r2, #1 +; CHECK-NEXT: it lt +; CHECK-NEXT: poplt {r7, pc} +; CHECK-NEXT: subs r2, #4 +; CHECK-NEXT: movs r3, #1 +; CHECK-NEXT: subs r1, #16 +; CHECK-NEXT: add.w lr, r3, r2, lsr #2 +; CHECK-NEXT: dls lr, lr +; CHECK-NEXT: .LBB22_1: @ %vector.body +; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vldrw.u32 q0, [r1, #16]! +; CHECK-NEXT: vptt.i32 ne, q0, zr +; CHECK-NEXT: vldrwt.u32 q1, [q0] +; CHECK-NEXT: vstrwt.32 q1, [r0], #16 +; CHECK-NEXT: le lr, .LBB22_1 +; CHECK-NEXT: @ %bb.2: @ %for.end +; CHECK-NEXT: pop {r7, pc} +entry: + %and = and i32 %n, -16 + %cmp11 = icmp sgt i32 %and, 0 + br i1 %cmp11, label %vector.body, label %for.end + +vector.body: ; preds = %entry, %vector.body + %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ] + %0 = getelementptr inbounds i32*, i32** %src, i32 %index + %1 = bitcast i32** %0 to <4 x i32*>* + %wide.load = load <4 x i32*>, <4 x i32*>* %1, align 4 + %2 = icmp ne <4 x i32*> %wide.load, zeroinitializer + %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %wide.load, i32 4, <4 x i1> %2, <4 x i32> undef) + %3 = getelementptr inbounds i32, i32* %dest, i32 %index + %4 = bitcast i32* %3 to <4 x i32>* + call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %wide.masked.gather, <4 x i32>* %4, i32 4, <4 x i1> %2) + %index.next = add i32 %index, 4 + %5 = icmp eq i32 %index.next, %and + br i1 %5, label %for.end, label %vector.body + +for.end: ; preds = %vector.body, %entry + ret void +} + +define void @foo_ptr_p_float(float* %dest, float** %src, i32 %n) { +; CHECK-LABEL: foo_ptr_p_float: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r7, lr} +; CHECK-NEXT: push {r7, lr} +; CHECK-NEXT: bic r2, r2, #15 +; CHECK-NEXT: cmp r2, #1 +; CHECK-NEXT: it lt +; CHECK-NEXT: poplt {r7, pc} +; CHECK-NEXT: subs r2, #4 +; CHECK-NEXT: movs r3, #1 +; CHECK-NEXT: subs r1, #16 +; CHECK-NEXT: add.w lr, r3, r2, lsr #2 +; CHECK-NEXT: dls lr, lr +; CHECK-NEXT: .LBB23_1: @ %vector.body +; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vldrw.u32 q0, [r1, #16]! +; CHECK-NEXT: vptt.i32 ne, q0, zr +; CHECK-NEXT: vldrwt.u32 q1, [q0] +; CHECK-NEXT: vstrwt.32 q1, [r0], #16 +; CHECK-NEXT: le lr, .LBB23_1 +; CHECK-NEXT: @ %bb.2: @ %for.end +; CHECK-NEXT: pop {r7, pc} +entry: + %and = and i32 %n, -16 + %cmp11 = icmp sgt i32 %and, 0 + br i1 %cmp11, label %vector.body, label %for.end + +vector.body: ; preds = %entry, %vector.body + %index = phi i32 [ %index.next, %vector.body ], [ 0, %entry ] + %0 = getelementptr inbounds float*, float** %src, i32 %index + %1 = bitcast float** %0 to <4 x float*>* + %wide.load = load <4 x float*>, <4 x float*>* %1, align 4 + %2 = icmp ne <4 x float*> %wide.load, zeroinitializer + %3 = bitcast <4 x float*> %wide.load to <4 x i32*> + %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %3, i32 4, <4 x i1> %2, <4 x i32> undef) + %4 = getelementptr inbounds float, float* %dest, i32 %index + %5 = bitcast float* %4 to <4 x i32>* + call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %wide.masked.gather, <4 x i32>* %5, i32 4, <4 x i1> %2) + %index.next = add i32 %index, 4 + %6 = icmp eq i32 %index.next, %and + br i1 %6, label %for.end, label %vector.body + +for.end: ; preds = %vector.body, %entry + ret void +} + +declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>) +declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>) +declare <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*>, i32, <8 x i1>, <8 x i32>) +declare <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*>, i32, <16 x i1>, <16 x i32>) +declare <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*>, i32, <2 x i1>, <2 x float>) +declare <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*>, i32, <4 x i1>, <4 x float>) +declare <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*>, i32, <8 x i1>, <8 x float>) +declare <2 x i16> @llvm.masked.gather.v2i16.v2p0i16(<2 x i16*>, i32, <2 x i1>, <2 x i16>) +declare <4 x i16> @llvm.masked.gather.v4i16.v4p0i16(<4 x i16*>, i32, <4 x i1>, <4 x i16>) +declare <8 x i16> @llvm.masked.gather.v8i16.v8p0i16(<8 x i16*>, i32, <8 x i1>, <8 x i16>) +declare <16 x i16> @llvm.masked.gather.v16i16.v16p0i16(<16 x i16*>, i32, <16 x i1>, <16 x i16>) +declare <4 x half> @llvm.masked.gather.v4f16.v4p0f16(<4 x half*>, i32, <4 x i1>, <4 x half>) +declare <8 x half> @llvm.masked.gather.v8f16.v8p0f16(<8 x half*>, i32, <8 x i1>, <8 x half>) +declare <16 x half> @llvm.masked.gather.v16f16.v16p0f16(<16 x half*>, i32, <16 x i1>, <16 x half>) +declare <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*>, i32, <4 x i1>, <4 x i8>) +declare <8 x i8> @llvm.masked.gather.v8i8.v8p0i8(<8 x i8*>, i32, <8 x i1>, <8 x i8>) +declare <16 x i8> @llvm.masked.gather.v16i8.v16p0i8(<16 x i8*>, i32, <16 x i1>, <16 x i8>) +declare <32 x i8> @llvm.masked.gather.v32i8.v32p0i8(<32 x i8*>, i32, <32 x i1>, <32 x i8>) +declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>) diff --git a/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll b/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll new file mode 100644 index 00000000000..70ad30aa60c --- /dev/null +++ b/test/CodeGen/Thumb2/mve-gather-scatter-opt.ll @@ -0,0 +1,44 @@ +; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o 2>/dev/null - | FileCheck --check-prefix NOGATSCAT %s +; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=-mve -enable-arm-maskedgatscat %s -o 2>/dev/null - | FileCheck --check-prefix NOMVE %s + +define arm_aapcs_vfpcc <4 x i32> @unscaled_i32_i32(i8* %base, <4 x i32>* %offptr) { +; NOGATSCAT-LABEL: unscaled_i32_i32: +; NOGATSCAT: @ %bb.0: @ %entry +; NOGATSCAT-NEXT: vldrw.u32 q0, [r1] +; NOGATSCAT-NEXT: vadd.i32 q0, q0, r0 +; NOGATSCAT-NEXT: vmov r0, s0 +; NOGATSCAT-NEXT: vmov r3, s1 +; NOGATSCAT-NEXT: vmov r1, s2 +; NOGATSCAT-NEXT: vmov r2, s3 +; NOGATSCAT-NEXT: ldr r0, [r0] +; NOGATSCAT-NEXT: ldr r3, [r3] +; NOGATSCAT-NEXT: vmov.32 q0[0], r0 +; NOGATSCAT-NEXT: ldr r1, [r1] +; NOGATSCAT-NEXT: vmov.32 q0[1], r3 +; NOGATSCAT-NEXT: ldr r2, [r2] +; NOGATSCAT-NEXT: vmov.32 q0[2], r1 +; NOGATSCAT-NEXT: vmov.32 q0[3], r2 +; NOGATSCAT-NEXT: bx lr + +; NOMVE-LABEL: unscaled_i32_i32: +; NOMVE: @ %bb.0: @ %entry +; NOMVE-NEXT: .save {r4, lr} +; NOMVE-NEXT: push {r4, lr} +; NOMVE-NEXT: ldm.w r1, {r2, r3, lr} +; NOMVE-NEXT: ldr r4, [r1, #12] +; NOMVE-NEXT: ldr.w r12, [r0, r2] +; NOMVE-NEXT: ldr r1, [r0, r3] +; NOMVE-NEXT: ldr.w r2, [r0, lr] +; NOMVE-NEXT: ldr r3, [r0, r4] +; NOMVE-NEXT: mov r0, r12 +; NOMVE-NEXT: pop {r4, pc} + +entry: + %offs = load <4 x i32>, <4 x i32>* %offptr, align 4 + %byte_ptrs = getelementptr inbounds i8, i8* %base, <4 x i32> %offs + %ptrs = bitcast <4 x i8*> %byte_ptrs to <4 x i32*> + %gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> , <4 x i32> undef) + ret <4 x i32> %gather +} + +declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)