mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2025-01-31 20:51:52 +01:00
df7173628b
Summary: This patch adds the following intrinsics for non-temporal gather loads and scatter stores: * aarch64_sve_ldnt1_gather_index * aarch64_sve_stnt1_scatter_index These intrinsics implement the "scalar + vector of indices" addressing mode. As opposed to regular and first-faulting gathers/scatters, there's no instruction that would take indices and then scale them. Instead, the indices for non-temporal gathers/scatters are scaled before the intrinsics are lowered to `ldnt1` instructions. The new ISD nodes, GLDNT1_INDEX and SSTNT1_INDEX, are only used as placeholders so that we can easily identify the cases implemented in this patch in performGatherLoadCombine and performScatterStoreCombined. Once encountered, they are replaced with: * GLDNT1_INDEX -> SPLAT_VECTOR + SHL + GLDNT1 * SSTNT1_INDEX -> SPLAT_VECTOR + SHL + SSTNT1 The patterns for lowering ISD::SHL for scalable vectors (required by this patch) were missing, so these are added too. Reviewed By: sdesmalen Differential Revision: https://reviews.llvm.org/D75601
91 lines
4.3 KiB
LLVM
91 lines
4.3 KiB
LLVM
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
|
|
|
|
;
|
|
; LDNT1H, LDNT1W, LDNT1D: base + 64-bit index
|
|
; e.g.
|
|
; lsl z0.d, z0.d, #1
|
|
; ldnt1h z0.d, p0/z, [z0.d, x0]
|
|
;
|
|
|
|
define <vscale x 2 x i64> @gldnt1h_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
|
|
; CHECK-LABEL: gldnt1h_index
|
|
; CHECK: lsl z0.d, z0.d, #1
|
|
; CHECK-NEXT: ldnt1h { z0.d }, p0/z, [z0.d, x0]
|
|
; CHECK-NEXT: ret
|
|
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
|
|
i16* %base,
|
|
<vscale x 2 x i64> %b)
|
|
%res = zext <vscale x 2 x i16> %load to <vscale x 2 x i64>
|
|
ret <vscale x 2 x i64> %res
|
|
}
|
|
|
|
define <vscale x 2 x i64> @gldnt1w_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
|
|
; CHECK-LABEL: gldnt1w_index
|
|
; CHECK: lsl z0.d, z0.d, #2
|
|
; CHECK-NEXT: ldnt1w { z0.d }, p0/z, [z0.d, x0]
|
|
; CHECK-NEXT: ret
|
|
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
|
|
i32* %base,
|
|
<vscale x 2 x i64> %b)
|
|
%res = zext <vscale x 2 x i32> %load to <vscale x 2 x i64>
|
|
ret <vscale x 2 x i64> %res
|
|
}
|
|
|
|
define <vscale x 2 x i64> @gldnt1d_index(<vscale x 2 x i1> %pg, i64* %base, <vscale x 2 x i64> %b) {
|
|
; CHECK-LABEL: gldnt1d_index
|
|
; CHECK: lsl z0.d, z0.d, #3
|
|
; CHECK-NEXT: ldnt1d { z0.d }, p0/z, [z0.d, x0]
|
|
; CHECK-NEXT: ret
|
|
%load = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i64(<vscale x 2 x i1> %pg,
|
|
i64* %base,
|
|
<vscale x 2 x i64> %b)
|
|
ret <vscale x 2 x i64> %load
|
|
}
|
|
|
|
define <vscale x 2 x double> @gldnt1d_index_double(<vscale x 2 x i1> %pg, double* %base, <vscale x 2 x i64> %b) {
|
|
; CHECK-LABEL: gldnt1d_index_double
|
|
; CHECK: lsl z0.d, z0.d, #3
|
|
; CHECK-NEXT: ldnt1d { z0.d }, p0/z, [z0.d, x0]
|
|
; CHECK-NEXT: ret
|
|
%load = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.gather.index.nxv2f64(<vscale x 2 x i1> %pg,
|
|
double* %base,
|
|
<vscale x 2 x i64> %b)
|
|
ret <vscale x 2 x double> %load
|
|
}
|
|
|
|
;
|
|
; LDNT1SH, LDNT1SW: base + 64-bit index
|
|
; e.g.
|
|
; lsl z0.d, z0.d, #1
|
|
; ldnt1sh z0.d, p0/z, [z0.d, x0]
|
|
;
|
|
|
|
define <vscale x 2 x i64> @gldnt1sh_index(<vscale x 2 x i1> %pg, i16* %base, <vscale x 2 x i64> %b) {
|
|
; CHECK-LABEL: gldnt1sh_index
|
|
; CHECK: lsl z0.d, z0.d, #1
|
|
; CHECK-NEXT: ldnt1sh { z0.d }, p0/z, [z0.d, x0]
|
|
; CHECK-NEXT: ret
|
|
%load = call <vscale x 2 x i16> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i16(<vscale x 2 x i1> %pg,
|
|
i16* %base,
|
|
<vscale x 2 x i64> %b)
|
|
%res = sext <vscale x 2 x i16> %load to <vscale x 2 x i64>
|
|
ret <vscale x 2 x i64> %res
|
|
}
|
|
|
|
define <vscale x 2 x i64> @gldnt1sw_index(<vscale x 2 x i1> %pg, i32* %base, <vscale x 2 x i64> %b) {
|
|
; CHECK-LABEL: gldnt1sw_index
|
|
; CHECK: lsl z0.d, z0.d, #2
|
|
; CHECK-NEXT: ldnt1sw { z0.d }, p0/z, [z0.d, x0]
|
|
; CHECK-NEXT: ret
|
|
%load = call <vscale x 2 x i32> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i32(<vscale x 2 x i1> %pg,
|
|
i32* %base,
|
|
<vscale x 2 x i64> %b)
|
|
%res = sext <vscale x 2 x i32> %load to <vscale x 2 x i64>
|
|
ret <vscale x 2 x i64> %res
|
|
}
|
|
|
|
declare <vscale x 2 x i16> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i16(<vscale x 2 x i1>, i16*, <vscale x 2 x i64>)
|
|
declare <vscale x 2 x i32> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i32(<vscale x 2 x i1>, i32*, <vscale x 2 x i64>)
|
|
declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.gather.index.nxv2i64(<vscale x 2 x i1>, i64*, <vscale x 2 x i64>)
|
|
declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.gather.index.nxv2f64(<vscale x 2 x i1>, double*, <vscale x 2 x i64>)
|