RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 20:51:52 +01:00
Code Issues Projects Releases Wiki Activity

[AMDGPU] Support unaligned flat scratch in TLI

Browse Source 
Adjust SITargetLowering::allowsMisalignedMemoryAccessesImpl for
unaligned flat scratch support. Mostly needed for global isel.

Differential Revision: https://reviews.llvm.org/D93669
This commit is contained in:
Stanislav Mekhanoshin 2020-12-21 15:40:46 -08:00
parent 92eadd3cde
commit b2c1643118
4 changed files with 61 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lib/Target/AMDGPU/SIISelLowering.cpp
View File

@ -1470,12 +1470,21 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
}
}
if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
bool AlignedBy4 = Alignment >= Align(4);
if (IsFast)
*IsFast = AlignedBy4;
return AlignedBy4 ||
Subtarget->enableFlatScratch() ||
Subtarget->hasUnalignedScratchAccess();
}
// FIXME: We have to be conservative here and assume that flat operations
// will access scratch. If we had access to the IR function, then we
// could determine if any private memory was used in the function.
if (!Subtarget->hasUnalignedScratchAccess() &&
(AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
AddrSpace == AMDGPUAS::FLAT_ADDRESS)) {
if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
!Subtarget->hasUnalignedScratchAccess()) {
bool AlignedBy4 = Alignment >= Align(4);
if (IsFast)
*IsFast = AlignedBy4;

11
test/CodeGen/AMDGPU/chain-hi-to-lo.ll
View File

@ -271,16 +271,9 @@ define amdgpu_kernel void @vload2_private(i16 addrspace(1)* nocapture readonly %
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: scratch_store_short off, v0, vcc_hi offset:8
; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0
; FLATSCR-NEXT: scratch_load_ushort v0, off, vcc_hi offset:4
; FLATSCR-NEXT: scratch_load_dword v0, off, vcc_hi offset:4
; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0
; FLATSCR-NEXT: scratch_load_ushort v3, off, vcc_hi offset:6
; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0
; FLATSCR-NEXT: s_waitcnt vmcnt(1)
; FLATSCR-NEXT: v_and_b32_e32 v0, 0xffff, v0
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: v_mov_b32_e32 v1, v3
; FLATSCR-NEXT: scratch_load_short_d16_hi v1, off, vcc_hi offset:8
; FLATSCR-NEXT: v_lshl_or_b32 v0, v3, 16, v0
; FLATSCR-NEXT: scratch_load_dword v1, off, vcc_hi offset:6
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
; FLATSCR-NEXT: s_endpgm

69
test/CodeGen/AMDGPU/unaligned-load-store.ll
View File

@ -1,6 +1,7 @@
; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s
; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=UNALIGNED %s
; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s
; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s
; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,UNALIGNED %s
; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s
; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,FLATSCR,ALIGNED %s
; SI-LABEL: {{^}}local_unaligned_load_store_i16:
; SI: ds_read_u8
@ -602,64 +603,70 @@ define amdgpu_kernel void @local_store_align1_v16i8(<16 x i8> addrspace(3)* %out
}
; SI-LABEL: {{^}}private_load_align1_f64:
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; MUBUF: buffer_load_ubyte
; FLATSCR: scratch_load_dwordx2
define double @private_load_align1_f64(double addrspace(5)* %in) {
%x = load double, double addrspace(5)* %in, align 1
ret double %x
}
; SI-LABEL: {{^}}private_store_align1_f64:
; SI: buffer_store_byte
; SI: buffer_store_byte
; SI: buffer_store_byte
; SI: buffer_store_byte
; SI: buffer_store_byte
; SI: buffer_store_byte
; SI: buffer_store_byte
; SI: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; MUBUF: buffer_store_byte
; FLATSCR: scratch_store_dwordx2
define void @private_store_align1_f64(double addrspace(5)* %out, double %x) #0 {
store double %x, double addrspace(5)* %out, align 1
ret void
}
; SI-LABEL: {{^}}private_load_align4_f64:
; SI: buffer_load_dword
; SI: buffer_load_dword
; MUBUF: buffer_load_dword
; MUBUF: buffer_load_dword
; FLATSCR: scratch_load_dwordx2
define double @private_load_align4_f64(double addrspace(5)* %in) {
%x = load double, double addrspace(5)* %in, align 4
ret double %x
}
; SI-LABEL: {{^}}private_store_align4_f64:
; SI: buffer_store_dword
; SI: buffer_store_dword
; MUBUF: buffer_store_dword
; MUBUF: buffer_store_dword
; FLATSCR: scratch_store_dwordx2
define void @private_store_align4_f64(double addrspace(5)* %out, double %x) #0 {
store double %x, double addrspace(5)* %out, align 4
ret void
}
; SI-LABEL: {{^}}private_load_align2_f64:
; SI: buffer_load_ushort
; SI: buffer_load_ushort
; SI: buffer_load_ushort
; SI: buffer_load_ushort
; MUBUF: buffer_load_ushort
; MUBUF: buffer_load_ushort
; MUBUF: buffer_load_ushort
; MUBUF: buffer_load_ushort
; FLATSCR: scratch_load_dwordx2
define double @private_load_align2_f64(double addrspace(5)* %in) {
%x = load double, double addrspace(5)* %in, align 2
ret double %x
}
; SI-LABEL: {{^}}private_store_align2_f64:
; SI: buffer_store_short
; SI: buffer_store_short
; SI: buffer_store_short
; SI: buffer_store_short
; MUBUF: buffer_store_short
; MUBUF: buffer_store_short
; MUBUF: buffer_store_short
; MUBUF: buffer_store_short
; FLATSCR: scratch_store_dwordx2
define void @private_store_align2_f64(double addrspace(5)* %out, double %x) #0 {
store double %x, double addrspace(5)* %out, align 2
ret void

35
test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
View File

@ -9,9 +9,6 @@ target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:3
; ALL-LABEL: @load_unknown_offset_align1_i8(
; ALL: alloca [128 x i8], align 1
; UNALIGNED: load <2 x i8>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
; ALIGNED: load i8, i8 addrspace(5)* %ptr0, align 1{{$}}
; ALIGNED: load i8, i8 addrspace(5)* %ptr1, align 1{{$}}
define amdgpu_kernel void @load_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 {
%alloca = alloca [128 x i8], align 1, addrspace(5)
%ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset
@ -60,13 +57,11 @@ define amdgpu_kernel void @load_unknown_offset_align1_i32(i32 addrspace(1)* noal
ret void
}
; FIXME: Should always increase alignment of the load
; Make sure alloca alignment isn't decreased
; ALL-LABEL: @load_alloca16_unknown_offset_align1_i32(
; ALL: alloca [128 x i32], align 16
; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
; ALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
; ALL: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 {
%alloca = alloca [128 x i32], align 16, addrspace(5)
%ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset
@ -128,11 +123,8 @@ define amdgpu_kernel void @store_unknown_offset_align1_i32(i32 addrspace(1)* noa
}
; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32(
; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
; ALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
; UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 1
; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5)
; ALL: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
%alloca = alloca [8 x i32], align 1, addrspace(5)
%out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
@ -148,11 +140,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
}
; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
; ALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
; UNALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 1
; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5)
; ALL: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
%alloca = alloca [8 x i8], align 1, addrspace(5)
%out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
@ -168,11 +157,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
}
; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i32(
; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
; ALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
; UNALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 1
; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5)
; ALL: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
%alloca = alloca [8 x i32], align 1, addrspace(5)
%out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
@ -188,11 +174,8 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
}
; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i8(
; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
; ALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
; UNALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 1
; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5)
; ALL: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() {
%alloca = alloca [8 x i8], align 1, addrspace(5)
%out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*