From c3f6130f4111d8533d1495f73353729b5cf0030a Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 21 Aug 2015 22:47:27 +0000 Subject: [PATCH] AMDGPU/SI: Better handle s_wait insertion We can wait on either VM, EXP or LGKM. The waits are independent. Without this patch, a wait inserted because of one of them would also wait for all the previous others. This patch makes s_wait only wait for the ones we need for the next instruction. Here's an example of subtle perf reduction this patch solves: This is without the patch: buffer_load_format_xyzw v[8:11], v0, s[44:47], 0 idxen buffer_load_format_xyzw v[12:15], v0, s[48:51], 0 idxen s_load_dwordx4 s[44:47], s[8:9], 0xc s_waitcnt lgkmcnt(0) buffer_load_format_xyzw v[16:19], v0, s[52:55], 0 idxen s_load_dwordx4 s[48:51], s[8:9], 0x10 s_waitcnt vmcnt(1) buffer_load_format_xyzw v[20:23], v0, s[44:47], 0 idxen The s_waitcnt vmcnt(1) is useless. The reason it is added is because the last buffer_load_format_xyzw needs s[44:47], which was issued by the first s_load_dwordx4. It waits for all VM before that call to have finished. Internally after every instruction, 3 counters (for VM, EXP and LGTM) are updated after every instruction. For example buffer_load_format_xyzw will increase the VM counter, and s_load_dwordx4 the LGKM one. Without the patch, for every defined register, the current 3 counters are stored, and are used to know how long to wait when an instruction needs the register. Because of that, the s[44:47] counter includes that to use the register you need to wait for the previous buffer_load_format_xyzw. Instead this patch stores only the counters that matter for the register, and puts zero for the other ones, since we don't need any wait for them. Patch by: Axel Davy Differential Revision: http://reviews.llvm.org/D11883 llvm-svn: 245755 --- lib/Target/AMDGPU/SIInsertWaits.cpp | 7 ++-- test/CodeGen/AMDGPU/setcc-opt.ll | 4 +-- test/CodeGen/AMDGPU/wait.ll | 55 +++++++++++++++++++++++++---- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/lib/Target/AMDGPU/SIInsertWaits.cpp b/lib/Target/AMDGPU/SIInsertWaits.cpp index df76b457af3..2379b1fcf6a 100644 --- a/lib/Target/AMDGPU/SIInsertWaits.cpp +++ b/lib/Target/AMDGPU/SIInsertWaits.cpp @@ -246,10 +246,13 @@ void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB, // Get the hardware counter increments and sum them up Counters Increment = getHwCounts(*I); + Counters Limit = ZeroCounts; unsigned Sum = 0; for (unsigned i = 0; i < 3; ++i) { LastIssued.Array[i] += Increment.Array[i]; + if (Increment.Array[i]) + Limit.Array[i] = LastIssued.Array[i]; Sum += Increment.Array[i]; } @@ -300,11 +303,11 @@ void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB, // Remember which registers we define if (Op.isDef()) - DefinedRegs[j] = LastIssued; + DefinedRegs[j] = Limit; // and which one we are using if (Op.isUse()) - UsedRegs[j] = LastIssued; + UsedRegs[j] = Limit; } } } diff --git a/test/CodeGen/AMDGPU/setcc-opt.ll b/test/CodeGen/AMDGPU/setcc-opt.ll index 4e6a10d6b78..86a1d3c12f3 100644 --- a/test/CodeGen/AMDGPU/setcc-opt.ll +++ b/test/CodeGen/AMDGPU/setcc-opt.ll @@ -146,7 +146,7 @@ define void @sext_bool_icmp_ne_k(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind ; GCN: v_mov_b32_e32 [[K255:v[0-9]+]], 0xff{{$}} ; GCN: v_cmp_ne_i32_e32 vcc, [[K255]], [[B]] ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc -; GCN-NEXT: buffer_store_byte [[RESULT]] +; GCN: buffer_store_byte [[RESULT]] ; GCN: s_endpgm define void @cmp_zext_k_i8max(i1 addrspace(1)* %out, i8 %b) nounwind { %b.ext = zext i8 %b to i32 @@ -191,7 +191,7 @@ define void @cmp_sext_k_neg1_i8_sext_arg(i1 addrspace(1)* %out, i8 signext %b) n ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0xff{{$}} ; GCN: v_cmp_ne_i32_e32 vcc, [[K]], [[B]]{{$}} ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc -; GCN-NEXT: buffer_store_byte [[RESULT]] +; GCN: buffer_store_byte [[RESULT]] ; GCN: s_endpgm define void @cmp_sext_k_neg1_i8_arg(i1 addrspace(1)* %out, i8 %b) nounwind { %b.ext = sext i8 %b to i32 diff --git a/test/CodeGen/AMDGPU/wait.ll b/test/CodeGen/AMDGPU/wait.ll index 5cc7577cad3..5aaf0028320 100644 --- a/test/CodeGen/AMDGPU/wait.ll +++ b/test/CodeGen/AMDGPU/wait.ll @@ -1,11 +1,16 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -strict-whitespace %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace %s +; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -strict-whitespace %s --check-prefix=DEFAULT +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace %s --check-prefix=DEFAULT +; RUN: llc -march=amdgcn --misched=ilpmax -mcpu=SI -verify-machineinstrs < %s | FileCheck -strict-whitespace %s --check-prefix=ILPMAX +; RUN: llc -march=amdgcn --misched=ilpmax -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace %s --check-prefix=ILPMAX +; The ilpmax scheduler is used for the second test to get the ordering we want for the test. -; CHECK-LABEL: {{^}}main: -; CHECK: s_load_dwordx4 -; CHECK: s_load_dwordx4 -; CHECK: s_waitcnt vmcnt(0) lgkmcnt(0){{$}} -; CHECK: s_endpgm +; DEFAULT-LABEL: {{^}}main: +; DEFAULT: s_load_dwordx4 +; DEFAULT: s_load_dwordx4 +; DEFAULT: s_waitcnt vmcnt(0) +; DEFAULT: exp +; DEFAULT: s_waitcnt lgkmcnt(0) +; DEFAULT: s_endpgm define void @main(<16 x i8> addrspace(2)* inreg %arg, <16 x i8> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, <16 x i8> addrspace(2)* inreg %arg3, <16 x i8> addrspace(2)* inreg %arg4, i32 inreg %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, float addrspace(2)* inreg %constptr) #0 { main_body: %tmp = getelementptr <16 x i8>, <16 x i8> addrspace(2)* %arg3, i32 0 @@ -29,6 +34,42 @@ main_body: ret void } +; ILPMAX-LABEL: {{^}}main2: +; ILPMAX: s_load_dwordx4 +; ILPMAX: s_waitcnt lgkmcnt(0) +; ILPMAX: buffer_load +; ILPMAX: s_load_dwordx4 +; ILPMAX: s_waitcnt lgkmcnt(0) +; ILPMAX: buffer_load +; ILPMAX: s_waitcnt vmcnt(1) +; ILPMAX: s_waitcnt vmcnt(0) +; ILPMAX: s_endpgm + +define void @main2([6 x <16 x i8>] addrspace(2)* byval, [17 x <16 x i8>] addrspace(2)* byval, [17 x <4 x i32>] addrspace(2)* byval, [34 x <8 x i32>] addrspace(2)* byval, [16 x <16 x i8>] addrspace(2)* +byval, i32 inreg, i32 inreg, i32, i32, i32, i32) #0 { +main_body: + %11 = getelementptr [16 x <16 x i8>], [16 x <16 x i8>] addrspace(2)* %4, i64 0, i64 0 + %12 = load <16 x i8>, <16 x i8> addrspace(2)* %11, align 16, !tbaa !0 + %13 = add i32 %5, %7 + %14 = call <4 x float> @llvm.SI.vs.load.input(<16 x i8> %12, i32 0, i32 %13) + %15 = extractelement <4 x float> %14, i32 0 + %16 = extractelement <4 x float> %14, i32 1 + %17 = extractelement <4 x float> %14, i32 2 + %18 = extractelement <4 x float> %14, i32 3 + %19 = getelementptr [16 x <16 x i8>], [16 x <16 x i8>] addrspace(2)* %4, i64 0, i64 1 + %20 = load <16 x i8>, <16 x i8> addrspace(2)* %19, align 16, !tbaa !0 + %21 = add i32 %5, %7 + %22 = call <4 x float> @llvm.SI.vs.load.input(<16 x i8> %20, i32 0, i32 %21) + %23 = extractelement <4 x float> %22, i32 0 + %24 = extractelement <4 x float> %22, i32 1 + %25 = extractelement <4 x float> %22, i32 2 + %26 = extractelement <4 x float> %22, i32 3 + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 12, i32 0, float %15, float %16, float %17, float %18) + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 32, i32 0, float %23, float %24, float %25, float %26) + ret void +} + + ; Function Attrs: noduplicate nounwind declare void @llvm.AMDGPU.barrier.global() #1