2012-12-11 22:25:42 +01:00
|
|
|
//===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
|
|
|
|
|
//
|
2019-01-19 09:50:56 +01:00
|
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
2012-12-11 22:25:42 +01:00
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
//
|
|
|
|
|
/// \file
|
2018-05-01 17:54:18 +02:00
|
|
|
/// Implements the AMDGPU specific subclass of TargetSubtarget.
|
2012-12-11 22:25:42 +01:00
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
#include "AMDGPUSubtarget.h"
|
2017-07-05 20:40:56 +02:00
|
|
|
#include "AMDGPU.h"
|
|
|
|
|
#include "AMDGPUCallLowering.h"
|
|
|
|
|
#include "AMDGPUInstructionSelector.h"
|
|
|
|
|
#include "AMDGPULegalizerInfo.h"
|
|
|
|
|
#include "AMDGPURegisterBankInfo.h"
|
2020-12-25 16:52:14 +01:00
|
|
|
#include "AMDGPUTargetMachine.h"
|
2017-02-08 14:02:33 +01:00
|
|
|
#include "SIMachineFunctionInfo.h"
|
2020-11-30 18:06:35 +01:00
|
|
|
#include "Utils/AMDGPUBaseInfo.h"
|
2014-07-13 04:08:26 +02:00
|
|
|
#include "llvm/ADT/SmallString.h"
|
2020-12-25 16:52:14 +01:00
|
|
|
#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
|
2015-01-29 17:55:25 +01:00
|
|
|
#include "llvm/CodeGen/MachineScheduler.h"
|
2017-11-03 23:32:11 +01:00
|
|
|
#include "llvm/CodeGen/TargetFrameLowering.h"
|
2020-12-25 16:52:14 +01:00
|
|
|
#include "llvm/IR/IntrinsicsAMDGPU.h"
|
|
|
|
|
#include "llvm/IR/IntrinsicsR600.h"
|
|
|
|
|
#include "llvm/IR/MDBuilder.h"
|
|
|
|
|
#include "llvm/MC/MCSubtargetInfo.h"
|
2016-12-12 23:23:53 +01:00
|
|
|
#include <algorithm>
|
2014-07-13 04:08:26 +02:00
|
|
|
|
2012-12-11 22:25:42 +01:00
|
|
|
using namespace llvm;
|
|
|
|
|
|
2014-04-22 00:55:11 +02:00
|
|
|
#define DEBUG_TYPE "amdgpu-subtarget"
|
|
|
|
|
|
2012-12-11 22:25:42 +01:00
|
|
|
#define GET_SUBTARGETINFO_TARGET_DESC
|
|
|
|
|
#define GET_SUBTARGETINFO_CTOR
|
2018-07-11 22:59:01 +02:00
|
|
|
#define AMDGPUSubtarget GCNSubtarget
|
2012-12-11 22:25:42 +01:00
|
|
|
#include "AMDGPUGenSubtargetInfo.inc"
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
#define GET_SUBTARGETINFO_TARGET_DESC
|
|
|
|
|
#define GET_SUBTARGETINFO_CTOR
|
2018-07-11 22:59:01 +02:00
|
|
|
#undef AMDGPUSubtarget
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
#include "R600GenSubtargetInfo.inc"
|
2012-12-11 22:25:42 +01:00
|
|
|
|
2019-07-11 23:25:00 +02:00
|
|
|
static cl::opt<bool> DisablePowerSched(
|
|
|
|
|
"amdgpu-disable-power-sched",
|
|
|
|
|
cl::desc("Disable scheduling to minimize mAI power bursts"),
|
|
|
|
|
cl::init(false));
|
|
|
|
|
|
2020-01-02 22:45:33 +01:00
|
|
|
static cl::opt<bool> EnableVGPRIndexMode(
|
|
|
|
|
"amdgpu-vgpr-index-mode",
|
|
|
|
|
cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
|
|
|
|
|
cl::init(false));
|
|
|
|
|
|
2020-10-21 23:27:03 +02:00
|
|
|
static cl::opt<bool> EnableFlatScratch(
|
|
|
|
|
"amdgpu-enable-flat-scratch",
|
|
|
|
|
cl::desc("Use flat scratch instructions"),
|
|
|
|
|
cl::init(false));
|
|
|
|
|
|
2020-10-12 05:51:53 +02:00
|
|
|
static cl::opt<bool> UseAA("amdgpu-use-aa-in-codegen",
|
|
|
|
|
cl::desc("Enable the use of AA during codegen."),
|
|
|
|
|
cl::init(true));
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
GCNSubtarget::~GCNSubtarget() = default;
|
2016-06-24 08:30:11 +02:00
|
|
|
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
R600Subtarget &
|
|
|
|
|
R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
|
|
|
|
|
StringRef GPU, StringRef FS) {
|
2019-03-29 20:14:54 +01:00
|
|
|
SmallString<256> FullFS("+promote-alloca,");
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
FullFS += FS;
|
2020-08-14 23:56:54 +02:00
|
|
|
ParseSubtargetFeatures(GPU, /*TuneCPU*/ GPU, FullFS);
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
|
|
|
|
|
HasMulU24 = getGeneration() >= EVERGREEN;
|
|
|
|
|
HasMulI24 = hasCaymanISA();
|
|
|
|
|
|
|
|
|
|
return *this;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
GCNSubtarget &
|
|
|
|
|
GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
|
2019-04-03 03:58:57 +02:00
|
|
|
StringRef GPU, StringRef FS) {
|
2014-07-26 00:22:39 +02:00
|
|
|
// Determine default and user-specified characteristics
|
2014-07-15 01:40:49 +02:00
|
|
|
//
|
|
|
|
|
// We want to be able to turn these off, but making this a subtarget feature
|
|
|
|
|
// for SI has the unhelpful behavior that it unsets everything else if you
|
|
|
|
|
// disable it.
|
[AMDGPU] Add support for TFE/LWE in image intrinsics. 2nd try
TFE and LWE support requires extra result registers that are written in the
event of a failure in order to detect that failure case.
The specific use-case that initiated these changes is sparse texture support.
This means that if image intrinsics are used with either option turned on, the
programmer must ensure that the return type can contain all of the expected
results. This can result in redundant registers since the vector size must be a
power-of-2.
This change takes roughly 6 parts:
1. Modify the instruction defs in tablegen to add new instruction variants that
can accomodate the extra return values.
2. Updates to lowerImage in SIISelLowering.cpp to accomodate setting TFE or LWE
(where the bulk of the work for these instruction types is now done)
3. Extra verification code to catch cases where intrinsics have been used but
insufficient return registers are used.
4. Modification to the adjustWritemask optimisation to account for TFE/LWE being
enabled (requires extra registers to be maintained for error return value).
5. An extra pass to zero initialize the error value return - this is because if
the error does not occur, the register is not written and thus must be zeroed
before use. Also added a new (on by default) option to ensure ALL return values
are zero-initialized that is required for sparse texture support.
6. Disable the inst_combine optimization in the presence of tfe/lwe (later TODO
for this to re-enable and handle correctly).
There's an additional fix now to avoid a dmask=0
For an image intrinsic with tfe where all result channels except tfe
were unused, I was getting an image instruction with dmask=0 and only a
single vgpr result for tfe. That is incorrect because the hardware
assumes there is at least one vgpr result, plus the one for tfe.
Fixed by forcing dmask to 1, which gives the desired two vgpr result
with tfe in the second one.
The TFE or LWE result is returned from the intrinsics using an aggregate
type. Look in the test code provided to see how this works, but in essence IR
code to invoke the intrinsic looks as follows:
%v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15,
i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue {<4 x float>, i32} %v, 0
%v.err = extractvalue {<4 x float>, i32} %v, 1
This re-submit of the change also includes a slight modification in
SIISelLowering.cpp to work-around a compiler bug for the powerpc_le
platform that caused a buildbot failure on a previous submission.
Differential revision: https://reviews.llvm.org/D48826
Change-Id: If222bc03642e76cf98059a6bef5d5bffeda38dda
Work around for ppcle compiler bug
Change-Id: Ie284cf24b2271215be1b9dc95b485fd15000e32b
llvm-svn: 351054
2019-01-14 12:55:24 +01:00
|
|
|
//
|
|
|
|
|
// Similarly we want enable-prt-strict-null to be on by default and not to
|
|
|
|
|
// unset everything else if it is disabled
|
2014-07-13 04:08:26 +02:00
|
|
|
|
2020-11-30 18:06:35 +01:00
|
|
|
SmallString<256> FullFS("+promote-alloca,+load-store-opt,+enable-ds128,");
|
2017-12-04 23:57:29 +01:00
|
|
|
|
2020-12-24 10:44:01 +01:00
|
|
|
// Turn on features that HSA ABI requires. Also turn on FlatForGlobal by default
|
|
|
|
|
if (isAmdHsaOS())
|
2020-11-11 15:33:55 +01:00
|
|
|
FullFS += "+flat-for-global,+unaligned-access-mode,+trap-handler,";
|
2017-01-23 23:31:03 +01:00
|
|
|
|
[AMDGPU] Add support for TFE/LWE in image intrinsics. 2nd try
TFE and LWE support requires extra result registers that are written in the
event of a failure in order to detect that failure case.
The specific use-case that initiated these changes is sparse texture support.
This means that if image intrinsics are used with either option turned on, the
programmer must ensure that the return type can contain all of the expected
results. This can result in redundant registers since the vector size must be a
power-of-2.
This change takes roughly 6 parts:
1. Modify the instruction defs in tablegen to add new instruction variants that
can accomodate the extra return values.
2. Updates to lowerImage in SIISelLowering.cpp to accomodate setting TFE or LWE
(where the bulk of the work for these instruction types is now done)
3. Extra verification code to catch cases where intrinsics have been used but
insufficient return registers are used.
4. Modification to the adjustWritemask optimisation to account for TFE/LWE being
enabled (requires extra registers to be maintained for error return value).
5. An extra pass to zero initialize the error value return - this is because if
the error does not occur, the register is not written and thus must be zeroed
before use. Also added a new (on by default) option to ensure ALL return values
are zero-initialized that is required for sparse texture support.
6. Disable the inst_combine optimization in the presence of tfe/lwe (later TODO
for this to re-enable and handle correctly).
There's an additional fix now to avoid a dmask=0
For an image intrinsic with tfe where all result channels except tfe
were unused, I was getting an image instruction with dmask=0 and only a
single vgpr result for tfe. That is incorrect because the hardware
assumes there is at least one vgpr result, plus the one for tfe.
Fixed by forcing dmask to 1, which gives the desired two vgpr result
with tfe in the second one.
The TFE or LWE result is returned from the intrinsics using an aggregate
type. Look in the test code provided to see how this works, but in essence IR
code to invoke the intrinsic looks as follows:
%v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15,
i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue {<4 x float>, i32} %v, 0
%v.err = extractvalue {<4 x float>, i32} %v, 1
This re-submit of the change also includes a slight modification in
SIISelLowering.cpp to work-around a compiler bug for the powerpc_le
platform that caused a buildbot failure on a previous submission.
Differential revision: https://reviews.llvm.org/D48826
Change-Id: If222bc03642e76cf98059a6bef5d5bffeda38dda
Work around for ppcle compiler bug
Change-Id: Ie284cf24b2271215be1b9dc95b485fd15000e32b
llvm-svn: 351054
2019-01-14 12:55:24 +01:00
|
|
|
FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
|
|
|
|
|
|
2019-06-13 21:18:29 +02:00
|
|
|
// Disable mutually exclusive bits.
|
2021-06-23 13:52:36 +02:00
|
|
|
if (FS.find_insensitive("+wavefrontsize") != StringRef::npos) {
|
|
|
|
|
if (FS.find_insensitive("wavefrontsize16") == StringRef::npos)
|
2019-06-13 21:18:29 +02:00
|
|
|
FullFS += "-wavefrontsize16,";
|
2021-06-23 13:52:36 +02:00
|
|
|
if (FS.find_insensitive("wavefrontsize32") == StringRef::npos)
|
2019-06-13 21:18:29 +02:00
|
|
|
FullFS += "-wavefrontsize32,";
|
2021-06-23 13:52:36 +02:00
|
|
|
if (FS.find_insensitive("wavefrontsize64") == StringRef::npos)
|
2019-06-13 21:18:29 +02:00
|
|
|
FullFS += "-wavefrontsize64,";
|
|
|
|
|
}
|
|
|
|
|
|
2014-07-13 04:08:26 +02:00
|
|
|
FullFS += FS;
|
|
|
|
|
|
2020-08-14 23:56:54 +02:00
|
|
|
ParseSubtargetFeatures(GPU, /*TuneCPU*/ GPU, FullFS);
|
2014-06-13 03:32:00 +02:00
|
|
|
|
2020-12-24 10:44:01 +01:00
|
|
|
// Implement the "generic" processors, which acts as the default when no
|
|
|
|
|
// generation features are enabled (e.g for -mcpu=''). HSA OS defaults to
|
|
|
|
|
// the first amdgcn target that supports flat addressing. Other OSes defaults
|
|
|
|
|
// to the first amdgcn target.
|
|
|
|
|
if (Gen == AMDGPUSubtarget::INVALID) {
|
|
|
|
|
Gen = TT.getOS() == Triple::AMDHSA ? AMDGPUSubtarget::SEA_ISLANDS
|
|
|
|
|
: AMDGPUSubtarget::SOUTHERN_ISLANDS;
|
|
|
|
|
}
|
|
|
|
|
|
2017-12-04 23:57:29 +01:00
|
|
|
// We don't support FP64 for EG/NI atm.
|
|
|
|
|
assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
|
|
|
|
|
|
2020-12-24 10:44:01 +01:00
|
|
|
// Targets must either support 64-bit offsets for MUBUF instructions, and/or
|
|
|
|
|
// support flat operations, otherwise they cannot access a 64-bit global
|
|
|
|
|
// address space
|
|
|
|
|
assert(hasAddr64() || hasFlat());
|
|
|
|
|
// Unless +-flat-for-global is specified, turn on FlatForGlobal for targets
|
|
|
|
|
// that do not support ADDR64 variants of MUBUF instructions. Such targets
|
|
|
|
|
// cannot use a 64 bit offset with a MUBUF instruction to access the global
|
|
|
|
|
// address space
|
|
|
|
|
if (!hasAddr64() && !FS.contains("flat-for-global") && !FlatForGlobal) {
|
|
|
|
|
ToggleFeature(AMDGPU::FeatureFlatForGlobal);
|
2017-01-27 18:42:26 +01:00
|
|
|
FlatForGlobal = true;
|
|
|
|
|
}
|
2020-12-24 10:44:01 +01:00
|
|
|
// Unless +-flat-for-global is specified, use MUBUF instructions for global
|
|
|
|
|
// address space access if flat operations are not available.
|
|
|
|
|
if (!hasFlat() && !FS.contains("flat-for-global") && FlatForGlobal) {
|
|
|
|
|
ToggleFeature(AMDGPU::FeatureFlatForGlobal);
|
|
|
|
|
FlatForGlobal = false;
|
|
|
|
|
}
|
2017-01-27 18:42:26 +01:00
|
|
|
|
2016-02-12 03:40:47 +01:00
|
|
|
// Set defaults if needed.
|
|
|
|
|
if (MaxPrivateElementSize == 0)
|
2016-05-11 02:28:54 +02:00
|
|
|
MaxPrivateElementSize = 4;
|
2016-02-12 03:40:47 +01:00
|
|
|
|
2017-08-07 16:58:04 +02:00
|
|
|
if (LDSBankCount == 0)
|
|
|
|
|
LDSBankCount = 32;
|
|
|
|
|
|
|
|
|
|
if (TT.getArch() == Triple::amdgcn) {
|
|
|
|
|
if (LocalMemorySize == 0)
|
|
|
|
|
LocalMemorySize = 32768;
|
|
|
|
|
|
|
|
|
|
// Do something sensible for unspecified target.
|
|
|
|
|
if (!HasMovrel && !HasVGPRIndexMode)
|
|
|
|
|
HasMovrel = true;
|
|
|
|
|
}
|
|
|
|
|
|
2019-02-08 20:18:01 +01:00
|
|
|
// Don't crash on invalid devices.
|
2020-06-01 19:16:58 +02:00
|
|
|
if (WavefrontSizeLog2 == 0)
|
|
|
|
|
WavefrontSizeLog2 = 5;
|
2019-02-08 20:18:01 +01:00
|
|
|
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
|
2021-05-13 15:08:36 +02:00
|
|
|
HasSMulHi = getGeneration() >= AMDGPUSubtarget::GFX9;
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
|
2020-11-30 18:06:35 +01:00
|
|
|
TargetID.setTargetIDFromFeaturesString(FS);
|
2019-05-16 16:48:34 +02:00
|
|
|
|
2020-11-30 18:06:35 +01:00
|
|
|
LLVM_DEBUG(dbgs() << "xnack setting for subtarget: "
|
|
|
|
|
<< TargetID.getXnackSetting() << '\n');
|
|
|
|
|
LLVM_DEBUG(dbgs() << "sramecc setting for subtarget: "
|
|
|
|
|
<< TargetID.getSramEccSetting() << '\n');
|
2019-04-03 03:58:57 +02:00
|
|
|
|
2014-07-26 00:22:39 +02:00
|
|
|
return *this;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-12 20:50:47 +02:00
|
|
|
AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) :
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
TargetTriple(TT),
|
2021-02-17 22:37:46 +01:00
|
|
|
GCN3Encoding(false),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
Has16BitInsts(false),
|
|
|
|
|
HasMadMixInsts(false),
|
2020-06-15 23:10:39 +02:00
|
|
|
HasMadMacF32Insts(false),
|
|
|
|
|
HasDsSrc2Insts(false),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
HasSDWA(false),
|
|
|
|
|
HasVOP3PInsts(false),
|
|
|
|
|
HasMulI24(true),
|
|
|
|
|
HasMulU24(true),
|
2021-05-13 15:08:36 +02:00
|
|
|
HasSMulHi(false),
|
2018-08-15 23:03:55 +02:00
|
|
|
HasInv2PiInlineImm(false),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
HasFminFmaxLegacy(true),
|
|
|
|
|
EnablePromoteAlloca(false),
|
2018-09-14 12:27:19 +02:00
|
|
|
HasTrigReducedRange(false),
|
2019-07-31 03:07:10 +02:00
|
|
|
MaxWavesPerEU(10),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
LocalMemorySize(0),
|
2020-06-01 19:16:58 +02:00
|
|
|
WavefrontSizeLog2(0)
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
{ }
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
|
2021-05-03 10:14:12 +02:00
|
|
|
const GCNTargetMachine &TM)
|
|
|
|
|
: // clang-format off
|
2020-08-14 23:56:54 +02:00
|
|
|
AMDGPUGenSubtargetInfo(TT, GPU, /*TuneCPU*/ GPU, FS),
|
2018-09-12 20:50:47 +02:00
|
|
|
AMDGPUSubtarget(TT),
|
2016-06-24 08:30:11 +02:00
|
|
|
TargetTriple(TT),
|
2020-11-30 18:06:35 +01:00
|
|
|
TargetID(*this),
|
2020-12-24 10:44:01 +01:00
|
|
|
Gen(INVALID),
|
2018-09-17 18:04:32 +02:00
|
|
|
InstrItins(getInstrItineraryForCPU(GPU)),
|
2016-06-24 08:30:11 +02:00
|
|
|
LDSBankCount(0),
|
|
|
|
|
MaxPrivateElementSize(0),
|
|
|
|
|
|
|
|
|
|
FastFMAF32(false),
|
2018-08-15 21:45:04 +02:00
|
|
|
FastDenormalF32(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
HalfRate64Ops(false),
|
2021-02-17 22:37:46 +01:00
|
|
|
FullRate64Ops(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
|
|
|
|
|
FlatForGlobal(false),
|
2017-06-02 19:40:26 +02:00
|
|
|
AutoWaitcntBeforeBarrier(false),
|
2016-10-14 20:10:39 +02:00
|
|
|
UnalignedScratchAccess(false),
|
2020-08-21 11:29:32 +02:00
|
|
|
UnalignedAccessMode(false),
|
2016-07-02 01:03:44 +02:00
|
|
|
|
2017-02-18 19:29:53 +01:00
|
|
|
HasApertureRegs(false),
|
2020-11-30 18:06:35 +01:00
|
|
|
SupportsXNACK(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
EnableXNACK(false),
|
2021-02-17 22:37:46 +01:00
|
|
|
EnableTgSplit(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
EnableCuMode(false),
|
2017-02-10 03:15:29 +01:00
|
|
|
TrapHandler(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
|
|
|
|
|
EnableLoadStoreOpt(false),
|
|
|
|
|
EnableUnsafeDSOffsetFolding(false),
|
|
|
|
|
EnableSIScheduler(false),
|
2018-04-11 00:48:23 +02:00
|
|
|
EnableDS128(false),
|
[AMDGPU] Add support for TFE/LWE in image intrinsics. 2nd try
TFE and LWE support requires extra result registers that are written in the
event of a failure in order to detect that failure case.
The specific use-case that initiated these changes is sparse texture support.
This means that if image intrinsics are used with either option turned on, the
programmer must ensure that the return type can contain all of the expected
results. This can result in redundant registers since the vector size must be a
power-of-2.
This change takes roughly 6 parts:
1. Modify the instruction defs in tablegen to add new instruction variants that
can accomodate the extra return values.
2. Updates to lowerImage in SIISelLowering.cpp to accomodate setting TFE or LWE
(where the bulk of the work for these instruction types is now done)
3. Extra verification code to catch cases where intrinsics have been used but
insufficient return registers are used.
4. Modification to the adjustWritemask optimisation to account for TFE/LWE being
enabled (requires extra registers to be maintained for error return value).
5. An extra pass to zero initialize the error value return - this is because if
the error does not occur, the register is not written and thus must be zeroed
before use. Also added a new (on by default) option to ensure ALL return values
are zero-initialized that is required for sparse texture support.
6. Disable the inst_combine optimization in the presence of tfe/lwe (later TODO
for this to re-enable and handle correctly).
There's an additional fix now to avoid a dmask=0
For an image intrinsic with tfe where all result channels except tfe
were unused, I was getting an image instruction with dmask=0 and only a
single vgpr result for tfe. That is incorrect because the hardware
assumes there is at least one vgpr result, plus the one for tfe.
Fixed by forcing dmask to 1, which gives the desired two vgpr result
with tfe in the second one.
The TFE or LWE result is returned from the intrinsics using an aggregate
type. Look in the test code provided to see how this works, but in essence IR
code to invoke the intrinsic looks as follows:
%v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15,
i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
%v.vec = extractvalue {<4 x float>, i32} %v, 0
%v.err = extractvalue {<4 x float>, i32} %v, 1
This re-submit of the change also includes a slight modification in
SIISelLowering.cpp to work-around a compiler bug for the powerpc_le
platform that caused a buildbot failure on a previous submission.
Differential revision: https://reviews.llvm.org/D48826
Change-Id: If222bc03642e76cf98059a6bef5d5bffeda38dda
Work around for ppcle compiler bug
Change-Id: Ie284cf24b2271215be1b9dc95b485fd15000e32b
llvm-svn: 351054
2019-01-14 12:55:24 +01:00
|
|
|
EnablePRTStrictNull(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
DumpCode(false),
|
|
|
|
|
|
|
|
|
|
FP64(false),
|
|
|
|
|
CIInsts(false),
|
2019-04-05 20:24:34 +02:00
|
|
|
GFX8Insts(false),
|
2017-02-18 20:12:26 +01:00
|
|
|
GFX9Insts(false),
|
2021-02-17 22:37:46 +01:00
|
|
|
GFX90AInsts(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
GFX10Insts(false),
|
2020-06-15 23:10:39 +02:00
|
|
|
GFX10_3Insts(false),
|
2019-04-05 20:24:34 +02:00
|
|
|
GFX7GFX8GFX9Insts(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
SGPRInitBug(false),
|
2021-05-03 10:14:12 +02:00
|
|
|
NegativeScratchOffsetBug(false),
|
|
|
|
|
NegativeUnalignedScratchOffsetBug(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
HasSMemRealTime(false),
|
2017-08-16 15:51:56 +02:00
|
|
|
HasIntClamp(false),
|
2018-04-30 21:08:16 +02:00
|
|
|
HasFmaMixInsts(false),
|
2016-10-12 20:00:51 +02:00
|
|
|
HasMovrel(false),
|
|
|
|
|
HasVGPRIndexMode(false),
|
2016-10-29 06:05:06 +02:00
|
|
|
HasScalarStores(false),
|
2018-04-02 18:10:25 +02:00
|
|
|
HasScalarAtomics(false),
|
2017-06-22 08:26:41 +02:00
|
|
|
HasSDWAOmod(false),
|
|
|
|
|
HasSDWAScalar(false),
|
|
|
|
|
HasSDWASdst(false),
|
|
|
|
|
HasSDWAMac(false),
|
[AMDGPU] SDWA: several fixes for V_CVT and VOPC instructions
Summary:
1. Instruction V_CVT_U32_F32 allow omod operand (see SIInstrInfo.td:1435). In fact this operand shouldn't be allowed here. This fix checks if SDWA pseudo instruction has OMod operand and then copy it.
2. There were several problems with support of VOPC instructions in SDWA peephole pass.
Reviewers: tstellar, arsenm, vpykhtin, airlied, kzhuravl
Subscribers: wdng, nhaehnle, yaxunl, dstuttard, tpr, sarnex, t-tye
Differential Revision: https://reviews.llvm.org/D34626
llvm-svn: 306413
2017-06-27 17:02:23 +02:00
|
|
|
HasSDWAOutModsVOPC(false),
|
2017-01-20 11:01:25 +01:00
|
|
|
HasDPP(false),
|
2019-06-12 20:02:41 +02:00
|
|
|
HasDPP8(false),
|
2021-02-17 22:37:46 +01:00
|
|
|
Has64BitDPP(false),
|
|
|
|
|
HasPackedFP32Ops(false),
|
|
|
|
|
HasExtendedImageInsts(false),
|
2018-08-28 17:07:30 +02:00
|
|
|
HasR128A16(false),
|
2020-01-29 10:04:36 +01:00
|
|
|
HasGFX10A16(false),
|
2020-03-25 08:38:22 +01:00
|
|
|
HasG16(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
HasNSAEncoding(false),
|
2021-07-22 03:43:25 +02:00
|
|
|
NSAMaxSize(0),
|
2021-06-08 22:54:42 +02:00
|
|
|
GFX10_AEncoding(false),
|
2020-06-15 23:10:39 +02:00
|
|
|
GFX10_BEncoding(false),
|
2018-04-30 21:08:16 +02:00
|
|
|
HasDLInsts(false),
|
2019-02-09 01:34:21 +01:00
|
|
|
HasDot1Insts(false),
|
|
|
|
|
HasDot2Insts(false),
|
2019-07-09 20:10:06 +02:00
|
|
|
HasDot3Insts(false),
|
|
|
|
|
HasDot4Insts(false),
|
2019-06-14 02:33:31 +02:00
|
|
|
HasDot5Insts(false),
|
|
|
|
|
HasDot6Insts(false),
|
2021-03-16 17:01:03 +01:00
|
|
|
HasDot7Insts(false),
|
2019-07-09 20:10:06 +02:00
|
|
|
HasMAIInsts(false),
|
|
|
|
|
HasPkFmacF16Inst(false),
|
|
|
|
|
HasAtomicFaddInsts(false),
|
2020-11-30 18:06:35 +01:00
|
|
|
SupportsSRAMECC(false),
|
2018-11-05 23:44:19 +01:00
|
|
|
EnableSRAMECC(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
HasNoSdstCMPX(false),
|
|
|
|
|
HasVscnt(false),
|
2020-06-15 23:10:39 +02:00
|
|
|
HasGetWaveIdInst(false),
|
|
|
|
|
HasSMemTimeInst(false),
|
2021-03-06 09:52:23 +01:00
|
|
|
HasShaderCyclesRegister(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
HasRegisterBanking(false),
|
|
|
|
|
HasVOP3Literal(false),
|
|
|
|
|
HasNoDataDepHazard(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
FlatAddressSpace(false),
|
2017-05-10 23:19:05 +02:00
|
|
|
FlatInstOffsets(false),
|
|
|
|
|
FlatGlobalInsts(false),
|
|
|
|
|
FlatScratchInsts(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
ScalarFlatScratchInsts(false),
|
2021-04-12 23:40:17 +02:00
|
|
|
HasArchitectedFlatScratch(false),
|
2017-07-20 19:42:47 +02:00
|
|
|
AddNoCarryInsts(false),
|
2018-01-12 22:12:19 +01:00
|
|
|
HasUnpackedD16VMem(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
LDSMisalignedBug(false),
|
2019-08-24 00:09:58 +02:00
|
|
|
HasMFMAInlineLiteralBug(false),
|
2020-11-11 15:33:55 +01:00
|
|
|
UnalignedBufferAccess(false),
|
2020-08-21 11:29:32 +02:00
|
|
|
UnalignedDSAccess(false),
|
2021-02-17 22:37:46 +01:00
|
|
|
HasPackedTID(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
|
2016-12-08 18:28:47 +01:00
|
|
|
ScalarizeGlobal(false),
|
2016-06-24 08:30:11 +02:00
|
|
|
|
2019-04-24 19:03:15 +02:00
|
|
|
HasVcmpxPermlaneHazard(false),
|
|
|
|
|
HasVMEMtoScalarWriteHazard(false),
|
|
|
|
|
HasSMEMtoVectorWriteHazard(false),
|
|
|
|
|
HasInstFwdPrefetchBug(false),
|
|
|
|
|
HasVcmpxExecWARHazard(false),
|
|
|
|
|
HasLdsBranchVmemWARHazard(false),
|
|
|
|
|
HasNSAtoVMEMBug(false),
|
2021-05-14 05:29:54 +02:00
|
|
|
HasNSAClauseBug(false),
|
2019-06-26 19:34:57 +02:00
|
|
|
HasOffset3fBug(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
HasFlatSegmentOffsetBug(false),
|
[AMDGPU] Implement hardware bug workaround for image instructions
Summary:
This implements a workaround for a hardware bug in gfx8 and gfx9,
where register usage is not estimated correctly for image_store and
image_gather4 instructions when D16 is used.
Change-Id: I4e30744da6796acac53a9b5ad37ac1c2035c8899
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D81172
2020-04-03 23:37:51 +02:00
|
|
|
HasImageStoreD16Bug(false),
|
|
|
|
|
HasImageGather4D16Bug(false),
|
2019-04-24 19:03:15 +02:00
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
FeatureDisable(false),
|
2018-07-12 00:15:15 +02:00
|
|
|
InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
|
2018-07-20 11:05:08 +02:00
|
|
|
TLInfo(TM, *this),
|
2018-07-11 22:59:01 +02:00
|
|
|
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0) {
|
2021-05-03 10:14:12 +02:00
|
|
|
// clang-format on
|
2019-07-31 03:07:10 +02:00
|
|
|
MaxWavesPerEU = AMDGPU::IsaInfo::getMaxWavesPerEU(this);
|
2018-07-11 22:59:01 +02:00
|
|
|
CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
|
2020-05-29 22:28:22 +02:00
|
|
|
InlineAsmLoweringInfo.reset(new InlineAsmLowering(getTargetLowering()));
|
2018-07-11 22:59:01 +02:00
|
|
|
Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
|
2019-09-19 18:26:14 +02:00
|
|
|
RegBankInfo.reset(new AMDGPURegisterBankInfo(*this));
|
2018-07-11 22:59:01 +02:00
|
|
|
InstSelector.reset(new AMDGPUInstructionSelector(
|
|
|
|
|
*this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
|
2014-01-22 22:55:43 +01:00
|
|
|
}
|
2014-12-02 23:00:07 +01:00
|
|
|
|
2020-10-21 23:27:03 +02:00
|
|
|
bool GCNSubtarget::enableFlatScratch() const {
|
2021-04-12 23:40:17 +02:00
|
|
|
return flatScratchIsArchitected() ||
|
|
|
|
|
(EnableFlatScratch && hasFlatScratchInsts());
|
2020-10-21 23:27:03 +02:00
|
|
|
}
|
|
|
|
|
|
2019-05-02 05:47:23 +02:00
|
|
|
unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
|
|
|
|
|
if (getGeneration() < GFX10)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
switch (Opcode) {
|
2021-01-07 19:56:02 +01:00
|
|
|
case AMDGPU::V_LSHLREV_B64_e64:
|
2019-05-02 05:47:23 +02:00
|
|
|
case AMDGPU::V_LSHLREV_B64_gfx10:
|
2021-01-07 19:56:02 +01:00
|
|
|
case AMDGPU::V_LSHL_B64_e64:
|
|
|
|
|
case AMDGPU::V_LSHRREV_B64_e64:
|
2019-05-02 05:47:23 +02:00
|
|
|
case AMDGPU::V_LSHRREV_B64_gfx10:
|
2021-01-07 19:56:02 +01:00
|
|
|
case AMDGPU::V_LSHR_B64_e64:
|
|
|
|
|
case AMDGPU::V_ASHRREV_I64_e64:
|
2019-05-02 05:47:23 +02:00
|
|
|
case AMDGPU::V_ASHRREV_I64_gfx10:
|
2021-01-07 19:56:02 +01:00
|
|
|
case AMDGPU::V_ASHR_I64_e64:
|
2019-05-02 05:47:23 +02:00
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 2;
|
|
|
|
|
}
|
|
|
|
|
|
2021-06-16 00:51:06 +02:00
|
|
|
/// This list was mostly derived from experimentation.
|
|
|
|
|
bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const {
|
|
|
|
|
switch (Opcode) {
|
|
|
|
|
case AMDGPU::V_CVT_F16_F32_e32:
|
|
|
|
|
case AMDGPU::V_CVT_F16_F32_e64:
|
|
|
|
|
case AMDGPU::V_CVT_F16_U16_e32:
|
|
|
|
|
case AMDGPU::V_CVT_F16_U16_e64:
|
|
|
|
|
case AMDGPU::V_CVT_F16_I16_e32:
|
|
|
|
|
case AMDGPU::V_CVT_F16_I16_e64:
|
|
|
|
|
case AMDGPU::V_RCP_F16_e64:
|
|
|
|
|
case AMDGPU::V_RCP_F16_e32:
|
|
|
|
|
case AMDGPU::V_RSQ_F16_e64:
|
|
|
|
|
case AMDGPU::V_RSQ_F16_e32:
|
|
|
|
|
case AMDGPU::V_SQRT_F16_e64:
|
|
|
|
|
case AMDGPU::V_SQRT_F16_e32:
|
|
|
|
|
case AMDGPU::V_LOG_F16_e64:
|
|
|
|
|
case AMDGPU::V_LOG_F16_e32:
|
|
|
|
|
case AMDGPU::V_EXP_F16_e64:
|
|
|
|
|
case AMDGPU::V_EXP_F16_e32:
|
|
|
|
|
case AMDGPU::V_SIN_F16_e64:
|
|
|
|
|
case AMDGPU::V_SIN_F16_e32:
|
|
|
|
|
case AMDGPU::V_COS_F16_e64:
|
|
|
|
|
case AMDGPU::V_COS_F16_e32:
|
|
|
|
|
case AMDGPU::V_FLOOR_F16_e64:
|
|
|
|
|
case AMDGPU::V_FLOOR_F16_e32:
|
|
|
|
|
case AMDGPU::V_CEIL_F16_e64:
|
|
|
|
|
case AMDGPU::V_CEIL_F16_e32:
|
|
|
|
|
case AMDGPU::V_TRUNC_F16_e64:
|
|
|
|
|
case AMDGPU::V_TRUNC_F16_e32:
|
|
|
|
|
case AMDGPU::V_RNDNE_F16_e64:
|
|
|
|
|
case AMDGPU::V_RNDNE_F16_e32:
|
|
|
|
|
case AMDGPU::V_FRACT_F16_e64:
|
|
|
|
|
case AMDGPU::V_FRACT_F16_e32:
|
|
|
|
|
case AMDGPU::V_FREXP_MANT_F16_e64:
|
|
|
|
|
case AMDGPU::V_FREXP_MANT_F16_e32:
|
|
|
|
|
case AMDGPU::V_FREXP_EXP_I16_F16_e64:
|
|
|
|
|
case AMDGPU::V_FREXP_EXP_I16_F16_e32:
|
|
|
|
|
case AMDGPU::V_LDEXP_F16_e64:
|
|
|
|
|
case AMDGPU::V_LDEXP_F16_e32:
|
|
|
|
|
case AMDGPU::V_LSHLREV_B16_e64:
|
|
|
|
|
case AMDGPU::V_LSHLREV_B16_e32:
|
|
|
|
|
case AMDGPU::V_LSHRREV_B16_e64:
|
|
|
|
|
case AMDGPU::V_LSHRREV_B16_e32:
|
|
|
|
|
case AMDGPU::V_ASHRREV_I16_e64:
|
|
|
|
|
case AMDGPU::V_ASHRREV_I16_e32:
|
|
|
|
|
case AMDGPU::V_ADD_U16_e64:
|
|
|
|
|
case AMDGPU::V_ADD_U16_e32:
|
|
|
|
|
case AMDGPU::V_SUB_U16_e64:
|
|
|
|
|
case AMDGPU::V_SUB_U16_e32:
|
|
|
|
|
case AMDGPU::V_SUBREV_U16_e64:
|
|
|
|
|
case AMDGPU::V_SUBREV_U16_e32:
|
|
|
|
|
case AMDGPU::V_MUL_LO_U16_e64:
|
|
|
|
|
case AMDGPU::V_MUL_LO_U16_e32:
|
|
|
|
|
case AMDGPU::V_ADD_F16_e64:
|
|
|
|
|
case AMDGPU::V_ADD_F16_e32:
|
|
|
|
|
case AMDGPU::V_SUB_F16_e64:
|
|
|
|
|
case AMDGPU::V_SUB_F16_e32:
|
|
|
|
|
case AMDGPU::V_SUBREV_F16_e64:
|
|
|
|
|
case AMDGPU::V_SUBREV_F16_e32:
|
|
|
|
|
case AMDGPU::V_MUL_F16_e64:
|
|
|
|
|
case AMDGPU::V_MUL_F16_e32:
|
|
|
|
|
case AMDGPU::V_MAX_F16_e64:
|
|
|
|
|
case AMDGPU::V_MAX_F16_e32:
|
|
|
|
|
case AMDGPU::V_MIN_F16_e64:
|
|
|
|
|
case AMDGPU::V_MIN_F16_e32:
|
|
|
|
|
case AMDGPU::V_MAX_U16_e64:
|
|
|
|
|
case AMDGPU::V_MAX_U16_e32:
|
|
|
|
|
case AMDGPU::V_MIN_U16_e64:
|
|
|
|
|
case AMDGPU::V_MIN_U16_e32:
|
|
|
|
|
case AMDGPU::V_MAX_I16_e64:
|
|
|
|
|
case AMDGPU::V_MAX_I16_e32:
|
|
|
|
|
case AMDGPU::V_MIN_I16_e64:
|
|
|
|
|
case AMDGPU::V_MIN_I16_e32:
|
|
|
|
|
// On gfx10, all 16-bit instructions preserve the high bits.
|
|
|
|
|
return getGeneration() <= AMDGPUSubtarget::GFX9;
|
|
|
|
|
case AMDGPU::V_MAD_F16_e64:
|
|
|
|
|
case AMDGPU::V_MADAK_F16:
|
|
|
|
|
case AMDGPU::V_MADMK_F16:
|
|
|
|
|
case AMDGPU::V_MAC_F16_e64:
|
|
|
|
|
case AMDGPU::V_MAC_F16_e32:
|
|
|
|
|
case AMDGPU::V_FMAMK_F16:
|
|
|
|
|
case AMDGPU::V_FMAAK_F16:
|
|
|
|
|
case AMDGPU::V_MAD_U16_e64:
|
|
|
|
|
case AMDGPU::V_MAD_I16_e64:
|
|
|
|
|
case AMDGPU::V_FMA_F16_e64:
|
|
|
|
|
case AMDGPU::V_FMAC_F16_e64:
|
|
|
|
|
case AMDGPU::V_FMAC_F16_e32:
|
|
|
|
|
case AMDGPU::V_DIV_FIXUP_F16_e64:
|
|
|
|
|
// In gfx9, the preferred handling of the unused high 16-bits changed. Most
|
|
|
|
|
// instructions maintain the legacy behavior of 0ing. Some instructions
|
|
|
|
|
// changed to preserving the high bits.
|
|
|
|
|
return getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
|
|
|
|
|
case AMDGPU::V_MAD_MIXLO_F16:
|
|
|
|
|
case AMDGPU::V_MAD_MIXHI_F16:
|
|
|
|
|
default:
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
|
2017-02-01 23:59:50 +01:00
|
|
|
const Function &F) const {
|
|
|
|
|
if (NWaves == 1)
|
2016-05-16 23:19:59 +02:00
|
|
|
return getLocalMemorySize();
|
2017-02-01 23:59:50 +01:00
|
|
|
unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
|
|
|
|
|
unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
|
2019-04-24 19:03:15 +02:00
|
|
|
if (!WorkGroupsPerCu)
|
|
|
|
|
return 0;
|
2017-02-01 23:59:50 +01:00
|
|
|
unsigned MaxWaves = getMaxWavesPerEU();
|
|
|
|
|
return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
|
2016-05-16 23:19:59 +02:00
|
|
|
}
|
|
|
|
|
|
2020-03-02 15:43:06 +01:00
|
|
|
// FIXME: Should return min,max range.
|
2018-07-11 22:59:01 +02:00
|
|
|
unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
|
2017-02-01 23:59:50 +01:00
|
|
|
const Function &F) const {
|
2020-03-02 15:43:06 +01:00
|
|
|
const unsigned MaxWorkGroupSize = getFlatWorkGroupSizes(F).second;
|
|
|
|
|
const unsigned MaxWorkGroupsPerCu = getMaxWorkGroupsPerCU(MaxWorkGroupSize);
|
|
|
|
|
if (!MaxWorkGroupsPerCu)
|
2019-04-24 19:03:15 +02:00
|
|
|
return 0;
|
2020-03-02 15:43:06 +01:00
|
|
|
|
|
|
|
|
const unsigned WaveSize = getWavefrontSize();
|
|
|
|
|
|
|
|
|
|
// FIXME: Do we need to account for alignment requirement of LDS rounding the
|
|
|
|
|
// size up?
|
|
|
|
|
// Compute restriction based on LDS usage
|
|
|
|
|
unsigned NumGroups = getLocalMemorySize() / (Bytes ? Bytes : 1u);
|
|
|
|
|
|
|
|
|
|
// This can be queried with more LDS than is possible, so just assume the
|
|
|
|
|
// worst.
|
|
|
|
|
if (NumGroups == 0)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
NumGroups = std::min(MaxWorkGroupsPerCu, NumGroups);
|
|
|
|
|
|
|
|
|
|
// Round to the number of waves.
|
|
|
|
|
const unsigned MaxGroupNumWaves = (MaxWorkGroupSize + WaveSize - 1) / WaveSize;
|
|
|
|
|
unsigned MaxWaves = NumGroups * MaxGroupNumWaves;
|
|
|
|
|
|
|
|
|
|
// Clamp to the maximum possible number of waves.
|
|
|
|
|
MaxWaves = std::min(MaxWaves, getMaxWavesPerEU());
|
|
|
|
|
|
|
|
|
|
// FIXME: Needs to be a multiple of the group size?
|
|
|
|
|
//MaxWaves = MaxGroupNumWaves * (MaxWaves / MaxGroupNumWaves);
|
|
|
|
|
|
|
|
|
|
assert(MaxWaves > 0 && MaxWaves <= getMaxWavesPerEU() &&
|
|
|
|
|
"computed invalid occupancy");
|
|
|
|
|
return MaxWaves;
|
2016-05-16 23:19:59 +02:00
|
|
|
}
|
|
|
|
|
|
AMDGPU: Remove #include "MCTargetDesc/AMDGPUMCTargetDesc.h" from common headers
Summary:
MCTargetDesc/AMDGPUMCTargetDesc.h contains enums for all the instuction
and register defintions, which are huge so we only want to include
them where needed.
This will also make it easier if we want to split the R600 and GCN
definitions into separate tablegenerated files.
I was unable to remove AMDGPUMCTargetDesc.h from SIMachineFunctionInfo.h
because it uses some enums from the header to initialize default values
for the SIMachineFunction class, so I ended up having to remove includes of
SIMachineFunctionInfo.h from headers too.
Reviewers: arsenm, nhaehnle
Reviewed By: nhaehnle
Subscribers: MatzeB, kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46272
llvm-svn: 332930
2018-05-22 04:03:23 +02:00
|
|
|
unsigned
|
2018-07-11 22:59:01 +02:00
|
|
|
AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const {
|
AMDGPU: Remove #include "MCTargetDesc/AMDGPUMCTargetDesc.h" from common headers
Summary:
MCTargetDesc/AMDGPUMCTargetDesc.h contains enums for all the instuction
and register defintions, which are huge so we only want to include
them where needed.
This will also make it easier if we want to split the R600 and GCN
definitions into separate tablegenerated files.
I was unable to remove AMDGPUMCTargetDesc.h from SIMachineFunctionInfo.h
because it uses some enums from the header to initialize default values
for the SIMachineFunction class, so I ended up having to remove includes of
SIMachineFunctionInfo.h from headers too.
Reviewers: arsenm, nhaehnle
Reviewed By: nhaehnle
Subscribers: MatzeB, kzhuravl, wdng, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46272
llvm-svn: 332930
2018-05-22 04:03:23 +02:00
|
|
|
const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
|
return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction());
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-23 19:09:35 +02:00
|
|
|
std::pair<unsigned, unsigned>
|
2018-07-11 22:59:01 +02:00
|
|
|
AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
|
2017-10-23 19:09:35 +02:00
|
|
|
switch (CC) {
|
|
|
|
|
case CallingConv::AMDGPU_VS:
|
|
|
|
|
case CallingConv::AMDGPU_LS:
|
|
|
|
|
case CallingConv::AMDGPU_HS:
|
|
|
|
|
case CallingConv::AMDGPU_ES:
|
|
|
|
|
case CallingConv::AMDGPU_GS:
|
|
|
|
|
case CallingConv::AMDGPU_PS:
|
|
|
|
|
return std::make_pair(1, getWavefrontSize());
|
|
|
|
|
default:
|
2019-08-27 18:34:40 +02:00
|
|
|
return std::make_pair(1u, getMaxFlatWorkGroupSize());
|
2017-10-23 19:09:35 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
|
2016-09-06 22:22:28 +02:00
|
|
|
const Function &F) const {
|
|
|
|
|
// Default minimum/maximum flat work group sizes.
|
|
|
|
|
std::pair<unsigned, unsigned> Default =
|
2017-10-23 19:09:35 +02:00
|
|
|
getDefaultFlatWorkGroupSize(F.getCallingConv());
|
2016-09-06 22:22:28 +02:00
|
|
|
|
|
|
|
|
// Requested minimum/maximum flat work group sizes.
|
|
|
|
|
std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
|
|
|
|
|
F, "amdgpu-flat-work-group-size", Default);
|
|
|
|
|
|
|
|
|
|
// Make sure requested minimum is less than requested maximum.
|
|
|
|
|
if (Requested.first > Requested.second)
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
// Make sure requested values do not violate subtarget's specifications.
|
|
|
|
|
if (Requested.first < getMinFlatWorkGroupSize())
|
|
|
|
|
return Default;
|
|
|
|
|
if (Requested.second > getMaxFlatWorkGroupSize())
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
return Requested;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
|
2016-09-06 22:22:28 +02:00
|
|
|
const Function &F) const {
|
|
|
|
|
// Default minimum/maximum number of waves per execution unit.
|
2017-02-09 22:33:23 +01:00
|
|
|
std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
|
2016-09-06 22:22:28 +02:00
|
|
|
|
|
|
|
|
// Default/requested minimum/maximum flat work group sizes.
|
|
|
|
|
std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
|
|
|
|
|
|
|
|
|
|
// If minimum/maximum flat work group sizes were explicitly requested using
|
|
|
|
|
// "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
|
|
|
|
|
// number of waves per execution unit to values implied by requested
|
|
|
|
|
// minimum/maximum flat work group sizes.
|
|
|
|
|
unsigned MinImpliedByFlatWorkGroupSize =
|
[AMDGPU] Rename overloaded getMaxWavesPerEU to getWavesPerEUForWorkGroup
Summary: I think Max in the name was misleading. NFC.
Reviewers: arsenm, rampitec
Subscribers: kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D76860
2020-03-26 15:58:31 +01:00
|
|
|
getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second);
|
[AMDGPU] Control num waves per EU for implicit work-group size
Summary:
If amdgpu-flat-work-group-size is not specified in LLVM IR, the backend
uses default value of 1024. For this, minimum waves per EU should be 4.
However, backend is still setting minimum value to 1 instead of calculated
value. This is not observed normally as frontend always provide
amdgpu-flat-work-group-size attribute.
Reviewers: rampitec, b-sumner, sameerds, msearles
Reviewed By: rampitec
Subscribers: qcolombet, arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D81991
2020-06-17 06:06:48 +02:00
|
|
|
Default.first = MinImpliedByFlatWorkGroupSize;
|
2020-07-14 15:18:15 +02:00
|
|
|
bool RequestedFlatWorkGroupSize =
|
|
|
|
|
F.hasFnAttribute("amdgpu-flat-work-group-size");
|
2016-09-06 22:22:28 +02:00
|
|
|
|
|
|
|
|
// Requested minimum/maximum number of waves per execution unit.
|
|
|
|
|
std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
|
|
|
|
|
F, "amdgpu-waves-per-eu", Default, true);
|
|
|
|
|
|
|
|
|
|
// Make sure requested minimum is less than requested maximum.
|
|
|
|
|
if (Requested.second && Requested.first > Requested.second)
|
|
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
// Make sure requested values do not violate subtarget's specifications.
|
|
|
|
|
if (Requested.first < getMinWavesPerEU() ||
|
2020-07-14 15:18:15 +02:00
|
|
|
Requested.second > getMaxWavesPerEU())
|
2016-09-06 22:22:28 +02:00
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
// Make sure requested values are compatible with values implied by requested
|
|
|
|
|
// minimum/maximum flat work group sizes.
|
|
|
|
|
if (RequestedFlatWorkGroupSize &&
|
2017-07-16 21:38:47 +02:00
|
|
|
Requested.first < MinImpliedByFlatWorkGroupSize)
|
2016-09-06 22:22:28 +02:00
|
|
|
return Default;
|
|
|
|
|
|
|
|
|
|
return Requested;
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-16 19:42:55 +02:00
|
|
|
static unsigned getReqdWorkGroupSize(const Function &Kernel, unsigned Dim) {
|
|
|
|
|
auto Node = Kernel.getMetadata("reqd_work_group_size");
|
|
|
|
|
if (Node && Node->getNumOperands() == 3)
|
|
|
|
|
return mdconst::extract<ConstantInt>(Node->getOperand(Dim))->getZExtValue();
|
|
|
|
|
return std::numeric_limits<unsigned>::max();
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-25 16:52:14 +01:00
|
|
|
bool AMDGPUSubtarget::isMesaKernel(const Function &F) const {
|
|
|
|
|
return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
|
|
|
|
|
}
|
|
|
|
|
|
2020-08-16 19:42:55 +02:00
|
|
|
unsigned AMDGPUSubtarget::getMaxWorkitemID(const Function &Kernel,
|
|
|
|
|
unsigned Dimension) const {
|
|
|
|
|
unsigned ReqdSize = getReqdWorkGroupSize(Kernel, Dimension);
|
|
|
|
|
if (ReqdSize != std::numeric_limits<unsigned>::max())
|
|
|
|
|
return ReqdSize - 1;
|
|
|
|
|
return getFlatWorkGroupSizes(Kernel).second - 1;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
|
2017-04-12 22:48:56 +02:00
|
|
|
Function *Kernel = I->getParent()->getParent();
|
|
|
|
|
unsigned MinSize = 0;
|
|
|
|
|
unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
|
|
|
|
|
bool IdQuery = false;
|
|
|
|
|
|
|
|
|
|
// If reqd_work_group_size is present it narrows value down.
|
|
|
|
|
if (auto *CI = dyn_cast<CallInst>(I)) {
|
|
|
|
|
const Function *F = CI->getCalledFunction();
|
|
|
|
|
if (F) {
|
|
|
|
|
unsigned Dim = UINT_MAX;
|
|
|
|
|
switch (F->getIntrinsicID()) {
|
|
|
|
|
case Intrinsic::amdgcn_workitem_id_x:
|
|
|
|
|
case Intrinsic::r600_read_tidig_x:
|
|
|
|
|
IdQuery = true;
|
2017-07-07 12:18:57 +02:00
|
|
|
LLVM_FALLTHROUGH;
|
2017-04-12 22:48:56 +02:00
|
|
|
case Intrinsic::r600_read_local_size_x:
|
|
|
|
|
Dim = 0;
|
|
|
|
|
break;
|
|
|
|
|
case Intrinsic::amdgcn_workitem_id_y:
|
|
|
|
|
case Intrinsic::r600_read_tidig_y:
|
|
|
|
|
IdQuery = true;
|
2017-07-07 12:18:57 +02:00
|
|
|
LLVM_FALLTHROUGH;
|
2017-04-12 22:48:56 +02:00
|
|
|
case Intrinsic::r600_read_local_size_y:
|
|
|
|
|
Dim = 1;
|
|
|
|
|
break;
|
|
|
|
|
case Intrinsic::amdgcn_workitem_id_z:
|
|
|
|
|
case Intrinsic::r600_read_tidig_z:
|
|
|
|
|
IdQuery = true;
|
2017-07-07 12:18:57 +02:00
|
|
|
LLVM_FALLTHROUGH;
|
2017-04-12 22:48:56 +02:00
|
|
|
case Intrinsic::r600_read_local_size_z:
|
|
|
|
|
Dim = 2;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
2020-08-16 19:42:55 +02:00
|
|
|
|
2017-04-12 22:48:56 +02:00
|
|
|
if (Dim <= 3) {
|
2020-08-16 19:42:55 +02:00
|
|
|
unsigned ReqdSize = getReqdWorkGroupSize(*Kernel, Dim);
|
|
|
|
|
if (ReqdSize != std::numeric_limits<unsigned>::max())
|
|
|
|
|
MinSize = MaxSize = ReqdSize;
|
2017-04-12 22:48:56 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!MaxSize)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
// Range metadata is [Lo, Hi). For ID query we need to pass max size
|
|
|
|
|
// as Hi. For size query we need to pass Hi + 1.
|
|
|
|
|
if (IdQuery)
|
|
|
|
|
MinSize = 0;
|
|
|
|
|
else
|
|
|
|
|
++MaxSize;
|
|
|
|
|
|
|
|
|
|
MDBuilder MDB(I->getContext());
|
|
|
|
|
MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
|
|
|
|
|
APInt(32, MaxSize));
|
|
|
|
|
I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-25 16:52:14 +01:00
|
|
|
unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
|
|
|
|
|
if (isMesaKernel(F))
|
|
|
|
|
return 16;
|
|
|
|
|
return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-20 11:05:08 +02:00
|
|
|
uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
|
2019-10-15 14:56:24 +02:00
|
|
|
Align &MaxAlign) const {
|
2018-07-20 11:05:08 +02:00
|
|
|
assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
|
|
|
|
|
F.getCallingConv() == CallingConv::SPIR_KERNEL);
|
|
|
|
|
|
|
|
|
|
const DataLayout &DL = F.getParent()->getDataLayout();
|
|
|
|
|
uint64_t ExplicitArgBytes = 0;
|
[Alignment][NFC] Deprecate Align::None()
Summary:
This is a follow up on https://reviews.llvm.org/D71473#inline-647262.
There's a caveat here that `Align(1)` relies on the compiler understanding of `Log2_64` implementation to produce good code. One could use `Align()` as a replacement but I believe it is less clear that the alignment is one in that case.
Reviewers: xbolva00, courbet, bollu
Subscribers: arsenm, dylanmckay, sdardis, nemanjai, jvesely, nhaehnle, hiraditya, kbarton, jrtc27, atanasyan, jsji, Jim, kerbowa, cfe-commits, llvm-commits
Tags: #clang, #llvm
Differential Revision: https://reviews.llvm.org/D73099
2020-01-21 15:00:04 +01:00
|
|
|
MaxAlign = Align(1);
|
2018-07-20 11:05:08 +02:00
|
|
|
|
|
|
|
|
for (const Argument &Arg : F.args()) {
|
2020-05-07 16:47:25 +02:00
|
|
|
const bool IsByRef = Arg.hasByRefAttr();
|
|
|
|
|
Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
|
|
|
|
|
MaybeAlign Alignment = IsByRef ? Arg.getParamAlign() : None;
|
|
|
|
|
if (!Alignment)
|
|
|
|
|
Alignment = DL.getABITypeAlign(ArgTy);
|
2018-07-20 11:05:08 +02:00
|
|
|
|
|
|
|
|
uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
|
2019-10-15 14:56:24 +02:00
|
|
|
ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;
|
2020-05-07 16:47:25 +02:00
|
|
|
MaxAlign = max(MaxAlign, Alignment);
|
2018-07-20 11:05:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ExplicitArgBytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
|
2019-10-15 14:56:24 +02:00
|
|
|
Align &MaxAlign) const {
|
2018-07-20 11:05:08 +02:00
|
|
|
uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
|
|
|
|
|
|
|
|
|
|
unsigned ExplicitOffset = getExplicitKernelArgOffset(F);
|
|
|
|
|
|
|
|
|
|
uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
|
|
|
|
|
unsigned ImplicitBytes = getImplicitArgNumBytes(F);
|
|
|
|
|
if (ImplicitBytes != 0) {
|
2019-10-15 14:56:24 +02:00
|
|
|
const Align Alignment = getAlignmentForImplicitArgPtr();
|
2018-07-20 11:05:08 +02:00
|
|
|
TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Being able to dereference past the end is useful for emitting scalar loads.
|
|
|
|
|
return alignTo(TotalSize, 4);
|
|
|
|
|
}
|
|
|
|
|
|
2021-01-20 13:48:02 +01:00
|
|
|
AMDGPUDwarfFlavour AMDGPUSubtarget::getAMDGPUDwarfFlavour() const {
|
|
|
|
|
return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32
|
|
|
|
|
: AMDGPUDwarfFlavour::Wave64;
|
|
|
|
|
}
|
|
|
|
|
|
2016-06-24 08:30:11 +02:00
|
|
|
R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
|
|
|
|
|
const TargetMachine &TM) :
|
2020-08-14 23:56:54 +02:00
|
|
|
R600GenSubtargetInfo(TT, GPU, /*TuneCPU*/GPU, FS),
|
2018-09-12 20:50:47 +02:00
|
|
|
AMDGPUSubtarget(TT),
|
2016-06-24 08:30:11 +02:00
|
|
|
InstrInfo(*this),
|
|
|
|
|
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
FMA(false),
|
|
|
|
|
CaymanISA(false),
|
|
|
|
|
CFALUBug(false),
|
|
|
|
|
HasVertexCache(false),
|
|
|
|
|
R600ALUInst(false),
|
|
|
|
|
FP64(false),
|
|
|
|
|
TexVTXClauseSize(0),
|
|
|
|
|
Gen(R600),
|
|
|
|
|
TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)),
|
2018-08-31 07:49:54 +02:00
|
|
|
InstrItins(getInstrItineraryForCPU(GPU)) { }
|
2016-06-24 08:30:11 +02:00
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
|
2016-06-28 02:11:26 +02:00
|
|
|
unsigned NumRegionInstrs) const {
|
2016-06-24 08:30:11 +02:00
|
|
|
// Track register pressure so the scheduler can try to decrease
|
|
|
|
|
// pressure once register usage is above the threshold defined by
|
|
|
|
|
// SIRegisterInfo::getRegPressureSetLimit()
|
|
|
|
|
Policy.ShouldTrackPressure = true;
|
|
|
|
|
|
|
|
|
|
// Enabling both top down and bottom up scheduling seems to give us less
|
|
|
|
|
// register spills than just using one of these approaches on its own.
|
|
|
|
|
Policy.OnlyTopDown = false;
|
|
|
|
|
Policy.OnlyBottomUp = false;
|
|
|
|
|
|
2017-02-14 15:29:05 +01:00
|
|
|
// Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
|
|
|
|
|
if (!enableSIScheduler())
|
|
|
|
|
Policy.ShouldTrackLaneMasks = true;
|
2016-06-24 08:30:11 +02:00
|
|
|
}
|
2015-01-29 17:55:25 +01:00
|
|
|
|
2019-04-24 19:03:15 +02:00
|
|
|
bool GCNSubtarget::hasMadF16() const {
|
2021-01-07 19:56:02 +01:00
|
|
|
return InstrInfo.pseudoToMCOpcode(AMDGPU::V_MAD_F16_e64) != -1;
|
2019-04-24 19:03:15 +02:00
|
|
|
}
|
|
|
|
|
|
2020-01-02 22:45:33 +01:00
|
|
|
bool GCNSubtarget::useVGPRIndexMode() const {
|
|
|
|
|
return !hasMovrel() || (EnableVGPRIndexMode && hasVGPRIndexMode());
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-12 05:51:53 +02:00
|
|
|
bool GCNSubtarget::useAA() const { return UseAA; }
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
|
2019-04-24 19:03:15 +02:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::GFX10)
|
2019-07-31 03:07:10 +02:00
|
|
|
return getMaxWavesPerEU();
|
2019-04-24 19:03:15 +02:00
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
|
2016-08-29 21:42:52 +02:00
|
|
|
if (SGPRs <= 80)
|
|
|
|
|
return 10;
|
|
|
|
|
if (SGPRs <= 88)
|
|
|
|
|
return 9;
|
|
|
|
|
if (SGPRs <= 100)
|
|
|
|
|
return 8;
|
|
|
|
|
return 7;
|
|
|
|
|
}
|
|
|
|
|
if (SGPRs <= 48)
|
|
|
|
|
return 10;
|
|
|
|
|
if (SGPRs <= 56)
|
|
|
|
|
return 9;
|
|
|
|
|
if (SGPRs <= 64)
|
|
|
|
|
return 8;
|
|
|
|
|
if (SGPRs <= 72)
|
|
|
|
|
return 7;
|
|
|
|
|
if (SGPRs <= 80)
|
|
|
|
|
return 6;
|
|
|
|
|
return 5;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
|
2019-07-19 23:29:51 +02:00
|
|
|
unsigned MaxWaves = getMaxWavesPerEU();
|
|
|
|
|
unsigned Granule = getVGPRAllocGranule();
|
|
|
|
|
if (VGPRs < Granule)
|
|
|
|
|
return MaxWaves;
|
|
|
|
|
unsigned RoundedRegs = ((VGPRs + Granule - 1) / Granule) * Granule;
|
2019-09-19 22:09:04 +02:00
|
|
|
return std::min(std::max(getTotalNumVGPRs() / RoundedRegs, 1u), MaxWaves);
|
2016-08-29 21:42:52 +02:00
|
|
|
}
|
2016-10-28 22:31:47 +02:00
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned
|
|
|
|
|
GCNSubtarget::getBaseReservedNumSGPRs(const bool HasFlatScratchInit) const {
|
2019-04-24 19:03:15 +02:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::GFX10)
|
|
|
|
|
return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
|
|
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
if (HasFlatScratchInit) {
|
2017-02-08 14:02:33 +01:00
|
|
|
if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
|
|
|
|
|
return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
|
|
|
|
|
if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
|
|
|
|
|
return 4; // FLAT_SCRATCH, VCC (in that order).
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (isXNACKEnabled())
|
|
|
|
|
return 4; // XNACK, VCC (in that order).
|
|
|
|
|
return 2; // VCC.
|
|
|
|
|
}
|
|
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
|
|
|
|
|
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
|
return getBaseReservedNumSGPRs(MFI.hasFlatScratchInit());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned GCNSubtarget::getReservedNumSGPRs(const Function &F) const {
|
|
|
|
|
// The logic to detect if the function has
|
2021-06-18 10:14:54 +02:00
|
|
|
// flat scratch init is slightly different than how
|
|
|
|
|
// SIMachineFunctionInfo constructor derives.
|
|
|
|
|
// We don't use amdgpu-calls, amdgpu-stack-objects
|
|
|
|
|
// attributes and isAmdHsaOrMesa here as it doesn't really matter.
|
|
|
|
|
// TODO: Outline this derivation logic and have just
|
|
|
|
|
// one common function in the backend to avoid duplication.
|
|
|
|
|
bool isEntry = AMDGPU::isEntryFunctionCC(F.getCallingConv());
|
2021-06-03 19:04:10 +02:00
|
|
|
bool FunctionHasFlatScratchInit = false;
|
2021-06-18 10:14:54 +02:00
|
|
|
if (hasFlatAddressSpace() && isEntry && !flatScratchIsArchitected() &&
|
|
|
|
|
enableFlatScratch()) {
|
|
|
|
|
FunctionHasFlatScratchInit = true;
|
2021-06-03 19:04:10 +02:00
|
|
|
}
|
|
|
|
|
return getBaseReservedNumSGPRs(FunctionHasFlatScratchInit);
|
|
|
|
|
}
|
|
|
|
|
|
2020-06-18 14:39:13 +02:00
|
|
|
unsigned GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
|
2019-07-31 03:07:10 +02:00
|
|
|
unsigned NumSGPRs,
|
|
|
|
|
unsigned NumVGPRs) const {
|
|
|
|
|
unsigned Occupancy =
|
|
|
|
|
std::min(getMaxWavesPerEU(),
|
2020-06-18 14:39:13 +02:00
|
|
|
getOccupancyWithLocalMemSize(LDSSize, F));
|
2019-07-31 03:07:10 +02:00
|
|
|
if (NumSGPRs)
|
|
|
|
|
Occupancy = std::min(Occupancy, getOccupancyWithNumSGPRs(NumSGPRs));
|
|
|
|
|
if (NumVGPRs)
|
|
|
|
|
Occupancy = std::min(Occupancy, getOccupancyWithNumVGPRs(NumVGPRs));
|
|
|
|
|
return Occupancy;
|
|
|
|
|
}
|
|
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned GCNSubtarget::getBaseMaxNumSGPRs(
|
|
|
|
|
const Function &F, std::pair<unsigned, unsigned> WavesPerEU,
|
|
|
|
|
unsigned PreloadedSGPRs, unsigned ReservedNumSGPRs) const {
|
2017-02-08 14:02:33 +01:00
|
|
|
// Compute maximum number of SGPRs function can use using default/requested
|
|
|
|
|
// minimum number of waves per execution unit.
|
|
|
|
|
unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
|
|
|
|
|
unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
|
|
|
|
|
|
|
|
|
|
// Check if maximum number of SGPRs was explicitly requested using
|
|
|
|
|
// "amdgpu-num-sgpr" attribute.
|
|
|
|
|
if (F.hasFnAttribute("amdgpu-num-sgpr")) {
|
|
|
|
|
unsigned Requested = AMDGPU::getIntegerAttribute(
|
|
|
|
|
F, "amdgpu-num-sgpr", MaxNumSGPRs);
|
|
|
|
|
|
|
|
|
|
// Make sure requested value does not violate subtarget's specifications.
|
2021-06-03 19:04:10 +02:00
|
|
|
if (Requested && (Requested <= ReservedNumSGPRs))
|
2017-02-08 14:02:33 +01:00
|
|
|
Requested = 0;
|
|
|
|
|
|
|
|
|
|
// If more SGPRs are required to support the input user/system SGPRs,
|
|
|
|
|
// increase to accommodate them.
|
|
|
|
|
//
|
|
|
|
|
// FIXME: This really ends up using the requested number of SGPRs + number
|
|
|
|
|
// of reserved special registers in total. Theoretically you could re-use
|
|
|
|
|
// the last input registers for these special registers, but this would
|
|
|
|
|
// require a lot of complexity to deal with the weird aliasing.
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned InputNumSGPRs = PreloadedSGPRs;
|
2017-02-08 14:02:33 +01:00
|
|
|
if (Requested && Requested < InputNumSGPRs)
|
|
|
|
|
Requested = InputNumSGPRs;
|
|
|
|
|
|
|
|
|
|
// Make sure requested value is compatible with values implied by
|
|
|
|
|
// default/requested minimum/maximum number of waves per execution unit.
|
|
|
|
|
if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
if (WavesPerEU.second &&
|
|
|
|
|
Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
|
|
|
|
|
if (Requested)
|
|
|
|
|
MaxNumSGPRs = Requested;
|
|
|
|
|
}
|
|
|
|
|
|
2016-10-28 22:31:47 +02:00
|
|
|
if (hasSGPRInitBug())
|
2017-02-08 15:05:23 +01:00
|
|
|
MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
|
2017-02-08 14:02:33 +01:00
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
return std::min(MaxNumSGPRs - ReservedNumSGPRs, MaxAddressableNumSGPRs);
|
2017-02-08 14:02:33 +01:00
|
|
|
}
|
|
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
|
2017-12-15 23:22:58 +01:00
|
|
|
const Function &F = MF.getFunction();
|
2017-02-08 14:02:33 +01:00
|
|
|
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
|
2021-06-03 19:04:10 +02:00
|
|
|
return getBaseMaxNumSGPRs(F, MFI.getWavesPerEU(), MFI.getNumPreloadedSGPRs(),
|
|
|
|
|
getReservedNumSGPRs(MF));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static unsigned getMaxNumPreloadedSGPRs() {
|
|
|
|
|
// Max number of user SGPRs
|
|
|
|
|
unsigned MaxUserSGPRs = 4 + // private segment buffer
|
|
|
|
|
2 + // Dispatch ptr
|
|
|
|
|
2 + // queue ptr
|
|
|
|
|
2 + // kernel segment ptr
|
|
|
|
|
2 + // dispatch ID
|
|
|
|
|
2 + // flat scratch init
|
|
|
|
|
2; // Implicit buffer ptr
|
|
|
|
|
// Max number of system SGPRs
|
|
|
|
|
unsigned MaxSystemSGPRs = 1 + // WorkGroupIDX
|
|
|
|
|
1 + // WorkGroupIDY
|
|
|
|
|
1 + // WorkGroupIDZ
|
|
|
|
|
1 + // WorkGroupInfo
|
|
|
|
|
1; // private segment wave byte offset
|
|
|
|
|
return MaxUserSGPRs + MaxSystemSGPRs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned GCNSubtarget::getMaxNumSGPRs(const Function &F) const {
|
|
|
|
|
return getBaseMaxNumSGPRs(F, getWavesPerEU(F), getMaxNumPreloadedSGPRs(),
|
|
|
|
|
getReservedNumSGPRs(F));
|
|
|
|
|
}
|
2017-02-08 14:02:33 +01:00
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned GCNSubtarget::getBaseMaxNumVGPRs(
|
|
|
|
|
const Function &F, std::pair<unsigned, unsigned> WavesPerEU) const {
|
2017-02-08 14:02:33 +01:00
|
|
|
// Compute maximum number of VGPRs function can use using default/requested
|
|
|
|
|
// minimum number of waves per execution unit.
|
|
|
|
|
unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
|
|
|
|
|
|
|
|
|
|
// Check if maximum number of VGPRs was explicitly requested using
|
|
|
|
|
// "amdgpu-num-vgpr" attribute.
|
|
|
|
|
if (F.hasFnAttribute("amdgpu-num-vgpr")) {
|
|
|
|
|
unsigned Requested = AMDGPU::getIntegerAttribute(
|
|
|
|
|
F, "amdgpu-num-vgpr", MaxNumVGPRs);
|
|
|
|
|
|
2021-02-17 22:37:46 +01:00
|
|
|
if (hasGFX90AInsts())
|
|
|
|
|
Requested *= 2;
|
|
|
|
|
|
2017-02-08 14:02:33 +01:00
|
|
|
// Make sure requested value is compatible with values implied by
|
|
|
|
|
// default/requested minimum/maximum number of waves per execution unit.
|
|
|
|
|
if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
if (WavesPerEU.second &&
|
|
|
|
|
Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
|
|
|
|
|
Requested = 0;
|
|
|
|
|
|
|
|
|
|
if (Requested)
|
|
|
|
|
MaxNumVGPRs = Requested;
|
|
|
|
|
}
|
2016-10-28 22:31:47 +02:00
|
|
|
|
2018-06-21 22:28:19 +02:00
|
|
|
return MaxNumVGPRs;
|
2016-10-28 22:31:47 +02:00
|
|
|
}
|
2017-09-19 22:54:38 +02:00
|
|
|
|
2021-06-03 19:04:10 +02:00
|
|
|
unsigned GCNSubtarget::getMaxNumVGPRs(const Function &F) const {
|
|
|
|
|
return getBaseMaxNumVGPRs(F, getWavesPerEU(F));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
|
|
|
|
|
const Function &F = MF.getFunction();
|
|
|
|
|
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
|
|
|
|
|
return getBaseMaxNumVGPRs(F, MFI.getWavesPerEU());
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-31 12:57:51 +02:00
|
|
|
void GCNSubtarget::adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use,
|
|
|
|
|
int UseOpIdx, SDep &Dep) const {
|
2020-01-10 21:28:37 +01:00
|
|
|
if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() ||
|
2020-03-31 12:57:51 +02:00
|
|
|
!Def->isInstr() || !Use->isInstr())
|
2020-01-10 21:28:37 +01:00
|
|
|
return;
|
|
|
|
|
|
2020-03-31 12:57:51 +02:00
|
|
|
MachineInstr *DefI = Def->getInstr();
|
|
|
|
|
MachineInstr *UseI = Use->getInstr();
|
2020-01-10 21:28:37 +01:00
|
|
|
|
2020-03-31 12:57:51 +02:00
|
|
|
if (DefI->isBundle()) {
|
2020-01-10 21:28:37 +01:00
|
|
|
const SIRegisterInfo *TRI = getRegisterInfo();
|
|
|
|
|
auto Reg = Dep.getReg();
|
2020-03-31 12:57:51 +02:00
|
|
|
MachineBasicBlock::const_instr_iterator I(DefI->getIterator());
|
|
|
|
|
MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end());
|
2020-01-14 02:01:36 +01:00
|
|
|
unsigned Lat = 0;
|
2020-01-10 21:28:37 +01:00
|
|
|
for (++I; I != E && I->isBundledWithPred(); ++I) {
|
2020-01-14 02:01:36 +01:00
|
|
|
if (I->modifiesRegister(Reg, TRI))
|
|
|
|
|
Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *I);
|
|
|
|
|
else if (Lat)
|
|
|
|
|
--Lat;
|
2020-01-10 21:28:37 +01:00
|
|
|
}
|
2020-01-14 02:01:36 +01:00
|
|
|
Dep.setLatency(Lat);
|
2020-03-31 12:57:51 +02:00
|
|
|
} else if (UseI->isBundle()) {
|
2020-01-14 02:01:36 +01:00
|
|
|
const SIRegisterInfo *TRI = getRegisterInfo();
|
|
|
|
|
auto Reg = Dep.getReg();
|
2020-03-31 12:57:51 +02:00
|
|
|
MachineBasicBlock::const_instr_iterator I(UseI->getIterator());
|
|
|
|
|
MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end());
|
|
|
|
|
unsigned Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *DefI);
|
2020-01-14 02:01:36 +01:00
|
|
|
for (++I; I != E && I->isBundledWithPred() && Lat; ++I) {
|
|
|
|
|
if (I->readsRegister(Reg, TRI))
|
|
|
|
|
break;
|
|
|
|
|
--Lat;
|
|
|
|
|
}
|
|
|
|
|
Dep.setLatency(Lat);
|
2020-01-10 21:28:37 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2017-11-01 00:21:30 +01:00
|
|
|
namespace {
|
2019-07-11 23:25:00 +02:00
|
|
|
struct FillMFMAShadowMutation : ScheduleDAGMutation {
|
|
|
|
|
const SIInstrInfo *TII;
|
|
|
|
|
|
|
|
|
|
ScheduleDAGMI *DAG;
|
|
|
|
|
|
|
|
|
|
FillMFMAShadowMutation(const SIInstrInfo *tii) : TII(tii) {}
|
|
|
|
|
|
|
|
|
|
bool isSALU(const SUnit *SU) const {
|
2019-07-15 17:34:05 +02:00
|
|
|
const MachineInstr *MI = SU->getInstr();
|
|
|
|
|
return MI && TII->isSALU(*MI) && !MI->isTerminator();
|
2019-07-11 23:25:00 +02:00
|
|
|
}
|
|
|
|
|
|
2019-10-24 19:34:47 +02:00
|
|
|
bool isVALU(const SUnit *SU) const {
|
|
|
|
|
const MachineInstr *MI = SU->getInstr();
|
|
|
|
|
return MI && TII->isVALU(*MI);
|
|
|
|
|
}
|
|
|
|
|
|
2019-07-11 23:25:00 +02:00
|
|
|
bool canAddEdge(const SUnit *Succ, const SUnit *Pred) const {
|
|
|
|
|
if (Pred->NodeNum < Succ->NodeNum)
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
SmallVector<const SUnit*, 64> Succs({Succ}), Preds({Pred});
|
|
|
|
|
|
|
|
|
|
for (unsigned I = 0; I < Succs.size(); ++I) {
|
|
|
|
|
for (const SDep &SI : Succs[I]->Succs) {
|
|
|
|
|
const SUnit *SU = SI.getSUnit();
|
2020-12-05 06:42:54 +01:00
|
|
|
if (SU != Succs[I] && !llvm::is_contained(Succs, SU))
|
2019-07-11 23:25:00 +02:00
|
|
|
Succs.push_back(SU);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
SmallPtrSet<const SUnit*, 32> Visited;
|
|
|
|
|
while (!Preds.empty()) {
|
|
|
|
|
const SUnit *SU = Preds.pop_back_val();
|
2020-12-05 06:42:54 +01:00
|
|
|
if (llvm::is_contained(Succs, SU))
|
2019-07-11 23:25:00 +02:00
|
|
|
return false;
|
|
|
|
|
Visited.insert(SU);
|
|
|
|
|
for (const SDep &SI : SU->Preds)
|
|
|
|
|
if (SI.getSUnit() != SU && !Visited.count(SI.getSUnit()))
|
|
|
|
|
Preds.push_back(SI.getSUnit());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Link as much SALU intructions in chain as possible. Return the size
|
|
|
|
|
// of the chain. Links up to MaxChain instructions.
|
|
|
|
|
unsigned linkSALUChain(SUnit *From, SUnit *To, unsigned MaxChain,
|
|
|
|
|
SmallPtrSetImpl<SUnit *> &Visited) const {
|
|
|
|
|
SmallVector<SUnit *, 8> Worklist({To});
|
|
|
|
|
unsigned Linked = 0;
|
|
|
|
|
|
|
|
|
|
while (!Worklist.empty() && MaxChain-- > 0) {
|
|
|
|
|
SUnit *SU = Worklist.pop_back_val();
|
|
|
|
|
if (!Visited.insert(SU).second)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "Inserting edge from\n" ; DAG->dumpNode(*From);
|
|
|
|
|
dbgs() << "to\n"; DAG->dumpNode(*SU); dbgs() << '\n');
|
|
|
|
|
|
|
|
|
|
if (SU->addPred(SDep(From, SDep::Artificial), false))
|
|
|
|
|
++Linked;
|
|
|
|
|
|
|
|
|
|
for (SDep &SI : From->Succs) {
|
|
|
|
|
SUnit *SUv = SI.getSUnit();
|
2019-10-24 19:34:47 +02:00
|
|
|
if (SUv != From && isVALU(SUv) && canAddEdge(SUv, SU))
|
2019-07-11 23:25:00 +02:00
|
|
|
SUv->addPred(SDep(SU, SDep::Artificial), false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (SDep &SI : SU->Succs) {
|
|
|
|
|
SUnit *Succ = SI.getSUnit();
|
|
|
|
|
if (Succ != SU && isSALU(Succ) && canAddEdge(From, Succ))
|
|
|
|
|
Worklist.push_back(Succ);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return Linked;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void apply(ScheduleDAGInstrs *DAGInstrs) override {
|
|
|
|
|
const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget<GCNSubtarget>();
|
|
|
|
|
if (!ST.hasMAIInsts() || DisablePowerSched)
|
|
|
|
|
return;
|
|
|
|
|
DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
|
|
|
|
|
const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel();
|
|
|
|
|
if (!TSchedModel || DAG->SUnits.empty())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
// Scan for MFMA long latency instructions and try to add a dependency
|
|
|
|
|
// of available SALU instructions to give them a chance to fill MFMA
|
|
|
|
|
// shadow. That is desirable to fill MFMA shadow with SALU instructions
|
|
|
|
|
// rather than VALU to prevent power consumption bursts and throttle.
|
|
|
|
|
auto LastSALU = DAG->SUnits.begin();
|
|
|
|
|
auto E = DAG->SUnits.end();
|
|
|
|
|
SmallPtrSet<SUnit*, 32> Visited;
|
|
|
|
|
for (SUnit &SU : DAG->SUnits) {
|
|
|
|
|
MachineInstr &MAI = *SU.getInstr();
|
|
|
|
|
if (!TII->isMAI(MAI) ||
|
2021-01-07 19:56:02 +01:00
|
|
|
MAI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
|
|
|
|
|
MAI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64)
|
2019-07-11 23:25:00 +02:00
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
unsigned Lat = TSchedModel->computeInstrLatency(&MAI) - 1;
|
|
|
|
|
|
|
|
|
|
LLVM_DEBUG(dbgs() << "Found MFMA: "; DAG->dumpNode(SU);
|
|
|
|
|
dbgs() << "Need " << Lat
|
|
|
|
|
<< " instructions to cover latency.\n");
|
|
|
|
|
|
|
|
|
|
// Find up to Lat independent scalar instructions as early as
|
|
|
|
|
// possible such that they can be scheduled after this MFMA.
|
|
|
|
|
for ( ; Lat && LastSALU != E; ++LastSALU) {
|
|
|
|
|
if (Visited.count(&*LastSALU))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
if (!isSALU(&*LastSALU) || !canAddEdge(&*LastSALU, &SU))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
Lat -= linkSALUChain(&SU, &*LastSALU, Lat, Visited);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
2017-11-01 00:21:30 +01:00
|
|
|
} // namespace
|
2017-09-19 22:54:38 +02:00
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
void GCNSubtarget::getPostRAMutations(
|
2017-09-19 22:54:38 +02:00
|
|
|
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
|
2019-08-15 17:54:37 +02:00
|
|
|
Mutations.push_back(std::make_unique<FillMFMAShadowMutation>(&InstrInfo));
|
2017-09-19 22:54:38 +02:00
|
|
|
}
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
|
2018-07-11 22:59:01 +02:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
else
|
2018-07-11 22:59:01 +02:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<R600Subtarget>());
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
}
|
|
|
|
|
|
2018-07-11 22:59:01 +02:00
|
|
|
const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
if (TM.getTargetTriple().getArch() == Triple::amdgcn)
|
2018-07-11 22:59:01 +02:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
else
|
2018-07-11 22:59:01 +02:00
|
|
|
return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<R600Subtarget>(F));
|
AMDGPU: Separate R600 and GCN TableGen files
Summary:
We now have two sets of generated TableGen files, one for R600 and one
for GCN, so each sub-target now has its own tables of instructions,
registers, ISel patterns, etc. This should help reduce compile time
since each sub-target now only has to consider information that
is specific to itself. This will also help prevent the R600
sub-target from slowing down new features for GCN, like disassembler
support, GlobalISel, etc.
Reviewers: arsenm, nhaehnle, jvesely
Reviewed By: arsenm
Subscribers: MatzeB, kzhuravl, wdng, mgorny, yaxunl, dstuttard, tpr, t-tye, javed.absar, llvm-commits
Differential Revision: https://reviews.llvm.org/D46365
llvm-svn: 335942
2018-06-29 01:47:12 +02:00
|
|
|
}
|
