RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 02:33:06 +01:00
Code Issues Projects Releases Wiki Activity

CodeGen: Store LLT instead of uint64_t in MachineMemOperand

Browse Source 
GlobalISel is relying on regular MachineMemOperands to track all of
the memory properties of accesses. Just the raw byte size is
insufficent to disambiguate all situations. For example, if we need to
split an unaligned extending load, we need to know the number of bits
in the original source value and can't infer it from the result
type. This is also a problem for extending vector loads.

This does decrease the maximum representable size from the full
uint64_t bytes to a maximum of 16-bits. No in tree testcases hit this,
other than places using UINT64_MAX for unknown sizes. This may be an
issue for G_MEMCPY and co., although they can just use unknown size
for large static sizes. This also has potential for backend abuse by
relying on the type when it really shouldn't be relevant after
selection.

This does not include the necessary MIR printer/parser changes to
represent this.
This commit is contained in:
Matt Arsenault 2021-05-19 22:06:14 -04:00
parent d18525d5b9
commit aa0b4488a5
12 changed files with 123 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/llvm/CodeGen/MachineFunction.h
View File

@ -865,12 +865,23 @@ public:
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, adjusting by an offset and using the given size.
/// MachineMemOperands are owned by the MachineFunction and need not be
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, uint64_t Size);
int64_t Offset, LLT Ty);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, uint64_t Size) {
return getMachineMemOperand(MMO, Offset, LLT::scalar(8 * Size));
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
/// an existing one, replacing only the MachinePointerInfo and size.
@ -879,6 +890,9 @@ public:
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
uint64_t Size);
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
const MachinePointerInfo &PtrInfo,
LLT Ty);
/// Allocate a new MachineMemOperand by copying an existing one,
/// replacing only AliasAnalysis information. MachineMemOperands are owned

29
include/llvm/CodeGen/MachineMemOperand.h
View File

@ -22,6 +22,7 @@
#include "llvm/IR/Value.h" // PointerLikeTypeTraits<Value*>
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/DataTypes.h"
#include "llvm/Support/LowLevelTypeImpl.h"
namespace llvm {
@ -168,7 +169,11 @@ private:
};
MachinePointerInfo PtrInfo;
uint64_t Size;
/// Track the memory type of the access. An access size which is unknown or
/// too large to be represented by LLT should use the invalid LLT.
LLT MemoryType;
Flags FlagVals;
Align BaseAlign;
MachineAtomicInfo AtomicInfo;
@ -187,6 +192,12 @@ public:
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
@ -217,11 +228,23 @@ public:
unsigned getAddrSpace() const { return PtrInfo.getAddrSpace(); }
/// Return the memory type of the memory reference. This should only be relied
/// on for GlobalISel G_* operation legalization.
LLT getMemoryType() const { return MemoryType; }
/// Return the size in bytes of the memory reference.
uint64_t getSize() const { return Size; }
uint64_t getSize() const {
return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0);
}
/// Return the size in bits of the memory reference.
uint64_t getSizeInBits() const { return Size * 8; }
uint64_t getSizeInBits() const {
return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0);
}
LLT getType() const {
return MemoryType;
}
/// Return the minimum known alignment in bytes of the actual memory
/// reference.

1
include/llvm/Support/LowLevelTypeImpl.h
View File

@ -366,6 +366,7 @@ private:
}
}
public:
uint64_t getUniqueRAWLLTData() const {
return ((uint64_t)RawData) << 2 | ((uint64_t)IsPointer) << 1 |
((uint64_t)IsVector);

38
lib/CodeGen/GlobalISel/IRTranslator.cpp
View File

@ -1300,7 +1300,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
AAMDNodes AAMetadata;
LI.getAAMetadata(AAMetadata);
auto MMO = MF->getMachineMemOperand(
Ptr, Flags, MRI->getType(Regs[i]).getSizeInBytes(),
Ptr, Flags, MRI->getType(Regs[i]),
commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges,
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
@ -1342,7 +1342,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
AAMDNodes AAMetadata;
SI.getAAMetadata(AAMetadata);
auto MMO = MF->getMachineMemOperand(
Ptr, Flags, MRI->getType(Vals[i]).getSizeInBytes(),
Ptr, Flags, MRI->getType(Vals[i]),
commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr,
SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
@ -1627,12 +1627,14 @@ void IRTranslator::getStackGuard(Register DstReg,
if (!Global)
return;
unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
MachinePointerInfo MPInfo(Global);
auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
MachineMemOperand::MODereferenceable;
MachineMemOperand *MemRef =
MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
DL->getPointerABIAlignment(0));
MachineMemOperand *MemRef = MF->getMachineMemOperand(
MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
MIB.setMemRefs({MemRef});
}
@ -2067,7 +2069,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
*MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
MachineMemOperand::MOStore |
MachineMemOperand::MOVolatile,
PtrTy.getSizeInBits() / 8, Align(8)));
PtrTy, Align(8)));
return true;
}
case Intrinsic::stacksave: {
@ -2364,10 +2366,11 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
Align Alignment = Info.align.getValueOr(
DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
uint64_t Size = Info.memVT.getStoreSize();
LLT MemTy = Info.memVT.isSimple()
? getLLTForMVT(Info.memVT.getSimpleVT())
: LLT::scalar(Info.memVT.getStoreSizeInBits());
MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
Info.flags, Size, Alignment));
Info.flags, MemTy, Alignment));
}
return true;
@ -2733,9 +2736,6 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
auto &TLI = *MF->getSubtarget().getTargetLowering();
auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
Type *ResType = I.getType();
Type *ValType = ResType->Type::getStructElementType(0);
auto Res = getOrCreateVRegs(I);
Register OldValRes = Res[0];
Register SuccessRes = Res[1];
@ -2749,9 +2749,9 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
MIRBuilder.buildAtomicCmpXchgWithSuccess(
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags,
DL->getTypeStoreSize(ValType), getMemOpAlign(I), AAMetadata, nullptr,
I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
getMemOpAlign(I), AAMetadata, nullptr, I.getSyncScopeID(),
I.getSuccessOrdering(), I.getFailureOrdering()));
return true;
}
@ -2761,8 +2761,6 @@ bool IRTranslator::translateAtomicRMW(const User &U,
auto &TLI = *MF->getSubtarget().getTargetLowering();
auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
Type *ResType = I.getType();
Register Res = getOrCreateVReg(I);
Register Addr = getOrCreateVReg(*I.getPointerOperand());
Register Val = getOrCreateVReg(*I.getValOperand());
@ -2818,9 +2816,9 @@ bool IRTranslator::translateAtomicRMW(const User &U,
MIRBuilder.buildAtomicRMW(
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, DL->getTypeStoreSize(ResType),
getMemOpAlign(I), AAMetadata, nullptr,
I.getSyncScopeID(), I.getOrdering()));
Flags, MRI->getType(Val), getMemOpAlign(I),
AAMetadata, nullptr, I.getSyncScopeID(),
I.getOrdering()));
return true;
}

12
lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
View File

@ -347,10 +347,9 @@ MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
MMOFlags |= MachineMemOperand::MOLoad;
assert((MMOFlags & MachineMemOperand::MOStore) == 0);
uint64_t Size = MemoryLocation::getSizeOrUnknown(
TypeSize::Fixed(Dst.getLLTTy(*getMRI()).getSizeInBytes()));
LLT Ty = Dst.getLLTTy(*getMRI());
MachineMemOperand *MMO =
getMF().getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo);
return buildLoad(Dst, Addr, *MMO);
}
@ -373,7 +372,7 @@ MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
MachineMemOperand &BaseMMO, int64_t Offset) {
LLT LoadTy = Dst.getLLTTy(*getMRI());
MachineMemOperand *OffsetMMO =
getMF().getMachineMemOperand(&BaseMMO, Offset, LoadTy.getSizeInBytes());
getMF().getMachineMemOperand(&BaseMMO, Offset, LoadTy);
if (Offset == 0) // This may be a size or type changing load.
return buildLoad(Dst, BasePtr, *OffsetMMO);
@ -406,10 +405,9 @@ MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
MMOFlags |= MachineMemOperand::MOStore;
assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
uint64_t Size = MemoryLocation::getSizeOrUnknown(
TypeSize::Fixed(Val.getLLTTy(*getMRI()).getSizeInBytes()));
LLT Ty = Val.getLLTTy(*getMRI());
MachineMemOperand *MMO =
getMF().getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo);
return buildStore(Val, Addr, *MMO);
}

4
lib/CodeGen/LowLevelType.cpp
View File

@ -56,8 +56,8 @@ LLT llvm::getLLTForMVT(MVT Ty) {
if (!Ty.isVector())
return LLT::scalar(Ty.getSizeInBits());
return LLT::vector(Ty.getVectorElementCount(),
Ty.getVectorElementType().getSizeInBits());
return LLT::scalarOrVector(Ty.getVectorElementCount(),
Ty.getVectorElementType().getSizeInBits());
}
const llvm::fltSemantics &llvm::getFltSemanticForLLT(LLT Ty) {

22
lib/CodeGen/MachineFunction.cpp
View File

@ -437,6 +437,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
SSID, Ordering, FailureOrdering);
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges, SSID,
Ordering, FailureOrdering);
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, uint64_t Size) {
return new (Allocator)
@ -445,9 +455,17 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator)
MachineMemOperand(PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
int64_t Offset, uint64_t Size) {
int64_t Offset, LLT Ty) {
const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
// If there is no pointer value, the offset isn't tracked so we need to adjust
@ -459,7 +477,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// Do not preserve ranges, since we don't necessarily know what the high bits
// are anymore.
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Size, Alignment,
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
MMO->getAAInfo(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}

19
lib/CodeGen/MachineOperand.cpp
View File

@ -1023,13 +1023,12 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
}
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
uint64_t s, Align a,
const AAMDNodes &AAInfo,
LLT type, Align a, const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlign(a), AAInfo(AAInfo),
Ranges(Ranges) {
: PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a),
AAInfo(AAInfo), Ranges(Ranges) {
assert((PtrInfo.V.isNull() || PtrInfo.V.is<const PseudoSourceValue *>() ||
isa<PointerType>(PtrInfo.V.get<const Value *>()->getType())) &&
"invalid pointer value");
@ -1043,11 +1042,21 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
uint64_t s, Align a,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(ptrinfo, f,
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
/// Profile - Gather unique data for the object.
///
void MachineMemOperand::Profile(FoldingSetNodeID &ID) const {
ID.AddInteger(getOffset());
ID.AddInteger(Size);
ID.AddInteger(getMemoryType().getUniqueRAWLLTData());
ID.AddPointer(getOpaqueValue());
ID.AddInteger(getFlags());
ID.AddInteger(getBaseAlign().value());

9
lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
View File

@ -1007,7 +1007,7 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
auto List = MIRBuilder.buildLoad(
PtrTy, ListPtr,
*MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad,
PtrSize, PtrAlign));
PtrTy, PtrAlign));
MachineInstrBuilder DstPtr;
if (Alignment > PtrAlign) {
@ -1019,11 +1019,12 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
} else
DstPtr = List;
uint64_t ValSize = MRI.getType(Dst).getSizeInBits() / 8;
LLT ValTy = MRI.getType(Dst);
uint64_t ValSize = ValTy.getSizeInBits() / 8;
MIRBuilder.buildLoad(
Dst, DstPtr,
*MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad,
ValSize, std::max(Alignment, PtrAlign)));
ValTy, std::max(Alignment, PtrAlign)));
auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign));
@ -1032,7 +1033,7 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
MIRBuilder.buildStore(NewList, ListPtr,
*MF.getMachineMemOperand(MachinePointerInfo(),
MachineMemOperand::MOStore,
PtrSize, PtrAlign));
PtrTy, PtrAlign));
MI.eraseFromParent();
return true;

3
lib/Target/ARM/ARMCallLowering.cpp
View File

@ -128,8 +128,7 @@ struct ARMOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
Register ExtReg = extendRegister(ValVReg, VA);
auto MMO = MIRBuilder.getMF().getMachineMemOperand(
MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(),
Align(1));
MPO, MachineMemOperand::MOStore, LLT(VA.getLocVT()), Align(1));
MIRBuilder.buildStore(ExtReg, Addr, *MMO);
}

152
test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
View File

@ -3330,147 +3330,11 @@ define amdgpu_kernel void @sdivrem_i3(i3 addrspace(1)* %out0, i3 addrspace(1)* %
ret void
}
define amdgpu_kernel void @sdivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
; GFX8-LABEL: sdivrem_i27:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_bfe_i32 s1, s1, 0x1b0000
; GFX8-NEXT: s_ashr_i32 s6, s1, 31
; GFX8-NEXT: s_add_i32 s1, s1, s6
; GFX8-NEXT: s_xor_b32 s7, s1, s6
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s7
; GFX8-NEXT: s_sub_i32 s1, 0, s7
; GFX8-NEXT: s_bfe_i32 s0, s0, 0x1b0000
; GFX8-NEXT: s_ashr_i32 s8, s0, 31
; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX8-NEXT: s_add_i32 s0, s0, s8
; GFX8-NEXT: s_xor_b32 s9, s0, s8
; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX8-NEXT: v_mul_lo_u32 v1, s1, v0
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX8-NEXT: s_xor_b32 s4, s8, s6
; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
; GFX8-NEXT: v_mul_hi_u32 v2, s9, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mul_lo_u32 v3, v2, s7
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2
; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s9, v3
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s7, v3
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s7, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s7, v3
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s7, v3
; GFX8-NEXT: v_xor_b32_e32 v2, s4, v2
; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s4, v2
; GFX8-NEXT: v_xor_b32_e32 v3, s8, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s8, v3
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: flat_store_dword v[0:1], v3
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: sdivrem_i27:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s1, s1, 0x1b0000
; GFX9-NEXT: s_ashr_i32 s6, s1, 31
; GFX9-NEXT: s_add_i32 s1, s1, s6
; GFX9-NEXT: s_xor_b32 s7, s1, s6
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
; GFX9-NEXT: s_sub_i32 s1, 0, s7
; GFX9-NEXT: s_bfe_i32 s0, s0, 0x1b0000
; GFX9-NEXT: s_ashr_i32 s8, s0, 31
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: s_add_i32 s0, s0, s8
; GFX9-NEXT: s_xor_b32 s9, s0, s8
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mul_lo_u32 v1, s1, v0
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: s_xor_b32 s4, s8, s6
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
; GFX9-NEXT: v_mul_hi_u32 v0, s9, v0
; GFX9-NEXT: v_mul_lo_u32 v1, v0, s7
; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v1
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; GFX9-NEXT: v_subrev_u32_e32 v3, s7, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v1
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; GFX9-NEXT: v_subrev_u32_e32 v3, s7, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX9-NEXT: v_xor_b32_e32 v0, s4, v0
; GFX9-NEXT: v_xor_b32_e32 v1, s8, v1
; GFX9-NEXT: v_subrev_u32_e32 v0, s4, v0
; GFX9-NEXT: v_subrev_u32_e32 v1, s8, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
; GFX9-NEXT: global_store_dword v2, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: sdivrem_i27:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_bfe_i32 s1, s1, 0x1b0000
; GFX10-NEXT: s_bfe_i32 s0, s0, 0x1b0000
; GFX10-NEXT: s_ashr_i32 s6, s1, 31
; GFX10-NEXT: s_ashr_i32 s8, s0, 31
; GFX10-NEXT: s_add_i32 s1, s1, s6
; GFX10-NEXT: s_add_i32 s0, s0, s8
; GFX10-NEXT: s_xor_b32 s7, s1, s6
; GFX10-NEXT: s_xor_b32 s0, s0, s8
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s7
; GFX10-NEXT: s_sub_i32 s1, 0, s7
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_mul_lo_u32 v1, s1, v0
; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0
; GFX10-NEXT: v_mul_lo_u32 v1, v0, s7
; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0
; GFX10-NEXT: v_sub_nc_u32_e32 v1, s0, v1
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: s_xor_b32 s4, s8, s6
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s7, v1
; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s7, v1
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s7, v1
; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s7, v1
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_xor_b32_e32 v0, s4, v0
; GFX10-NEXT: v_xor_b32_e32 v1, s8, v1
; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s4, v0
; GFX10-NEXT: v_subrev_nc_u32_e32 v1, s8, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-NEXT: s_endpgm
%div = sdiv i27 %x, %y
store i27 %div, i27 addrspace(1)* %out0
%rem = srem i27 %x, %y
store i27 %rem, i27 addrspace(1)* %out1
ret void
}
; FIXME: Reenable test
; define amdgpu_kernel void @sdivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
; %div = sdiv i27 %x, %y
; store i27 %div, i27 addrspace(1)* %out0
; %rem = srem i27 %x, %y
; store i27 %rem, i27 addrspace(1)* %out1
; ret void
; }

122
test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
View File

@ -2634,117 +2634,11 @@ define amdgpu_kernel void @udivrem_i3(i3 addrspace(1)* %out0, i3 addrspace(1)* %
ret void
}
define amdgpu_kernel void @udivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
; GFX8-LABEL: udivrem_i27:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX8-NEXT: s_mov_b32 s2, 0x7ffffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_and_b32 s6, s1, s2
; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s6
; GFX8-NEXT: s_sub_i32 s1, 0, s6
; GFX8-NEXT: s_and_b32 s7, s0, s2
; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX8-NEXT: v_mul_lo_u32 v1, s1, v0
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
; GFX8-NEXT: v_mul_hi_u32 v2, s7, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mul_lo_u32 v3, v2, s6
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2
; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s7, v3
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s6, v3
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s6, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2
; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s6, v3
; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s6, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: flat_store_dword v[0:1], v3
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udivrem_i27:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX9-NEXT: s_mov_b32 s2, 0x7ffffff
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s6, s1, s2
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
; GFX9-NEXT: s_sub_i32 s1, 0, s6
; GFX9-NEXT: s_and_b32 s7, s0, s2
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mul_lo_u32 v1, s1, v0
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
; GFX9-NEXT: v_mul_hi_u32 v0, s7, v0
; GFX9-NEXT: v_mul_lo_u32 v1, v0, s6
; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
; GFX9-NEXT: v_sub_u32_e32 v1, s7, v1
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; GFX9-NEXT: v_subrev_u32_e32 v3, s6, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; GFX9-NEXT: v_subrev_u32_e32 v3, s6, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
; GFX9-NEXT: global_store_dword v2, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: udivrem_i27:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10
; GFX10-NEXT: s_mov_b32 s2, 0x7ffffff
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s6, s1, s2
; GFX10-NEXT: s_and_b32 s0, s0, s2
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6
; GFX10-NEXT: s_sub_i32 s1, 0, s6
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_mul_lo_u32 v1, s1, v0
; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1
; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0
; GFX10-NEXT: v_mul_lo_u32 v1, v0, s6
; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0
; GFX10-NEXT: v_sub_nc_u32_e32 v1, s0, v1
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s6, v1
; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s6, v1
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s6, v1
; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s6, v1
; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-NEXT: global_store_dword v2, v1, s[2:3]
; GFX10-NEXT: s_endpgm
%div = udiv i27 %x, %y
store i27 %div, i27 addrspace(1)* %out0
%rem = urem i27 %x, %y
store i27 %rem, i27 addrspace(1)* %out1
ret void
}
; FIXME: Reenable test
; define amdgpu_kernel void @udivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) {
; %div = udiv i27 %x, %y
; store i27 %div, i27 addrspace(1)* %out0
; %rem = urem i27 %x, %y
; store i27 %rem, i27 addrspace(1)* %out1
; ret void
; }