From aa0b4488a5d75f15e422a6b09bc54c9f24f965a9 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 19 May 2021 22:06:14 -0400 Subject: [PATCH] CodeGen: Store LLT instead of uint64_t in MachineMemOperand GlobalISel is relying on regular MachineMemOperands to track all of the memory properties of accesses. Just the raw byte size is insufficent to disambiguate all situations. For example, if we need to split an unaligned extending load, we need to know the number of bits in the original source value and can't infer it from the result type. This is also a problem for extending vector loads. This does decrease the maximum representable size from the full uint64_t bytes to a maximum of 16-bits. No in tree testcases hit this, other than places using UINT64_MAX for unknown sizes. This may be an issue for G_MEMCPY and co., although they can just use unknown size for large static sizes. This also has potential for backend abuse by relying on the type when it really shouldn't be relevant after selection. This does not include the necessary MIR printer/parser changes to represent this. --- include/llvm/CodeGen/MachineFunction.h | 16 +- include/llvm/CodeGen/MachineMemOperand.h | 29 +++- include/llvm/Support/LowLevelTypeImpl.h | 1 + lib/CodeGen/GlobalISel/IRTranslator.cpp | 38 +++-- lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 12 +- lib/CodeGen/LowLevelType.cpp | 4 +- lib/CodeGen/MachineFunction.cpp | 22 ++- lib/CodeGen/MachineOperand.cpp | 19 ++- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 9 +- lib/Target/ARM/ARMCallLowering.cpp | 3 +- test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 152 +----------------- test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 122 +------------- 12 files changed, 123 insertions(+), 304 deletions(-) diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index 01e3bc03255..e9ce813428d 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -865,12 +865,23 @@ public: AtomicOrdering Ordering = AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); + MachineMemOperand *getMachineMemOperand( + MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, + Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(), + const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System, + AtomicOrdering Ordering = AtomicOrdering::NotAtomic, + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); + /// getMachineMemOperand - Allocate a new MachineMemOperand by copying /// an existing one, adjusting by an offset and using the given size. /// MachineMemOperands are owned by the MachineFunction and need not be /// explicitly deallocated. MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, - int64_t Offset, uint64_t Size); + int64_t Offset, LLT Ty); + MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, + int64_t Offset, uint64_t Size) { + return getMachineMemOperand(MMO, Offset, LLT::scalar(8 * Size)); + } /// getMachineMemOperand - Allocate a new MachineMemOperand by copying /// an existing one, replacing only the MachinePointerInfo and size. @@ -879,6 +890,9 @@ public: MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, uint64_t Size); + MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, + const MachinePointerInfo &PtrInfo, + LLT Ty); /// Allocate a new MachineMemOperand by copying an existing one, /// replacing only AliasAnalysis information. MachineMemOperands are owned diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index 0bef1c1035d..8c1fb1d7785 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -22,6 +22,7 @@ #include "llvm/IR/Value.h" // PointerLikeTypeTraits #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/LowLevelTypeImpl.h" namespace llvm { @@ -168,7 +169,11 @@ private: }; MachinePointerInfo PtrInfo; - uint64_t Size; + + /// Track the memory type of the access. An access size which is unknown or + /// too large to be represented by LLT should use the invalid LLT. + LLT MemoryType; + Flags FlagVals; Align BaseAlign; MachineAtomicInfo AtomicInfo; @@ -187,6 +192,12 @@ public: SyncScope::ID SSID = SyncScope::System, AtomicOrdering Ordering = AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); + MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a, + const AAMDNodes &AAInfo = AAMDNodes(), + const MDNode *Ranges = nullptr, + SyncScope::ID SSID = SyncScope::System, + AtomicOrdering Ordering = AtomicOrdering::NotAtomic, + AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); const MachinePointerInfo &getPointerInfo() const { return PtrInfo; } @@ -217,11 +228,23 @@ public: unsigned getAddrSpace() const { return PtrInfo.getAddrSpace(); } + /// Return the memory type of the memory reference. This should only be relied + /// on for GlobalISel G_* operation legalization. + LLT getMemoryType() const { return MemoryType; } + /// Return the size in bytes of the memory reference. - uint64_t getSize() const { return Size; } + uint64_t getSize() const { + return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0); + } /// Return the size in bits of the memory reference. - uint64_t getSizeInBits() const { return Size * 8; } + uint64_t getSizeInBits() const { + return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0); + } + + LLT getType() const { + return MemoryType; + } /// Return the minimum known alignment in bytes of the actual memory /// reference. diff --git a/include/llvm/Support/LowLevelTypeImpl.h b/include/llvm/Support/LowLevelTypeImpl.h index 379e37101e9..3d631dc6d53 100644 --- a/include/llvm/Support/LowLevelTypeImpl.h +++ b/include/llvm/Support/LowLevelTypeImpl.h @@ -366,6 +366,7 @@ private: } } +public: uint64_t getUniqueRAWLLTData() const { return ((uint64_t)RawData) << 2 | ((uint64_t)IsPointer) << 1 | ((uint64_t)IsVector); diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp index 42c63849910..a3a8f10c011 100644 --- a/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1300,7 +1300,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { AAMDNodes AAMetadata; LI.getAAMetadata(AAMetadata); auto MMO = MF->getMachineMemOperand( - Ptr, Flags, MRI->getType(Regs[i]).getSizeInBytes(), + Ptr, Flags, MRI->getType(Regs[i]), commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, Ranges, LI.getSyncScopeID(), LI.getOrdering()); MIRBuilder.buildLoad(Regs[i], Addr, *MMO); @@ -1342,7 +1342,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) { AAMDNodes AAMetadata; SI.getAAMetadata(AAMetadata); auto MMO = MF->getMachineMemOperand( - Ptr, Flags, MRI->getType(Vals[i]).getSizeInBytes(), + Ptr, Flags, MRI->getType(Vals[i]), commonAlignment(BaseAlign, Offsets[i] / 8), AAMetadata, nullptr, SI.getSyncScopeID(), SI.getOrdering()); MIRBuilder.buildStore(Vals[i], Addr, *MMO); @@ -1627,12 +1627,14 @@ void IRTranslator::getStackGuard(Register DstReg, if (!Global) return; + unsigned AddrSpace = Global->getType()->getPointerAddressSpace(); + LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace)); + MachinePointerInfo MPInfo(Global); auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable; - MachineMemOperand *MemRef = - MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8, - DL->getPointerABIAlignment(0)); + MachineMemOperand *MemRef = MF->getMachineMemOperand( + MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace)); MIB.setMemRefs({MemRef}); } @@ -2067,7 +2069,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore | MachineMemOperand::MOVolatile, - PtrTy.getSizeInBits() / 8, Align(8))); + PtrTy, Align(8))); return true; } case Intrinsic::stacksave: { @@ -2364,10 +2366,11 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) { Align Alignment = Info.align.getValueOr( DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext()))); - - uint64_t Size = Info.memVT.getStoreSize(); + LLT MemTy = Info.memVT.isSimple() + ? getLLTForMVT(Info.memVT.getSimpleVT()) + : LLT::scalar(Info.memVT.getStoreSizeInBits()); MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal), - Info.flags, Size, Alignment)); + Info.flags, MemTy, Alignment)); } return true; @@ -2733,9 +2736,6 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U, auto &TLI = *MF->getSubtarget().getTargetLowering(); auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); - Type *ResType = I.getType(); - Type *ValType = ResType->Type::getStructElementType(0); - auto Res = getOrCreateVRegs(I); Register OldValRes = Res[0]; Register SuccessRes = Res[1]; @@ -2749,9 +2749,9 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U, MIRBuilder.buildAtomicCmpXchgWithSuccess( OldValRes, SuccessRes, Addr, Cmp, NewVal, *MF->getMachineMemOperand( - MachinePointerInfo(I.getPointerOperand()), Flags, - DL->getTypeStoreSize(ValType), getMemOpAlign(I), AAMetadata, nullptr, - I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering())); + MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp), + getMemOpAlign(I), AAMetadata, nullptr, I.getSyncScopeID(), + I.getSuccessOrdering(), I.getFailureOrdering())); return true; } @@ -2761,8 +2761,6 @@ bool IRTranslator::translateAtomicRMW(const User &U, auto &TLI = *MF->getSubtarget().getTargetLowering(); auto Flags = TLI.getAtomicMemOperandFlags(I, *DL); - Type *ResType = I.getType(); - Register Res = getOrCreateVReg(I); Register Addr = getOrCreateVReg(*I.getPointerOperand()); Register Val = getOrCreateVReg(*I.getValOperand()); @@ -2818,9 +2816,9 @@ bool IRTranslator::translateAtomicRMW(const User &U, MIRBuilder.buildAtomicRMW( Opcode, Res, Addr, Val, *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), - Flags, DL->getTypeStoreSize(ResType), - getMemOpAlign(I), AAMetadata, nullptr, - I.getSyncScopeID(), I.getOrdering())); + Flags, MRI->getType(Val), getMemOpAlign(I), + AAMetadata, nullptr, I.getSyncScopeID(), + I.getOrdering())); return true; } diff --git a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 3352d1989d1..54ac62793b0 100644 --- a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -347,10 +347,9 @@ MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr, MMOFlags |= MachineMemOperand::MOLoad; assert((MMOFlags & MachineMemOperand::MOStore) == 0); - uint64_t Size = MemoryLocation::getSizeOrUnknown( - TypeSize::Fixed(Dst.getLLTTy(*getMRI()).getSizeInBytes())); + LLT Ty = Dst.getLLTTy(*getMRI()); MachineMemOperand *MMO = - getMF().getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); + getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo); return buildLoad(Dst, Addr, *MMO); } @@ -373,7 +372,7 @@ MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset( MachineMemOperand &BaseMMO, int64_t Offset) { LLT LoadTy = Dst.getLLTTy(*getMRI()); MachineMemOperand *OffsetMMO = - getMF().getMachineMemOperand(&BaseMMO, Offset, LoadTy.getSizeInBytes()); + getMF().getMachineMemOperand(&BaseMMO, Offset, LoadTy); if (Offset == 0) // This may be a size or type changing load. return buildLoad(Dst, BasePtr, *OffsetMMO); @@ -406,10 +405,9 @@ MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr, MMOFlags |= MachineMemOperand::MOStore; assert((MMOFlags & MachineMemOperand::MOLoad) == 0); - uint64_t Size = MemoryLocation::getSizeOrUnknown( - TypeSize::Fixed(Val.getLLTTy(*getMRI()).getSizeInBytes())); + LLT Ty = Val.getLLTTy(*getMRI()); MachineMemOperand *MMO = - getMF().getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); + getMF().getMachineMemOperand(PtrInfo, MMOFlags, Ty, Alignment, AAInfo); return buildStore(Val, Addr, *MMO); } diff --git a/lib/CodeGen/LowLevelType.cpp b/lib/CodeGen/LowLevelType.cpp index 1cfd4fee8e5..62e9c6b629d 100644 --- a/lib/CodeGen/LowLevelType.cpp +++ b/lib/CodeGen/LowLevelType.cpp @@ -56,8 +56,8 @@ LLT llvm::getLLTForMVT(MVT Ty) { if (!Ty.isVector()) return LLT::scalar(Ty.getSizeInBits()); - return LLT::vector(Ty.getVectorElementCount(), - Ty.getVectorElementType().getSizeInBits()); + return LLT::scalarOrVector(Ty.getVectorElementCount(), + Ty.getVectorElementType().getSizeInBits()); } const llvm::fltSemantics &llvm::getFltSemanticForLLT(LLT Ty) { diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index dff38f09022..39feb92a975 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -437,6 +437,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand( SSID, Ordering, FailureOrdering); } +MachineMemOperand *MachineFunction::getMachineMemOperand( + MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, + Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges, + SyncScope::ID SSID, AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) { + return new (Allocator) + MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges, SSID, + Ordering, FailureOrdering); +} + MachineMemOperand *MachineFunction::getMachineMemOperand( const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, uint64_t Size) { return new (Allocator) @@ -445,9 +455,17 @@ MachineMemOperand *MachineFunction::getMachineMemOperand( MMO->getSuccessOrdering(), MMO->getFailureOrdering()); } +MachineMemOperand *MachineFunction::getMachineMemOperand( + const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) { + return new (Allocator) + MachineMemOperand(PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(), + AAMDNodes(), nullptr, MMO->getSyncScopeID(), + MMO->getSuccessOrdering(), MMO->getFailureOrdering()); +} + MachineMemOperand * MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, - int64_t Offset, uint64_t Size) { + int64_t Offset, LLT Ty) { const MachinePointerInfo &PtrInfo = MMO->getPointerInfo(); // If there is no pointer value, the offset isn't tracked so we need to adjust @@ -459,7 +477,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, // Do not preserve ranges, since we don't necessarily know what the high bits // are anymore. return new (Allocator) MachineMemOperand( - PtrInfo.getWithOffset(Offset), MMO->getFlags(), Size, Alignment, + PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment, MMO->getAAInfo(), nullptr, MMO->getSyncScopeID(), MMO->getSuccessOrdering(), MMO->getFailureOrdering()); } diff --git a/lib/CodeGen/MachineOperand.cpp b/lib/CodeGen/MachineOperand.cpp index 79f59cf67e5..b020c423757 100644 --- a/lib/CodeGen/MachineOperand.cpp +++ b/lib/CodeGen/MachineOperand.cpp @@ -1023,13 +1023,12 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) { } MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, - uint64_t s, Align a, - const AAMDNodes &AAInfo, + LLT type, Align a, const AAMDNodes &AAInfo, const MDNode *Ranges, SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering) - : PtrInfo(ptrinfo), Size(s), FlagVals(f), BaseAlign(a), AAInfo(AAInfo), - Ranges(Ranges) { + : PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a), + AAInfo(AAInfo), Ranges(Ranges) { assert((PtrInfo.V.isNull() || PtrInfo.V.is() || isa(PtrInfo.V.get()->getType())) && "invalid pointer value"); @@ -1043,11 +1042,21 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, assert(getFailureOrdering() == FailureOrdering && "Value truncated"); } +MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, + uint64_t s, Align a, + const AAMDNodes &AAInfo, + const MDNode *Ranges, SyncScope::ID SSID, + AtomicOrdering Ordering, + AtomicOrdering FailureOrdering) + : MachineMemOperand(ptrinfo, f, + s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a, + AAInfo, Ranges, SSID, Ordering, FailureOrdering) {} + /// Profile - Gather unique data for the object. /// void MachineMemOperand::Profile(FoldingSetNodeID &ID) const { ID.AddInteger(getOffset()); - ID.AddInteger(Size); + ID.AddInteger(getMemoryType().getUniqueRAWLLTData()); ID.AddPointer(getOpaqueValue()); ID.AddInteger(getFlags()); ID.AddInteger(getBaseAlign().value()); diff --git a/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 0f778284ade..80f7cdded3e 100644 --- a/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1007,7 +1007,7 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, auto List = MIRBuilder.buildLoad( PtrTy, ListPtr, *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, - PtrSize, PtrAlign)); + PtrTy, PtrAlign)); MachineInstrBuilder DstPtr; if (Alignment > PtrAlign) { @@ -1019,11 +1019,12 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, } else DstPtr = List; - uint64_t ValSize = MRI.getType(Dst).getSizeInBits() / 8; + LLT ValTy = MRI.getType(Dst); + uint64_t ValSize = ValTy.getSizeInBits() / 8; MIRBuilder.buildLoad( Dst, DstPtr, *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, - ValSize, std::max(Alignment, PtrAlign))); + ValTy, std::max(Alignment, PtrAlign))); auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign)); @@ -1032,7 +1033,7 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, MIRBuilder.buildStore(NewList, ListPtr, *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOStore, - PtrSize, PtrAlign)); + PtrTy, PtrAlign)); MI.eraseFromParent(); return true; diff --git a/lib/Target/ARM/ARMCallLowering.cpp b/lib/Target/ARM/ARMCallLowering.cpp index 78fcbbb632c..ee600477ed9 100644 --- a/lib/Target/ARM/ARMCallLowering.cpp +++ b/lib/Target/ARM/ARMCallLowering.cpp @@ -128,8 +128,7 @@ struct ARMOutgoingValueHandler : public CallLowering::OutgoingValueHandler { Register ExtReg = extendRegister(ValVReg, VA); auto MMO = MIRBuilder.getMF().getMachineMemOperand( - MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(), - Align(1)); + MPO, MachineMemOperand::MOStore, LLT(VA.getLocVT()), Align(1)); MIRBuilder.buildStore(ExtReg, Addr, *MMO); } diff --git a/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll index 984b6a21e3a..ce1066abc37 100644 --- a/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll +++ b/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll @@ -3330,147 +3330,11 @@ define amdgpu_kernel void @sdivrem_i3(i3 addrspace(1)* %out0, i3 addrspace(1)* % ret void } -define amdgpu_kernel void @sdivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) { -; GFX8-LABEL: sdivrem_i27: -; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_bfe_i32 s1, s1, 0x1b0000 -; GFX8-NEXT: s_ashr_i32 s6, s1, 31 -; GFX8-NEXT: s_add_i32 s1, s1, s6 -; GFX8-NEXT: s_xor_b32 s7, s1, s6 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s7 -; GFX8-NEXT: s_sub_i32 s1, 0, s7 -; GFX8-NEXT: s_bfe_i32 s0, s0, 0x1b0000 -; GFX8-NEXT: s_ashr_i32 s8, s0, 31 -; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX8-NEXT: s_add_i32 s0, s0, s8 -; GFX8-NEXT: s_xor_b32 s9, s0, s8 -; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX8-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX8-NEXT: s_xor_b32 s4, s8, s6 -; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 -; GFX8-NEXT: v_mul_hi_u32 v2, s9, v0 -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v0, s0 -; GFX8-NEXT: v_mov_b32_e32 v1, s1 -; GFX8-NEXT: v_mul_lo_u32 v3, v2, s7 -; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s9, v3 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s7, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s7, v3 -; GFX8-NEXT: v_xor_b32_e32 v2, s4, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s4, v2 -; GFX8-NEXT: v_xor_b32_e32 v3, s8, v3 -; GFX8-NEXT: flat_store_dword v[0:1], v2 -; GFX8-NEXT: v_mov_b32_e32 v0, s2 -; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s8, v3 -; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: flat_store_dword v[0:1], v3 -; GFX8-NEXT: s_endpgm -; -; GFX9-LABEL: sdivrem_i27: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX9-NEXT: v_mov_b32_e32 v2, 0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_bfe_i32 s1, s1, 0x1b0000 -; GFX9-NEXT: s_ashr_i32 s6, s1, 31 -; GFX9-NEXT: s_add_i32 s1, s1, s6 -; GFX9-NEXT: s_xor_b32 s7, s1, s6 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7 -; GFX9-NEXT: s_sub_i32 s1, 0, s7 -; GFX9-NEXT: s_bfe_i32 s0, s0, 0x1b0000 -; GFX9-NEXT: s_ashr_i32 s8, s0, 31 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_add_i32 s0, s0, s8 -; GFX9-NEXT: s_xor_b32 s9, s0, s8 -; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX9-NEXT: s_xor_b32 s4, s8, s6 -; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX9-NEXT: v_add_u32_e32 v0, v0, v1 -; GFX9-NEXT: v_mul_hi_u32 v0, s9, v0 -; GFX9-NEXT: v_mul_lo_u32 v1, v0, s7 -; GFX9-NEXT: v_add_u32_e32 v3, 1, v0 -; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_subrev_u32_e32 v3, s7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_add_u32_e32 v3, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_subrev_u32_e32 v3, s7, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_xor_b32_e32 v0, s4, v0 -; GFX9-NEXT: v_xor_b32_e32 v1, s8, v1 -; GFX9-NEXT: v_subrev_u32_e32 v0, s4, v0 -; GFX9-NEXT: v_subrev_u32_e32 v1, s8, v1 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_store_dword v2, v0, s[0:1] -; GFX9-NEXT: global_store_dword v2, v1, s[2:3] -; GFX9-NEXT: s_endpgm -; -; GFX10-LABEL: sdivrem_i27: -; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_bfe_i32 s1, s1, 0x1b0000 -; GFX10-NEXT: s_bfe_i32 s0, s0, 0x1b0000 -; GFX10-NEXT: s_ashr_i32 s6, s1, 31 -; GFX10-NEXT: s_ashr_i32 s8, s0, 31 -; GFX10-NEXT: s_add_i32 s1, s1, s6 -; GFX10-NEXT: s_add_i32 s0, s0, s8 -; GFX10-NEXT: s_xor_b32 s7, s1, s6 -; GFX10-NEXT: s_xor_b32 s0, s0, s8 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s7 -; GFX10-NEXT: s_sub_i32 s1, 0, s7 -; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX10-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1 -; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX10-NEXT: v_mul_lo_u32 v1, v0, s7 -; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0 -; GFX10-NEXT: v_sub_nc_u32_e32 v1, s0, v1 -; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX10-NEXT: s_xor_b32 s4, s8, s6 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s7, v1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s7, v1 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s7, v1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s7, v1 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_mov_b32_e32 v2, 0 -; GFX10-NEXT: v_xor_b32_e32 v0, s4, v0 -; GFX10-NEXT: v_xor_b32_e32 v1, s8, v1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s4, v0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v1, s8, v1 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: global_store_dword v2, v0, s[0:1] -; GFX10-NEXT: global_store_dword v2, v1, s[2:3] -; GFX10-NEXT: s_endpgm - %div = sdiv i27 %x, %y - store i27 %div, i27 addrspace(1)* %out0 - %rem = srem i27 %x, %y - store i27 %rem, i27 addrspace(1)* %out1 - ret void -} +; FIXME: Reenable test +; define amdgpu_kernel void @sdivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) { +; %div = sdiv i27 %x, %y +; store i27 %div, i27 addrspace(1)* %out0 +; %rem = srem i27 %x, %y +; store i27 %rem, i27 addrspace(1)* %out1 +; ret void +; } diff --git a/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll index 89c4569aaa2..0cb228f6c65 100644 --- a/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll +++ b/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll @@ -2634,117 +2634,11 @@ define amdgpu_kernel void @udivrem_i3(i3 addrspace(1)* %out0, i3 addrspace(1)* % ret void } -define amdgpu_kernel void @udivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) { -; GFX8-LABEL: udivrem_i27: -; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX8-NEXT: s_mov_b32 s2, 0x7ffffff -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_and_b32 s6, s1, s2 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX8-NEXT: s_sub_i32 s1, 0, s6 -; GFX8-NEXT: s_and_b32 s7, s0, s2 -; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX8-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 -; GFX8-NEXT: v_mul_hi_u32 v2, s7, v0 -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v0, s0 -; GFX8-NEXT: v_mov_b32_e32 v1, s1 -; GFX8-NEXT: v_mul_lo_u32 v3, v2, s6 -; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s7, v3 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s6, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s6, v3 -; GFX8-NEXT: flat_store_dword v[0:1], v2 -; GFX8-NEXT: v_mov_b32_e32 v0, s2 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX8-NEXT: v_mov_b32_e32 v1, s3 -; GFX8-NEXT: flat_store_dword v[0:1], v3 -; GFX8-NEXT: s_endpgm -; -; GFX9-LABEL: udivrem_i27: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX9-NEXT: s_mov_b32 s2, 0x7ffffff -; GFX9-NEXT: v_mov_b32_e32 v2, 0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_and_b32 s6, s1, s2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX9-NEXT: s_sub_i32 s1, 0, s6 -; GFX9-NEXT: s_and_b32 s7, s0, s2 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX9-NEXT: v_add_u32_e32 v0, v0, v1 -; GFX9-NEXT: v_mul_hi_u32 v0, s7, v0 -; GFX9-NEXT: v_mul_lo_u32 v1, v0, s6 -; GFX9-NEXT: v_add_u32_e32 v3, 1, v0 -; GFX9-NEXT: v_sub_u32_e32 v1, s7, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_subrev_u32_e32 v3, s6, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_add_u32_e32 v3, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_subrev_u32_e32 v3, s6, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_store_dword v2, v0, s[0:1] -; GFX9-NEXT: global_store_dword v2, v1, s[2:3] -; GFX9-NEXT: s_endpgm -; -; GFX10-LABEL: udivrem_i27: -; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX10-NEXT: s_mov_b32 s2, 0x7ffffff -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_and_b32 s6, s1, s2 -; GFX10-NEXT: s_and_b32 s0, s0, s2 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX10-NEXT: s_sub_i32 s1, 0, s6 -; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX10-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX10-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v1 -; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX10-NEXT: v_mul_lo_u32 v1, v0, s6 -; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0 -; GFX10-NEXT: v_sub_nc_u32_e32 v1, s0, v1 -; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s6, v1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s6, v1 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: v_add_nc_u32_e32 v2, 1, v0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s6, v1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s6, v1 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo -; GFX10-NEXT: v_mov_b32_e32 v2, 0 -; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: global_store_dword v2, v0, s[0:1] -; GFX10-NEXT: global_store_dword v2, v1, s[2:3] -; GFX10-NEXT: s_endpgm - %div = udiv i27 %x, %y - store i27 %div, i27 addrspace(1)* %out0 - %rem = urem i27 %x, %y - store i27 %rem, i27 addrspace(1)* %out1 - ret void -} +; FIXME: Reenable test +; define amdgpu_kernel void @udivrem_i27(i27 addrspace(1)* %out0, i27 addrspace(1)* %out1, i27 %x, i27 %y) { +; %div = udiv i27 %x, %y +; store i27 %div, i27 addrspace(1)* %out0 +; %rem = urem i27 %x, %y +; store i27 %rem, i27 addrspace(1)* %out1 +; ret void +; }