RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 20:51:52 +01:00
Code Issues Projects Releases Wiki Activity

Add OffsetIsScalable to getMemOperandWithOffset

Browse Source 
Summary:
Making `Scale` a `TypeSize` in AArch64InstrInfo::getMemOpInfo,
has the effect that all places where this information is used
(notably, TargetInstrInfo::getMemOperandWithOffset) will need
to consider Scale - and derived, Offset - possibly being scalable.

This patch adds a new operand `bool &OffsetIsScalable` to
TargetInstrInfo::getMemOperandWithOffset and fixes up all
the places where this function is used, to consider the
offset possibly being scalable.

In most cases, this means bailing out because the algorithm does not
(or cannot) support scalable offsets in places where it does some
form of alias checking for example.

Reviewers: rovka, efriedma, kristof.beyls

Reviewed By: efriedma

Subscribers: wuzish, kerbowa, MatzeB, arsenm, nemanjai, jvesely, nhaehnle, hiraditya, kbarton, javed.absar, asb, rbar, johnrusso, simoncook, sabuasal, niosHD, jrtc27, MaskRay, zzheng, edward-jones, rogfer01, MartinMosbeck, brucehoult, the_o, PkmX, jocewei, jsji, Jim, lenary, s.egerton, pzheng, sameer.abuasal, apazos, luismarques, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D72758
This commit is contained in:
Sander de Smalen 2020-02-18 14:32:26 +00:00
parent a8a9374ec7
commit 04e619f3c1
19 changed files with 163 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/llvm/CodeGen/TargetInstrInfo.h
View File

@ -1242,6 +1242,7 @@ public:
/// to handle a single base operand.
bool getMemOperandWithOffset(const MachineInstr &MI,
const MachineOperand *&BaseOp, int64_t &Offset,
bool &OffsetIsScalable,
const TargetRegisterInfo *TRI) const;
/// Get the base operands and byte offset of an instruction that reads/writes
@ -1250,9 +1251,11 @@ public:
/// It returns false if no base operands and offset was found.
/// It is not guaranteed to always recognize base operands and offsets in all
/// cases.
virtual bool getMemOperandsWithOffset(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const {
virtual bool
getMemOperandsWithOffset(const MachineInstr &MI,
SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, bool &OffsetIsScalable,
const TargetRegisterInfo *TRI) const {
return false;
}

8
lib/CodeGen/ImplicitNullChecks.cpp
View File

@ -364,12 +364,18 @@ ImplicitNullChecks::isSuitableMemoryOp(const MachineInstr &MI,
unsigned PointerReg,
ArrayRef<MachineInstr *> PrevInsts) {
int64_t Offset;
bool OffsetIsScalable;
const MachineOperand *BaseOp;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI) ||
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI) ||
!BaseOp->isReg() || BaseOp->getReg() != PointerReg)
return SR_Unsuitable;
// FIXME: This algorithm assumes instructions have fixed-size offsets.
if (OffsetIsScalable)
return SR_Unsuitable;
// We want the mem access to be issued at a sane offset from PointerReg,
// so that if PointerReg is null then the access reliably page faults.
if (!(MI.mayLoadOrStore() && !MI.isPredicable() &&

25
lib/CodeGen/MachinePipeliner.cpp
View File

@ -693,9 +693,13 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) {
// offset, then mark the dependence as loop carried potentially.
const MachineOperand *BaseOp1, *BaseOp2;
int64_t Offset1, Offset2;
if (TII->getMemOperandWithOffset(LdMI, BaseOp1, Offset1, TRI) &&
TII->getMemOperandWithOffset(MI, BaseOp2, Offset2, TRI)) {
bool Offset1IsScalable, Offset2IsScalable;
if (TII->getMemOperandWithOffset(LdMI, BaseOp1, Offset1,
Offset1IsScalable, TRI) &&
TII->getMemOperandWithOffset(MI, BaseOp2, Offset2,
Offset2IsScalable, TRI)) {
if (BaseOp1->isIdenticalTo(*BaseOp2) &&
Offset1IsScalable == Offset2IsScalable &&
(int)Offset1 < (int)Offset2) {
assert(TII->areMemAccessesTriviallyDisjoint(LdMI, MI) &&
"What happened to the chain edge?");
@ -2058,7 +2062,12 @@ bool SwingSchedulerDAG::computeDelta(MachineInstr &MI, unsigned &Delta) {
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
const MachineOperand *BaseOp;
int64_t Offset;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI))
bool OffsetIsScalable;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
return false;
// FIXME: This algorithm assumes instructions have fixed-size offsets.
if (OffsetIsScalable)
return false;
if (!BaseOp->isReg())
@ -2236,11 +2245,17 @@ bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep,
const MachineOperand *BaseOpS, *BaseOpD;
int64_t OffsetS, OffsetD;
bool OffsetSIsScalable, OffsetDIsScalable;
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
if (!TII->getMemOperandWithOffset(*SI, BaseOpS, OffsetS, TRI) ||
!TII->getMemOperandWithOffset(*DI, BaseOpD, OffsetD, TRI))
if (!TII->getMemOperandWithOffset(*SI, BaseOpS, OffsetS, OffsetSIsScalable,
TRI) ||
!TII->getMemOperandWithOffset(*DI, BaseOpD, OffsetD, OffsetDIsScalable,
TRI))
return true;
assert(!OffsetSIsScalable && !OffsetDIsScalable &&
"Expected offsets to be byte offsets");
if (!BaseOpS->isIdenticalTo(*BaseOpD))
return true;

4
lib/CodeGen/MachineScheduler.cpp
View File

@ -1567,7 +1567,9 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
for (SUnit *SU : MemOps) {
SmallVector<const MachineOperand *, 4> BaseOps;
int64_t Offset;
if (TII->getMemOperandsWithOffset(*SU->getInstr(), BaseOps, Offset, TRI))
bool OffsetIsScalable;
if (TII->getMemOperandsWithOffset(*SU->getInstr(), BaseOps, Offset,
OffsetIsScalable, TRI))
MemOpRecords.push_back(MemOpInfo(SU, BaseOps, Offset));
#ifndef NDEBUG
for (auto *Op : BaseOps)

3
lib/CodeGen/MachineSink.cpp
View File

@ -764,7 +764,8 @@ static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI,
const MachineOperand *BaseOp;
int64_t Offset;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI))
bool OffsetIsScalable;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
return false;
if (!BaseOp->isReg())

7
lib/CodeGen/ModuloSchedule.cpp
View File

@ -913,7 +913,12 @@ bool ModuloScheduleExpander::computeDelta(MachineInstr &MI, unsigned &Delta) {
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
const MachineOperand *BaseOp;
int64_t Offset;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI))
bool OffsetIsScalable;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
return false;
// FIXME: This algorithm assumes instructions have fixed-size offsets.
if (OffsetIsScalable)
return false;
if (!BaseOp->isReg())

12
lib/CodeGen/TargetInstrInfo.cpp
View File

@ -1031,9 +1031,9 @@ CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
// Default implementation of getMemOperandWithOffset.
bool TargetInstrInfo::getMemOperandWithOffset(
const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
const TargetRegisterInfo *TRI) const {
bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
SmallVector<const MachineOperand *, 4> BaseOps;
if (!getMemOperandsWithOffset(MI, BaseOps, Offset, TRI) ||
if (!getMemOperandsWithOffset(MI, BaseOps, Offset, OffsetIsScalable, TRI) ||
BaseOps.size() != 1)
return false;
BaseOp = BaseOps.front();
@ -1137,6 +1137,7 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
DIExpression *Expr = DIExpression::get(MF->getFunction().getContext(), {});
int64_t Offset;
bool OffsetIsScalable;
// To simplify the sub-register handling, verify that we only need to
// consider physical registers.
@ -1175,7 +1176,12 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
return None;
const MachineOperand *BaseOp;
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI))
if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
TRI))
return None;
// FIXME: Scalable offsets are not yet handled in the offset code below.
if (OffsetIsScalable)
return None;
// TODO: Can currently only handle mem instructions with a single define.

145
lib/Target/AArch64/AArch64InstrInfo.cpp
View File

@ -942,6 +942,7 @@ bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
unsigned WidthA = 0, WidthB = 0;
bool OffsetAIsScalable = false, OffsetBIsScalable = false;
assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
@ -955,9 +956,14 @@ bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
// base are identical, and the offset of a lower memory access +
// the width doesn't overlap the offset of a higher memory access,
// then the memory accesses are different.
if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
// If OffsetAIsScalable and OffsetBIsScalable are both true, they
// are assumed to have the same scale (vscale).
if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
WidthA, TRI) &&
getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
WidthB, TRI)) {
if (BaseOpA->isIdenticalTo(*BaseOpB) &&
OffsetAIsScalable == OffsetBIsScalable) {
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
@ -1999,13 +2005,15 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
bool AArch64InstrInfo::getMemOperandsWithOffset(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const {
int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI)
const {
if (!LdSt.mayLoadOrStore())
return false;
const MachineOperand *BaseOp;
unsigned Width;
if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
Width, TRI))
return false;
BaseOps.push_back(BaseOp);
return true;
@ -2013,7 +2021,8 @@ bool AArch64InstrInfo::getMemOperandsWithOffset(
bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
unsigned &Width, const TargetRegisterInfo *TRI) const {
bool &OffsetIsScalable, unsigned &Width,
const TargetRegisterInfo *TRI) const {
assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
// Handle only loads/stores with base register followed by immediate offset.
if (LdSt.getNumExplicitOperands() == 3) {
@ -2032,7 +2041,7 @@ bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
// Get the scaling factor for the instruction and set the width for the
// instruction.
unsigned Scale = 0;
TypeSize Scale(0U, false);
int64_t Dummy1, Dummy2;
// If this returns false, then it's an instruction we don't want to handle.
@ -2044,12 +2053,13 @@ bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
// set to 1.
if (LdSt.getNumExplicitOperands() == 3) {
BaseOp = &LdSt.getOperand(1);
Offset = LdSt.getOperand(2).getImm() * Scale;
Offset = LdSt.getOperand(2).getImm() * Scale.getKnownMinSize();
} else {
assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
BaseOp = &LdSt.getOperand(2);
Offset = LdSt.getOperand(3).getImm() * Scale;
Offset = LdSt.getOperand(3).getImm() * Scale.getKnownMinSize();
}
OffsetIsScalable = Scale.isScalable();
if (!BaseOp->isReg() && !BaseOp->isFI())
return false;
@ -2065,27 +2075,28 @@ AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
return OfsOp;
}
bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
unsigned &Width, int64_t &MinOffset,
int64_t &MaxOffset) {
const unsigned SVEMaxBytesPerVector = AArch64::SVEMaxBitsPerVector / 8;
switch (Opcode) {
// Not a memory operation or something we want to handle.
default:
Scale = Width = 0;
Scale = TypeSize::Fixed(0);
Width = 0;
MinOffset = MaxOffset = 0;
return false;
case AArch64::STRWpost:
case AArch64::LDRWpost:
Width = 32;
Scale = 4;
Scale = TypeSize::Fixed(4);
MinOffset = -256;
MaxOffset = 255;
break;
case AArch64::LDURQi:
case AArch64::STURQi:
Width = 16;
Scale = 1;
Scale = TypeSize::Fixed(1);
MinOffset = -256;
MaxOffset = 255;
break;
@ -2095,7 +2106,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::STURXi:
case AArch64::STURDi:
Width = 8;
Scale = 1;
Scale = TypeSize::Fixed(1);
MinOffset = -256;
MaxOffset = 255;
break;
@ -2105,7 +2116,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::STURWi:
case AArch64::STURSi:
Width = 4;
Scale = 1;
Scale = TypeSize::Fixed(1);
MinOffset = -256;
MaxOffset = 255;
break;
@ -2116,7 +2127,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::STURHi:
case AArch64::STURHHi:
Width = 2;
Scale = 1;
Scale = TypeSize::Fixed(1);
MinOffset = -256;
MaxOffset = 255;
break;
@ -2127,7 +2138,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::STURBi:
case AArch64::STURBBi:
Width = 1;
Scale = 1;
Scale = TypeSize::Fixed(1);
MinOffset = -256;
MaxOffset = 255;
break;
@ -2135,14 +2146,15 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::LDNPQi:
case AArch64::STPQi:
case AArch64::STNPQi:
Scale = 16;
Scale = TypeSize::Fixed(16);
Width = 32;
MinOffset = -64;
MaxOffset = 63;
break;
case AArch64::LDRQui:
case AArch64::STRQui:
Scale = Width = 16;
Scale = TypeSize::Fixed(16);
Width = 16;
MinOffset = 0;
MaxOffset = 4095;
break;
@ -2154,7 +2166,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::STPDi:
case AArch64::STNPXi:
case AArch64::STNPDi:
Scale = 8;
Scale = TypeSize::Fixed(8);
Width = 16;
MinOffset = -64;
MaxOffset = 63;
@ -2164,7 +2176,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::LDRDui:
case AArch64::STRXui:
case AArch64::STRDui:
Scale = Width = 8;
Scale = TypeSize::Fixed(8);
Width = 8;
MinOffset = 0;
MaxOffset = 4095;
break;
@ -2176,7 +2189,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::STPSi:
case AArch64::STNPWi:
case AArch64::STNPSi:
Scale = 4;
Scale = TypeSize::Fixed(4);
Width = 8;
MinOffset = -64;
MaxOffset = 63;
@ -2186,7 +2199,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::LDRSWui:
case AArch64::STRWui:
case AArch64::STRSui:
Scale = Width = 4;
Scale = TypeSize::Fixed(4);
Width = 4;
MinOffset = 0;
MaxOffset = 4095;
break;
@ -2196,7 +2210,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::LDRSHXui:
case AArch64::STRHui:
case AArch64::STRHHui:
Scale = Width = 2;
Scale = TypeSize::Fixed(2);
Width = 2;
MinOffset = 0;
MaxOffset = 4095;
break;
@ -2206,18 +2221,19 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::LDRSBXui:
case AArch64::STRBui:
case AArch64::STRBBui:
Scale = Width = 1;
Scale = TypeSize::Fixed(1);
Width = 1;
MinOffset = 0;
MaxOffset = 4095;
break;
case AArch64::ADDG:
Scale = 16;
Scale = TypeSize::Fixed(16);
Width = 0;
MinOffset = 0;
MaxOffset = 63;
break;
case AArch64::TAGPstack:
Scale = 16;
Scale = TypeSize::Fixed(16);
Width = 0;
// TAGP with a negative offset turns into SUBP, which has a maximum offset
// of 63 (not 64!).
@ -2227,20 +2243,21 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::LDG:
case AArch64::STGOffset:
case AArch64::STZGOffset:
Scale = Width = 16;
Scale = TypeSize::Fixed(16);
Width = 16;
MinOffset = -256;
MaxOffset = 255;
break;
case AArch64::LDR_PXI:
case AArch64::STR_PXI:
Scale = 2;
Scale = TypeSize::Scalable(2);
Width = SVEMaxBytesPerVector / 8;
MinOffset = -256;
MaxOffset = 255;
break;
case AArch64::LDR_ZXI:
case AArch64::STR_ZXI:
Scale = 16;
Scale = TypeSize::Scalable(16);
Width = SVEMaxBytesPerVector;
MinOffset = -256;
MaxOffset = 255;
@ -2255,20 +2272,21 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, unsigned &Scale,
case AArch64::ST1D_IMM:
// A full vectors worth of data
// Width = mbytes * elements
Scale = 16;
Scale = TypeSize::Scalable(16);
Width = SVEMaxBytesPerVector;
MinOffset = -8;
MaxOffset = 7;
break;
case AArch64::ST2GOffset:
case AArch64::STZ2GOffset:
Scale = 16;
Scale = TypeSize::Fixed(16);
Width = 32;
MinOffset = -256;
MaxOffset = 255;
break;
case AArch64::STGPi:
Scale = Width = 16;
Scale = TypeSize::Fixed(16);
Width = 16;
MinOffset = -64;
MaxOffset = 63;
break;
@ -3457,26 +3475,6 @@ MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
return nullptr;
}
static bool isSVEScaledImmInstruction(unsigned Opcode) {
switch (Opcode) {
case AArch64::LDR_ZXI:
case AArch64::STR_ZXI:
case AArch64::LDR_PXI:
case AArch64::STR_PXI:
case AArch64::LD1B_IMM:
case AArch64::LD1H_IMM:
case AArch64::LD1W_IMM:
case AArch64::LD1D_IMM:
case AArch64::ST1B_IMM:
case AArch64::ST1H_IMM:
case AArch64::ST1W_IMM:
case AArch64::ST1D_IMM:
return true;
default:
return false;
}
}
int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
StackOffset &SOffset,
bool *OutUseUnscaledOp,
@ -3515,16 +3513,17 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
}
// Get the min/max offset and the scale.
unsigned Scale, Width;
TypeSize ScaleValue(0U, false);
unsigned Width;
int64_t MinOff, MaxOff;
if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), Scale, Width, MinOff,
if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), ScaleValue, Width, MinOff,
MaxOff))
llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
// Construct the complete offset.
bool IsMulVL = isSVEScaledImmInstruction(MI.getOpcode());
int64_t Offset =
IsMulVL ? (SOffset.getScalableBytes()) : (SOffset.getBytes());
bool IsMulVL = ScaleValue.isScalable();
unsigned Scale = ScaleValue.getKnownMinSize();
int64_t Offset = IsMulVL ? SOffset.getScalableBytes() : SOffset.getBytes();
const MachineOperand &ImmOpnd =
MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
@ -3537,9 +3536,14 @@ int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode());
bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
if (useUnscaledOp &&
!AArch64InstrInfo::getMemOpInfo(*UnscaledOp, Scale, Width, MinOff, MaxOff))
!AArch64InstrInfo::getMemOpInfo(*UnscaledOp, ScaleValue, Width, MinOff,
MaxOff))
llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
Scale = ScaleValue.getKnownMinSize();
assert(IsMulVL == ScaleValue.isScalable() &&
"Unscaled opcode has different value for scalable");
int64_t Remainder = Offset % Scale;
assert(!(Remainder && useUnscaledOp) &&
"Cannot have remainder when using unscaled op");
@ -5864,23 +5868,29 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
if (MI.mayLoadOrStore()) {
const MachineOperand *Base; // Filled with the base operand of MI.
int64_t Offset; // Filled with the offset of MI.
bool OffsetIsScalable;
// Does it allow us to offset the base operand and is the base the
// register SP?
if (!getMemOperandWithOffset(MI, Base, Offset, &TRI) || !Base->isReg() ||
Base->getReg() != AArch64::SP)
if (!getMemOperandWithOffset(MI, Base, Offset, OffsetIsScalable, &TRI) ||
!Base->isReg() || Base->getReg() != AArch64::SP)
return false;
// Fixe-up code below assumes bytes.
if (OffsetIsScalable)
return false;
// Find the minimum/maximum offset for this instruction and check
// if fixing it up would be in range.
int64_t MinOffset,
MaxOffset; // Unscaled offsets for the instruction.
unsigned Scale; // The scale to multiply the offsets by.
TypeSize Scale(0U, false); // The scale to multiply the offsets by.
unsigned DummyWidth;
getMemOpInfo(MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
Offset += 16; // Update the offset to what it would be if we outlined.
if (Offset < MinOffset * Scale || Offset > MaxOffset * Scale)
if (Offset < MinOffset * (int64_t)Scale.getFixedSize() ||
Offset > MaxOffset * (int64_t)Scale.getFixedSize())
return false;
// It's in range, so we can outline it.
@ -6258,26 +6268,29 @@ void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
const MachineOperand *Base;
unsigned Width;
int64_t Offset;
bool OffsetIsScalable;
// Is this a load or store with an immediate offset with SP as the base?
if (!MI.mayLoadOrStore() ||
!getMemOperandWithOffsetWidth(MI, Base, Offset, Width, &RI) ||
!getMemOperandWithOffsetWidth(MI, Base, Offset, OffsetIsScalable, Width,
&RI) ||
(Base->isReg() && Base->getReg() != AArch64::SP))
continue;
// It is, so we have to fix it up.
unsigned Scale;
TypeSize Scale(0U, false);
int64_t Dummy1, Dummy2;
MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(MI);
assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
getMemOpInfo(MI.getOpcode(), Scale, Width, Dummy1, Dummy2);
assert(Scale != 0 && "Unexpected opcode!");
assert(!OffsetIsScalable && "Expected offset to be a byte offset");
// We've pushed the return address to the stack, so add 16 to the offset.
// This is safe, since we already checked if it would overflow when we
// checked if this instruction was legal to outline.
int64_t NewImm = (Offset + 16) / Scale;
int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedSize();
StackOffsetOperand.setImm(NewImm);
}
}

13
lib/Target/AArch64/AArch64InstrInfo.h
View File

@ -19,6 +19,7 @@
#include "llvm/ADT/Optional.h"
#include "llvm/CodeGen/MachineCombinerPattern.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/Support/TypeSize.h"
#define GET_INSTRINFO_HEADER
#include "AArch64GenInstrInfo.inc"
@ -114,11 +115,17 @@ public:
bool getMemOperandsWithOffset(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const override;
int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI)
const override;
/// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.
/// This is true for some SVE instructions like ldr/str that have a
/// 'reg + imm' addressing mode where the immediate is an index to the
/// scalable vector located at 'reg + imm * vscale x #bytes'.
bool getMemOperandWithOffsetWidth(const MachineInstr &MI,
const MachineOperand *&BaseOp,
int64_t &Offset, unsigned &Width,
int64_t &Offset, bool &OffsetIsScalable,
unsigned &Width,
const TargetRegisterInfo *TRI) const;
/// Return the immediate offset of the base register in a load/store \p LdSt.
@ -128,7 +135,7 @@ public:
/// \p Scale, \p Width, \p MinOffset, and \p MaxOffset accordingly.
///
/// For unscaled instructions, \p Scale is set to 1.
static bool getMemOpInfo(unsigned Opcode, unsigned &Scale, unsigned &Width,
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, unsigned &Width,
int64_t &MinOffset, int64_t &MaxOffset);
bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,

4
lib/Target/AArch64/AArch64StorePairSuppress.cpp
View File

@ -149,7 +149,9 @@ bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
continue;
const MachineOperand *BaseOp;
int64_t Offset;
if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI) &&
bool OffsetIsScalable;
if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
TRI) &&
BaseOp->isReg()) {
Register BaseReg = BaseOp->getReg();
if (PrevBaseReg == BaseReg) {

9
lib/Target/AMDGPU/SIInstrInfo.cpp
View File

@ -260,11 +260,13 @@ static bool isStride64(unsigned Opc) {
bool SIInstrInfo::getMemOperandsWithOffset(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const {
int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI)
const {
if (!LdSt.mayLoadOrStore())
return false;
unsigned Opc = LdSt.getOpcode();
OffsetIsScalable = false;
const MachineOperand *BaseOp, *OffsetOp;
if (isDS(LdSt)) {
@ -2610,8 +2612,9 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
const MachineInstr &MIb) const {
SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
int64_t Offset0, Offset1;
if (!getMemOperandsWithOffset(MIa, BaseOps0, Offset0, &RI) ||
!getMemOperandsWithOffset(MIb, BaseOps1, Offset1, &RI))
bool Offset0IsScalable, Offset1IsScalable;
if (!getMemOperandsWithOffset(MIa, BaseOps0, Offset0, Offset0IsScalable, &RI) ||
!getMemOperandsWithOffset(MIb, BaseOps1, Offset1, Offset1IsScalable, &RI))
return false;
if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))

2
lib/Target/AMDGPU/SIInstrInfo.h
View File

@ -184,7 +184,7 @@ public:
bool
getMemOperandsWithOffset(const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset,
int64_t &Offset, bool &OffsetIsScalable,
const TargetRegisterInfo *TRI) const final;
bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,

3
lib/Target/AMDGPU/SIMachineScheduler.cpp
View File

@ -1952,8 +1952,9 @@ void SIScheduleDAGMI::schedule()
int64_t OffLatReg;
if (SITII->isLowLatencyInstruction(*SU->getInstr())) {
IsLowLatencySU[i] = 1;
bool OffsetIsScalable;
if (SITII->getMemOperandWithOffset(*SU->getInstr(), BaseLatOp, OffLatReg,
TRI))
OffsetIsScalable, TRI))
LowLatencyOffset[i] = OffLatReg;
} else if (SITII->isHighLatencyDef(SU->getInstr()->getOpcode()))
IsHighLatencySU[i] = 1;

3
lib/Target/Hexagon/HexagonInstrInfo.cpp
View File

@ -2948,8 +2948,9 @@ bool HexagonInstrInfo::addLatencyToSchedule(const MachineInstr &MI1,
/// Get the base register and byte offset of a load/store instr.
bool HexagonInstrInfo::getMemOperandsWithOffset(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const {
int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
unsigned AccessSize = 0;
OffsetIsScalable = false;
const MachineOperand *BaseOp = getBaseAndOffset(LdSt, Offset, AccessSize);
if (!BaseOp || !BaseOp->isReg())
return false;

2
lib/Target/Hexagon/HexagonInstrInfo.h
View File

@ -207,7 +207,7 @@ public:
bool
getMemOperandsWithOffset(const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset,
int64_t &Offset, bool &OffsetIsScalable,
const TargetRegisterInfo *TRI) const override;
/// Reverses the branch condition of the specified condition list,

3
lib/Target/Lanai/LanaiInstrInfo.cpp
View File

@ -797,7 +797,7 @@ bool LanaiInstrInfo::getMemOperandWithOffsetWidth(
bool LanaiInstrInfo::getMemOperandsWithOffset(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const {
int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
switch (LdSt.getOpcode()) {
default:
return false;
@ -812,6 +812,7 @@ bool LanaiInstrInfo::getMemOperandsWithOffset(
case Lanai::LDBz_RI:
const MachineOperand *BaseOp;
unsigned Width;
OffsetIsScalable = false;
if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
return false;
BaseOps.push_back(BaseOp);

2
lib/Target/Lanai/LanaiInstrInfo.h
View File

@ -70,7 +70,7 @@ public:
bool
getMemOperandsWithOffset(const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset,
int64_t &Offset, bool &OffsetIsScalable,
const TargetRegisterInfo *TRI) const override;
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,

3
lib/Target/X86/X86InstrInfo.cpp
View File

@ -3191,7 +3191,7 @@ static unsigned getLoadStoreRegOpcode(unsigned Reg,
bool X86InstrInfo::getMemOperandsWithOffset(
const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset, const TargetRegisterInfo *TRI) const {
int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const {
const MCInstrDesc &Desc = MemOp.getDesc();
int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
if (MemRefBegin < 0)
@ -3222,6 +3222,7 @@ bool X86InstrInfo::getMemOperandsWithOffset(
if (!BaseOp->isReg())
return false;
OffsetIsScalable = false;
BaseOps.push_back(BaseOp);
return true;
}

2
lib/Target/X86/X86InstrInfo.h
View File

@ -294,7 +294,7 @@ public:
bool
getMemOperandsWithOffset(const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps,
int64_t &Offset,
int64_t &Offset, bool &OffsetIsScalable,
const TargetRegisterInfo *TRI) const override;
bool analyzeBranchPredicate(MachineBasicBlock &MBB,
TargetInstrInfo::MachineBranchPredicate &MBP,