RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2025-01-31 20:51:52 +01:00
Code Issues Projects Releases Wiki Activity

[SelectionDAG] Fix the representation of ISD::STEP_VECTOR.

Browse Source 
The existing rule about the operand type is strange.  Instead, just say
the operand is a TargetConstant with the right width.  (Legalization
ignores TargetConstants, so it doesn't matter if that width is legal.)

Highlights:

1. I had to substantially rewrite the AArch64 isel patterns to expect a
TargetConstant.  Nothing too exotic, but maybe a little hairy. Maybe
worth considering a target-specific node with some dagcombines instead
of this complicated nest of isel patterns.
2. Our behavior on RV32 for vectors of i64 has changed slightly. In
particular, we correctly preserve the width of the arithmetic through
legalization.  This changes the DAG a bit. Maybe room for
improvement here.
3. I explicitly defined the behavior around overflow. This is necessary
to make the DAGCombine transforms legal, and I don't think it causes any
practical issues.

Differential Revision: https://reviews.llvm.org/D105673
This commit is contained in:
Eli Friedman 2021-07-08 16:14:33 -07:00
parent 5edc17d32b
commit 4384bac220
8 changed files with 294 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
include/llvm/CodeGen/ISDOpcodes.h
View File

@ -601,15 +601,12 @@ enum NodeType {
/// STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised
/// of a linear sequence of unsigned values starting from 0 with a step of
/// IMM, where IMM must be a vector index constant integer value which must
/// fit in the vector element type.
/// Note that IMM may be a smaller type than the vector element type, in
/// which case the step is implicitly sign-extended to the vector element
/// type. IMM may also be a larger type than the vector element type, in
/// which case the step is implicitly truncated to the vector element type.
/// IMM, where IMM must be a TargetConstant with type equal to the vector
/// element type. The arithmetic is performed modulo the bitwidth of the
/// element.
///
/// The operation does not support returning fixed-width vectors or
/// non-constant operands. If the sequence value exceeds the limit allowed
/// for the element type then the values for those lanes are undefined.
/// non-constant operands.
STEP_VECTOR,
/// MULHU/MULHS - Multiply high - Multiply two integers of type iN,

2
include/llvm/CodeGen/SelectionDAG.h
View File

@ -835,7 +835,7 @@ public:
/// Returns a vector of type ResVT whose elements contain the linear sequence
/// <0, Step, Step * 2, Step * 3, ...>
SDValue getStepVector(const SDLoc &DL, EVT ResVT, SDValue Step);
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal);
/// Returns a vector of type ResVT whose elements contain the linear sequence
/// <0, 1, 2, 3, ...>

24
lib/CodeGen/SelectionDAG/DAGCombiner.cpp
View File

@ -2526,8 +2526,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
N1.getOpcode() == ISD::STEP_VECTOR) {
const APInt &C0 = N0->getConstantOperandAPInt(0);
const APInt &C1 = N1->getConstantOperandAPInt(0);
EVT SVT = N0.getOperand(0).getValueType();
SDValue NewStep = DAG.getConstant(C0 + C1, DL, SVT);
APInt NewStep = C0 + C1;
return DAG.getStepVector(DL, VT, NewStep);
}
@ -2537,11 +2536,7 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
(N1.getOpcode() == ISD::STEP_VECTOR)) {
const APInt &SV0 = N0.getOperand(1)->getConstantOperandAPInt(0);
const APInt &SV1 = N1->getConstantOperandAPInt(0);
EVT SVT = N1.getOperand(0).getValueType();
assert(N1.getOperand(0).getValueType() ==
N0.getOperand(1)->getOperand(0).getValueType() &&
"Different operand types of STEP_VECTOR.");
SDValue NewStep = DAG.getConstant(SV0 + SV1, DL, SVT);
APInt NewStep = SV0 + SV1;
SDValue SV = DAG.getStepVector(DL, VT, NewStep);
return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV);
}
@ -3576,8 +3571,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
// canonicalize (sub X, step_vector(C)) to (add X, step_vector(-C))
if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) {
SDValue NewStep = DAG.getConstant(-N1.getConstantOperandAPInt(0), DL,
N1.getOperand(0).getValueType());
APInt NewStep = -N1.getConstantOperandAPInt(0);
return DAG.getNode(ISD::ADD, DL, VT, N0,
DAG.getStepVector(DL, VT, NewStep));
}
@ -3961,9 +3955,7 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
if (N0.getOpcode() == ISD::STEP_VECTOR)
if (ISD::isConstantSplatVector(N1.getNode(), MulVal)) {
const APInt &C0 = N0.getConstantOperandAPInt(0);
EVT SVT = N0.getOperand(0).getValueType();
SDValue NewStep = DAG.getConstant(
C0 * MulVal.sextOrTrunc(SVT.getSizeInBits()), SDLoc(N), SVT);
APInt NewStep = C0 * MulVal;
return DAG.getStepVector(SDLoc(N), VT, NewStep);
}
@ -8476,10 +8468,10 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
if (N0.getOpcode() == ISD::STEP_VECTOR)
if (ISD::isConstantSplatVector(N1.getNode(), ShlVal)) {
const APInt &C0 = N0.getConstantOperandAPInt(0);
EVT SVT = N0.getOperand(0).getValueType();
SDValue NewStep = DAG.getConstant(
C0 << ShlVal.sextOrTrunc(SVT.getSizeInBits()), SDLoc(N), SVT);
return DAG.getStepVector(SDLoc(N), VT, NewStep);
if (ShlVal.ult(C0.getBitWidth())) {
APInt NewStep = C0 << ShlVal;
return DAG.getStepVector(SDLoc(N), VT, NewStep);
}
}
return SDValue();

6
lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
View File

@ -4861,11 +4861,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "Type must be promoted to a vector type");
EVT NOutElemVT = TLI.getTypeToTransformTo(*DAG.getContext(),
NOutVT.getVectorElementType());
APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
SDValue Step = DAG.getConstant(StepVal.getSExtValue(), dl, NOutElemVT);
return DAG.getStepVector(dl, NOutVT, Step);
return DAG.getStepVector(dl, NOutVT,
StepVal.sext(NOutVT.getScalarSizeInBits()));
}
SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {

27
lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View File

@ -1748,19 +1748,21 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
}
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
EVT OpVT = TLI->getTypeToTransformTo(*getContext(), ResVT.getScalarType());
return getStepVector(DL, ResVT, getConstant(1, DL, OpVT));
APInt One(ResVT.getScalarSizeInBits(), 1);
return getStepVector(DL, ResVT, One);
}
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, SDValue Step) {
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) {
assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
if (ResVT.isScalableVector())
return getNode(ISD::STEP_VECTOR, DL, ResVT, Step);
return getNode(
ISD::STEP_VECTOR, DL, ResVT,
getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
EVT OpVT = Step.getValueType();
APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
SmallVector<SDValue, 16> OpsStepConstants;
for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
OpsStepConstants.push_back(getConstant(StepVal * i, DL, OpVT));
OpsStepConstants.push_back(
getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
return getBuildVector(ResVT, DL, OpsStepConstants);
}
@ -4780,14 +4782,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
case ISD::STEP_VECTOR:
assert(VT.isScalableVector() &&
"STEP_VECTOR can only be used with scalable types");
assert(VT.getScalarSizeInBits() >= 8 &&
"STEP_VECTOR can only be used with vectors of integers that are at "
"least 8 bits wide");
assert(isa<ConstantSDNode>(Operand) &&
cast<ConstantSDNode>(Operand)->getAPIntValue().isSignedIntN(
VT.getScalarSizeInBits()) &&
"Expected STEP_VECTOR integer constant to fit in "
"the vector element type");
assert(OpOpcode == ISD::TargetConstant &&
VT.getVectorElementType() == Operand.getValueType() &&
"Unexpected step operand");
break;
case ISD::FREEZE:
assert(VT == Operand.getValueType() && "Unexpected VT!");

116
lib/Target/AArch64/SVEInstrFormats.td
View File

@ -4835,6 +4835,14 @@ multiclass sve_fp_2op_p_pd<bits<3> opc, string asm,
//SVE Index Generation Group
//===----------------------------------------------------------------------===//
def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>;
def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>;
def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>;
def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>;
def i64imm_32bit_tgt : TImmLeaf<i64, [{
return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm);
}]>;
class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty,
Operand imm_ty>
: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b),
@ -4858,23 +4866,23 @@ multiclass sve_int_index_ii<string asm, SDPatternOperator step_vector, SDPattern
def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>;
def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>;
def : Pat<(nxv16i8 (step_vector simm5_8b:$imm5b)),
(!cast<Instruction>(NAME # "_B") (i32 0), simm5_8b:$imm5b)>;
def : Pat<(nxv8i16 (step_vector simm5_16b:$imm5b)),
(!cast<Instruction>(NAME # "_H") (i32 0), simm5_16b:$imm5b)>;
def : Pat<(nxv4i32 (step_vector simm5_32b:$imm5b)),
def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)),
(!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;
def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)),
(!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;
def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)),
(!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>;
def : Pat<(nxv2i64 (step_vector simm5_64b:$imm5b)),
def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)),
(!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>;
// add(step_vector(step), dup(X)) -> index(X, step).
def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b:$imm5b)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))),
(!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, simm5_8b:$imm5b)>;
def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b:$imm5b)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))),
(!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, simm5_16b:$imm5b)>;
def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b:$imm5b)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))),
def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))),
(!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;
def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))),
(!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>;
def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))),
(!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>;
def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b:$imm5b)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))),
def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))),
(!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>;
}
@ -4901,33 +4909,33 @@ multiclass sve_int_index_ir<string asm, SDPatternOperator step_vector, SDPattern
def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>;
def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>;
def : Pat<(nxv16i8 (step_vector (i32 imm:$imm))),
(!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(nxv8i16 (step_vector (i32 imm:$imm))),
(!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(nxv4i32 (step_vector (i32 imm:$imm))),
(!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(nxv2i64 (step_vector (i64 imm:$imm))),
(!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") imm:$imm))>;
def : Pat<(nxv2i64 (step_vector (i64 !cast<ImmLeaf>("i64imm_32bit"):$imm))),
(!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") imm:$imm)), sub_32))>;
def : Pat<(nxv16i8 (step_vector i8:$imm)),
(!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;
def : Pat<(nxv8i16 (step_vector i16:$imm)),
(!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;
def : Pat<(nxv4i32 (step_vector i32:$imm)),
(!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>;
def : Pat<(nxv2i64 (step_vector i64:$imm)),
(!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>;
def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)),
(!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>;
// add(step_vector(step), dup(X)) -> index(X, step).
def : Pat<(add (nxv16i8 (step_vector_oneuse (i32 imm:$imm))), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))),
(!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(add (nxv8i16 (step_vector_oneuse (i32 imm:$imm))), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))),
(!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(add (nxv4i32 (step_vector_oneuse (i32 imm:$imm))), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))),
(!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse (i64 imm:$imm))), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))),
(!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") imm:$imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse (i64 !cast<ImmLeaf>("i64imm_32bit"):$imm))), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))),
(!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") imm:$imm)), sub_32))>;
def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))),
(!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;
def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))),
(!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;
def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))),
(!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))),
(!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))),
(!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>;
// mul(step_vector(1), dup(Y)) -> index(0, Y).
def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i32 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))),
def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>;
def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i32 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),
def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>;
def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>;
@ -4935,9 +4943,9 @@ multiclass sve_int_index_ir<string asm, SDPatternOperator step_vector, SDPattern
(!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>;
// add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y).
def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i32 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))),
def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))),
(!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>;
def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i32 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))),
def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))),
(!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>;
def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))),
(!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>;
@ -4969,13 +4977,13 @@ multiclass sve_int_index_ri<string asm, SDPatternOperator step_vector, SDPattern
def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>;
// add(step_vector(step), dup(X)) -> index(X, step).
def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b:$imm5)), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_B") GPR32:$Rm, simm5_8b:$imm5)>;
def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b:$imm5)), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_H") GPR32:$Rm, simm5_16b:$imm5)>;
def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b:$imm5)), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))),
def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>;
def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>;
def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))),
(!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>;
def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b:$imm5)), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))),
def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))),
(!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>;
}
@ -5003,21 +5011,21 @@ multiclass sve_int_index_rr<string asm, SDPatternOperator step_vector, SDPattern
def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>;
// add(step_vector(step), dup(X)) -> index(X, step).
def : Pat<(add (nxv16i8 (step_vector_oneuse (i32 imm:$imm))), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(add (nxv8i16 (step_vector_oneuse (i32 imm:$imm))), (nxv8i16 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(add (nxv4i32 (step_vector_oneuse (i32 imm:$imm))), (nxv4i32 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") imm:$imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse (i64 imm:$imm))), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))),
(!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") imm:$imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse (i64 !cast<ImmLeaf>("i64imm_32bit"):$imm))), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))),
(!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") imm:$imm)), sub_32))>;
def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;
def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>;
def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))),
(!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>;
def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))),
(!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>;
// add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y).
def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i32 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))),
def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>;
def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i32 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),(nxv8i16 (AArch64dup(i32 GPR32:$Rn)))),
def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),(nxv8i16 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>;
def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))),(nxv4i32 (AArch64dup(i32 GPR32:$Rn)))),
(!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>;

7
lib/Target/RISCV/RISCVISelLowering.cpp
View File

@ -4070,16 +4070,15 @@ SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
uint64_t StepValImm = Op.getConstantOperandVal(0);
if (StepValImm != 1) {
assert(Op.getOperand(0).getValueType() == XLenVT &&
"Unexpected step value type");
if (isPowerOf2_64(StepValImm)) {
SDValue StepVal =
DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
} else {
SDValue StepVal =
DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Op.getOperand(0));
SDValue StepVal = lowerScalarSplat(
DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
DL, DAG, Subtarget);
StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
}
}

239
test/CodeGen/RISCV/rvv/stepvector.ll
View File

@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s
; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s
; RUN: llc -mtriple=riscv32 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV32
; RUN: llc -mtriple=riscv64 -mattr=+m,+experimental-v -verify-machineinstrs < %s | FileCheck %s -check-prefixes=CHECK,RV64
declare <vscale x 1 x i8> @llvm.experimental.stepvector.nxv1i8()
@ -431,6 +431,43 @@ entry:
ret <vscale x 8 x i64> %3
}
define <vscale x 8 x i64> @mul_bigimm_stepvector_nxv8i64() {
; RV32-LABEL: mul_bigimm_stepvector_nxv8i64:
; RV32: # %bb.0: # %entry
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: addi a0, zero, 7
; RV32-NEXT: sw a0, 12(sp)
; RV32-NEXT: lui a0, 797989
; RV32-NEXT: addi a0, a0, -683
; RV32-NEXT: sw a0, 8(sp)
; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v8, (a0), zero
; RV32-NEXT: vid.v v16
; RV32-NEXT: vmul.vv v8, v16, v8
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
; RV64-LABEL: mul_bigimm_stepvector_nxv8i64:
; RV64: # %bb.0: # %entry
; RV64-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV64-NEXT: vid.v v8
; RV64-NEXT: lui a0, 1987
; RV64-NEXT: addiw a0, a0, -731
; RV64-NEXT: slli a0, a0, 12
; RV64-NEXT: addi a0, a0, -683
; RV64-NEXT: vmul.vx v8, v8, a0
; RV64-NEXT: ret
entry:
%0 = insertelement <vscale x 8 x i64> poison, i64 33333333333, i32 0
%1 = shufflevector <vscale x 8 x i64> %0, <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
%2 = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
%3 = mul <vscale x 8 x i64> %2, %1
ret <vscale x 8 x i64> %3
}
define <vscale x 8 x i64> @shl_stepvector_nxv8i64() {
; CHECK-LABEL: shl_stepvector_nxv8i64:
; CHECK: # %bb.0: # %entry
@ -449,27 +486,59 @@ entry:
declare <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
define <vscale x 16 x i64> @stepvector_nxv16i64() {
; CHECK-LABEL: stepvector_nxv16i64:
; CHECK: # %bb.0:
; CHECK-NEXT: csrr a0, vlenb
; CHECK-NEXT: vsetvli a1, zero, e64, m8, ta, mu
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: vadd.vx v16, v8, a0
; CHECK-NEXT: ret
; RV32-LABEL: stepvector_nxv16i64:
; RV32: # %bb.0:
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw zero, 12(sp)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: sw a0, 8(sp)
; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v16, (a0), zero
; RV32-NEXT: vid.v v8
; RV32-NEXT: vadd.vv v16, v8, v16
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
; RV64-LABEL: stepvector_nxv16i64:
; RV64: # %bb.0:
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, mu
; RV64-NEXT: vid.v v8
; RV64-NEXT: vadd.vx v16, v8, a0
; RV64-NEXT: ret
%v = call <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
ret <vscale x 16 x i64> %v
}
define <vscale x 16 x i64> @add_stepvector_nxv16i64() {
; CHECK-LABEL: add_stepvector_nxv16i64:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: csrr a0, vlenb
; CHECK-NEXT: slli a0, a0, 1
; CHECK-NEXT: vsetvli a1, zero, e64, m8, ta, mu
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: vsll.vi v8, v8, 1
; CHECK-NEXT: vadd.vx v16, v8, a0
; CHECK-NEXT: ret
; RV32-LABEL: add_stepvector_nxv16i64:
; RV32: # %bb.0: # %entry
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw zero, 12(sp)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: slli a0, a0, 1
; RV32-NEXT: sw a0, 8(sp)
; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v16, (a0), zero
; RV32-NEXT: vid.v v8
; RV32-NEXT: vsll.vi v8, v8, 1
; RV32-NEXT: vadd.vv v16, v8, v16
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
; RV64-LABEL: add_stepvector_nxv16i64:
; RV64: # %bb.0: # %entry
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: slli a0, a0, 1
; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, mu
; RV64-NEXT: vid.v v8
; RV64-NEXT: vsll.vi v8, v8, 1
; RV64-NEXT: vadd.vx v16, v8, a0
; RV64-NEXT: ret
entry:
%0 = call <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
%1 = call <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
@ -478,17 +547,39 @@ entry:
}
define <vscale x 16 x i64> @mul_stepvector_nxv16i64() {
; CHECK-LABEL: mul_stepvector_nxv16i64:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: addi a0, zero, 3
; CHECK-NEXT: vmul.vx v8, v8, a0
; CHECK-NEXT: csrr a0, vlenb
; CHECK-NEXT: slli a1, a0, 1
; CHECK-NEXT: add a0, a1, a0
; CHECK-NEXT: vadd.vx v16, v8, a0
; CHECK-NEXT: ret
; RV32-LABEL: mul_stepvector_nxv16i64:
; RV32: # %bb.0: # %entry
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: srli a1, a0, 3
; RV32-NEXT: addi a2, zero, 24
; RV32-NEXT: mulhu a1, a1, a2
; RV32-NEXT: sw a1, 12(sp)
; RV32-NEXT: slli a1, a0, 1
; RV32-NEXT: add a0, a1, a0
; RV32-NEXT: sw a0, 8(sp)
; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v16, (a0), zero
; RV32-NEXT: vid.v v8
; RV32-NEXT: addi a0, zero, 3
; RV32-NEXT: vmul.vx v8, v8, a0
; RV32-NEXT: vadd.vv v16, v8, v16
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
; RV64-LABEL: mul_stepvector_nxv16i64:
; RV64: # %bb.0: # %entry
; RV64-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV64-NEXT: vid.v v8
; RV64-NEXT: addi a0, zero, 3
; RV64-NEXT: vmul.vx v8, v8, a0
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: slli a1, a0, 1
; RV64-NEXT: add a0, a1, a0
; RV64-NEXT: vadd.vx v16, v8, a0
; RV64-NEXT: ret
entry:
%0 = insertelement <vscale x 16 x i64> poison, i64 3, i32 0
%1 = shufflevector <vscale x 16 x i64> %0, <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
@ -497,16 +588,88 @@ entry:
ret <vscale x 16 x i64> %3
}
define <vscale x 16 x i64> @mul_bigimm_stepvector_nxv16i64() {
; RV32-LABEL: mul_bigimm_stepvector_nxv16i64:
; RV32: # %bb.0: # %entry
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: addi a0, zero, 7
; RV32-NEXT: sw a0, 12(sp)
; RV32-NEXT: lui a0, 797989
; RV32-NEXT: addi a0, a0, -683
; RV32-NEXT: sw a0, 8(sp)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: lui a1, 11557
; RV32-NEXT: addi a1, a1, -683
; RV32-NEXT: mul a1, a0, a1
; RV32-NEXT: sw a1, 8(sp)
; RV32-NEXT: srli a0, a0, 3
; RV32-NEXT: addi a1, zero, 62
; RV32-NEXT: mul a1, a0, a1
; RV32-NEXT: lui a2, 92455
; RV32-NEXT: addi a2, a2, -1368
; RV32-NEXT: mulhu a0, a0, a2
; RV32-NEXT: add a0, a0, a1
; RV32-NEXT: sw a0, 12(sp)
; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v8, (a0), zero
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v16, (a0), zero
; RV32-NEXT: vid.v v24
; RV32-NEXT: vmul.vv v8, v24, v8
; RV32-NEXT: vadd.vv v16, v8, v16
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
; RV64-LABEL: mul_bigimm_stepvector_nxv16i64:
; RV64: # %bb.0: # %entry
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: lui a1, 1987
; RV64-NEXT: addiw a1, a1, -731
; RV64-NEXT: slli a1, a1, 12
; RV64-NEXT: addi a1, a1, -683
; RV64-NEXT: mul a0, a0, a1
; RV64-NEXT: vsetvli a2, zero, e64, m8, ta, mu
; RV64-NEXT: vid.v v8
; RV64-NEXT: vmul.vx v8, v8, a1
; RV64-NEXT: vadd.vx v16, v8, a0
; RV64-NEXT: ret
entry:
%0 = insertelement <vscale x 16 x i64> poison, i64 33333333333, i32 0
%1 = shufflevector <vscale x 16 x i64> %0, <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
%2 = call <vscale x 16 x i64> @llvm.experimental.stepvector.nxv16i64()
%3 = mul <vscale x 16 x i64> %2, %1
ret <vscale x 16 x i64> %3
}
define <vscale x 16 x i64> @shl_stepvector_nxv16i64() {
; CHECK-LABEL: shl_stepvector_nxv16i64:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: csrr a0, vlenb
; CHECK-NEXT: slli a0, a0, 2
; CHECK-NEXT: vsetvli a1, zero, e64, m8, ta, mu
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: vsll.vi v8, v8, 2
; CHECK-NEXT: vadd.vx v16, v8, a0
; CHECK-NEXT: ret
; RV32-LABEL: shl_stepvector_nxv16i64:
; RV32: # %bb.0: # %entry
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw zero, 12(sp)
; RV32-NEXT: csrr a0, vlenb
; RV32-NEXT: slli a0, a0, 2
; RV32-NEXT: sw a0, 8(sp)
; RV32-NEXT: vsetvli a0, zero, e64, m8, ta, mu
; RV32-NEXT: addi a0, sp, 8
; RV32-NEXT: vlse64.v v16, (a0), zero
; RV32-NEXT: vid.v v8
; RV32-NEXT: vsll.vi v8, v8, 2
; RV32-NEXT: vadd.vv v16, v8, v16
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
; RV64-LABEL: shl_stepvector_nxv16i64:
; RV64: # %bb.0: # %entry
; RV64-NEXT: csrr a0, vlenb
; RV64-NEXT: slli a0, a0, 2
; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, mu
; RV64-NEXT: vid.v v8
; RV64-NEXT: vsll.vi v8, v8, 2
; RV64-NEXT: vadd.vx v16, v8, a0
; RV64-NEXT: ret
entry:
%0 = insertelement <vscale x 16 x i64> poison, i64 2, i32 0
%1 = shufflevector <vscale x 16 x i64> %0, <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer