mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-10-20 19:42:54 +02:00
b91f66143d
This patch assumes that the dependents to be scanned for the ExitSU are its predecessors; otherwise, the successors of the instr are scanned. Furthermore, sometimes the ExitSU was being fused twice, since it may be fused once when scanning the successors from the beginning of the BB and then again when scanning the predecessors of ExitSU. Thus, when scanning the successors of an instr, skip the ExitSU. llvm-svn: 299974
273 lines
9.1 KiB
C++
273 lines
9.1 KiB
C++
//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
|
|
//
|
|
// The LLVM Compiler Infrastructure
|
|
//
|
|
// This file is distributed under the University of Illinois Open Source
|
|
// License. See LICENSE.TXT for details.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// \file This file contains the AArch64 implementation of the DAG scheduling mutation
|
|
// to pair instructions back to back.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "AArch64MacroFusion.h"
|
|
#include "AArch64Subtarget.h"
|
|
#include "llvm/ADT/Statistic.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
#include "llvm/Target/TargetInstrInfo.h"
|
|
|
|
#define DEBUG_TYPE "misched"
|
|
|
|
STATISTIC(NumFused, "Number of instr pairs fused");
|
|
|
|
using namespace llvm;
|
|
|
|
static cl::opt<bool> EnableMacroFusion("aarch64-misched-fusion", cl::Hidden,
|
|
cl::desc("Enable scheduling for macro fusion."), cl::init(true));
|
|
|
|
namespace {
|
|
|
|
/// \brief Verify that the instr pair, FirstMI and SecondMI, should be fused
|
|
/// together. Given an anchor instr, when the other instr is unspecified, then
|
|
/// check if the anchor instr may be part of a fused pair at all.
|
|
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
|
|
const TargetSubtargetInfo &TSI,
|
|
const MachineInstr *FirstMI,
|
|
const MachineInstr *SecondMI) {
|
|
assert((FirstMI || SecondMI) && "At least one instr must be specified");
|
|
|
|
const AArch64InstrInfo &II = static_cast<const AArch64InstrInfo&>(TII);
|
|
const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
|
|
|
|
// Assume wildcards for unspecified instrs.
|
|
unsigned FirstOpcode =
|
|
FirstMI ? FirstMI->getOpcode()
|
|
: static_cast<unsigned>(AArch64::INSTRUCTION_LIST_END);
|
|
unsigned SecondOpcode =
|
|
SecondMI ? SecondMI->getOpcode()
|
|
: static_cast<unsigned>(AArch64::INSTRUCTION_LIST_END);
|
|
|
|
if (ST.hasArithmeticBccFusion())
|
|
// Fuse CMN, CMP, TST followed by Bcc.
|
|
if (SecondOpcode == AArch64::Bcc)
|
|
switch (FirstOpcode) {
|
|
default:
|
|
return false;
|
|
case AArch64::ADDSWri:
|
|
case AArch64::ADDSWrr:
|
|
case AArch64::ADDSXri:
|
|
case AArch64::ADDSXrr:
|
|
case AArch64::ANDSWri:
|
|
case AArch64::ANDSWrr:
|
|
case AArch64::ANDSXri:
|
|
case AArch64::ANDSXrr:
|
|
case AArch64::SUBSWri:
|
|
case AArch64::SUBSWrr:
|
|
case AArch64::SUBSXri:
|
|
case AArch64::SUBSXrr:
|
|
case AArch64::BICSWrr:
|
|
case AArch64::BICSXrr:
|
|
return true;
|
|
case AArch64::ADDSWrs:
|
|
case AArch64::ADDSXrs:
|
|
case AArch64::ANDSWrs:
|
|
case AArch64::ANDSXrs:
|
|
case AArch64::SUBSWrs:
|
|
case AArch64::SUBSXrs:
|
|
case AArch64::BICSWrs:
|
|
case AArch64::BICSXrs:
|
|
// Shift value can be 0 making these behave like the "rr" variant...
|
|
return !II.hasShiftedReg(*FirstMI);
|
|
case AArch64::INSTRUCTION_LIST_END:
|
|
return true;
|
|
}
|
|
|
|
if (ST.hasArithmeticCbzFusion())
|
|
// Fuse ALU operations followed by CBZ/CBNZ.
|
|
if (SecondOpcode == AArch64::CBNZW || SecondOpcode == AArch64::CBNZX ||
|
|
SecondOpcode == AArch64::CBZW || SecondOpcode == AArch64::CBZX)
|
|
switch (FirstOpcode) {
|
|
default:
|
|
return false;
|
|
case AArch64::ADDWri:
|
|
case AArch64::ADDWrr:
|
|
case AArch64::ADDXri:
|
|
case AArch64::ADDXrr:
|
|
case AArch64::ANDWri:
|
|
case AArch64::ANDWrr:
|
|
case AArch64::ANDXri:
|
|
case AArch64::ANDXrr:
|
|
case AArch64::EORWri:
|
|
case AArch64::EORWrr:
|
|
case AArch64::EORXri:
|
|
case AArch64::EORXrr:
|
|
case AArch64::ORRWri:
|
|
case AArch64::ORRWrr:
|
|
case AArch64::ORRXri:
|
|
case AArch64::ORRXrr:
|
|
case AArch64::SUBWri:
|
|
case AArch64::SUBWrr:
|
|
case AArch64::SUBXri:
|
|
case AArch64::SUBXrr:
|
|
return true;
|
|
case AArch64::ADDWrs:
|
|
case AArch64::ADDXrs:
|
|
case AArch64::ANDWrs:
|
|
case AArch64::ANDXrs:
|
|
case AArch64::SUBWrs:
|
|
case AArch64::SUBXrs:
|
|
case AArch64::BICWrs:
|
|
case AArch64::BICXrs:
|
|
// Shift value can be 0 making these behave like the "rr" variant...
|
|
return !II.hasShiftedReg(*FirstMI);
|
|
case AArch64::INSTRUCTION_LIST_END:
|
|
return true;
|
|
}
|
|
|
|
if (ST.hasFuseAES())
|
|
// Fuse AES crypto operations.
|
|
switch(FirstOpcode) {
|
|
// AES encode.
|
|
case AArch64::AESErr:
|
|
return SecondOpcode == AArch64::AESMCrr ||
|
|
SecondOpcode == AArch64::INSTRUCTION_LIST_END;
|
|
// AES decode.
|
|
case AArch64::AESDrr:
|
|
return SecondOpcode == AArch64::AESIMCrr ||
|
|
SecondOpcode == AArch64::INSTRUCTION_LIST_END;
|
|
}
|
|
|
|
if (ST.hasFuseLiterals())
|
|
// Fuse literal generation operations.
|
|
switch (FirstOpcode) {
|
|
// PC relative address.
|
|
case AArch64::ADRP:
|
|
return SecondOpcode == AArch64::ADDXri ||
|
|
SecondOpcode == AArch64::INSTRUCTION_LIST_END;
|
|
// 32 bit immediate.
|
|
case AArch64::MOVZWi:
|
|
return (SecondOpcode == AArch64::MOVKWi &&
|
|
SecondMI->getOperand(3).getImm() == 16) ||
|
|
SecondOpcode == AArch64::INSTRUCTION_LIST_END;
|
|
// Lower half of 64 bit immediate.
|
|
case AArch64::MOVZXi:
|
|
return (SecondOpcode == AArch64::MOVKXi &&
|
|
SecondMI->getOperand(3).getImm() == 16) ||
|
|
SecondOpcode == AArch64::INSTRUCTION_LIST_END;
|
|
// Upper half of 64 bit immediate.
|
|
case AArch64::MOVKXi:
|
|
return FirstMI->getOperand(3).getImm() == 32 &&
|
|
((SecondOpcode == AArch64::MOVKXi &&
|
|
SecondMI->getOperand(3).getImm() == 48) ||
|
|
SecondOpcode == AArch64::INSTRUCTION_LIST_END);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// \brief Implement the fusion of instr pairs in the scheduling DAG,
|
|
/// anchored at the instr in AnchorSU..
|
|
static bool scheduleAdjacentImpl(ScheduleDAGMI *DAG, SUnit &AnchorSU) {
|
|
const MachineInstr *AnchorMI = AnchorSU.getInstr();
|
|
if (!AnchorMI || AnchorMI->isPseudo() || AnchorMI->isTransient())
|
|
return false;
|
|
|
|
// If the anchor instr is the ExitSU, then consider its predecessors;
|
|
// otherwise, its successors.
|
|
bool Preds = (&AnchorSU == &DAG->ExitSU);
|
|
SmallVectorImpl<SDep> &AnchorDeps = Preds ? AnchorSU.Preds : AnchorSU.Succs;
|
|
|
|
const MachineInstr *FirstMI = Preds ? nullptr : AnchorMI;
|
|
const MachineInstr *SecondMI = Preds ? AnchorMI : nullptr;
|
|
|
|
// Check if the anchor instr may be fused.
|
|
if (!shouldScheduleAdjacent(*DAG->TII, DAG->MF.getSubtarget(),
|
|
FirstMI, SecondMI))
|
|
return false;
|
|
|
|
// Explorer for fusion candidates among the dependencies of the anchor instr.
|
|
for (SDep &Dep : AnchorDeps) {
|
|
// Ignore dependencies that don't enforce ordering.
|
|
if (Dep.isWeak())
|
|
continue;
|
|
|
|
SUnit &DepSU = *Dep.getSUnit();
|
|
// Ignore the ExitSU if the dependents are successors.
|
|
if (!Preds && &DepSU == &DAG->ExitSU)
|
|
continue;
|
|
|
|
const MachineInstr *DepMI = DepSU.getInstr();
|
|
if (!DepMI || DepMI->isPseudo() || DepMI->isTransient())
|
|
continue;
|
|
|
|
FirstMI = Preds ? DepMI : AnchorMI;
|
|
SecondMI = Preds ? AnchorMI : DepMI;
|
|
if (!shouldScheduleAdjacent(*DAG->TII, DAG->MF.getSubtarget(),
|
|
FirstMI, SecondMI))
|
|
continue;
|
|
|
|
// Create a single weak edge between the adjacent instrs. The only effect is
|
|
// to cause bottom-up scheduling to heavily prioritize the clustered instrs.
|
|
SUnit &FirstSU = Preds ? DepSU : AnchorSU;
|
|
SUnit &SecondSU = Preds ? AnchorSU : DepSU;
|
|
DAG->addEdge(&SecondSU, SDep(&FirstSU, SDep::Cluster));
|
|
|
|
// Adjust the latency between the anchor instr and its
|
|
// predecessors/successors.
|
|
for (SDep &IDep : AnchorDeps)
|
|
if (IDep.getSUnit() == &DepSU)
|
|
IDep.setLatency(0);
|
|
|
|
// Adjust the latency between the dependent instr and its
|
|
// successors/predecessors.
|
|
for (SDep &IDep : Preds ? DepSU.Succs : DepSU.Preds)
|
|
if (IDep.getSUnit() == &AnchorSU)
|
|
IDep.setLatency(0);
|
|
|
|
DEBUG(dbgs() << DAG->MF.getName() << "(): Macro fuse ";
|
|
FirstSU.print(dbgs(), DAG); dbgs() << " - ";
|
|
SecondSU.print(dbgs(), DAG); dbgs() << " / ";
|
|
dbgs() << DAG->TII->getName(FirstMI->getOpcode()) << " - " <<
|
|
DAG->TII->getName(SecondMI->getOpcode()) << '\n'; );
|
|
|
|
++NumFused;
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/// \brief Post-process the DAG to create cluster edges between instrs that may
|
|
/// be fused by the processor into a single operation.
|
|
class AArch64MacroFusion : public ScheduleDAGMutation {
|
|
public:
|
|
AArch64MacroFusion() {}
|
|
|
|
void apply(ScheduleDAGInstrs *DAGInstrs) override;
|
|
};
|
|
|
|
void AArch64MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) {
|
|
ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
|
|
|
|
// For each of the SUnits in the scheduling block, try to fuse the instr in it
|
|
// with one in its successors.
|
|
for (SUnit &ISU : DAG->SUnits)
|
|
scheduleAdjacentImpl(DAG, ISU);
|
|
|
|
// Try to fuse the instr in the ExitSU with one in its predecessors.
|
|
scheduleAdjacentImpl(DAG, DAG->ExitSU);
|
|
}
|
|
|
|
} // end namespace
|
|
|
|
|
|
namespace llvm {
|
|
|
|
std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
|
|
return EnableMacroFusion ? make_unique<AArch64MacroFusion>() : nullptr;
|
|
}
|
|
|
|
} // end namespace llvm
|