mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2025-01-31 20:51:52 +01:00
78e18785d2
Utilise a similar ('late') lowering strategy to D47882. The changes to
AtomicExpandPass allow this strategy to be utilised by other targets which
implement shouldExpandAtomicCmpXchgInIR.
All cmpxchg are lowered as 'strong' currently and failure ordering is ignored.
This is conservative but correct.
Differential Revision: https://reviews.llvm.org/D48131
llvm-svn: 347914
557 lines
19 KiB
C++
557 lines
19 KiB
C++
//===-- RISCVExpandPseudoInsts.cpp - Expand pseudo instructions -----------===//
|
|
//
|
|
// The LLVM Compiler Infrastructure
|
|
//
|
|
// This file is distributed under the University of Illinois Open Source
|
|
// License. See LICENSE.TXT for details.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file contains a pass that expands pseudo instructions into target
|
|
// instructions. This pass should be run after register allocation but before
|
|
// the post-regalloc scheduling pass.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "RISCV.h"
|
|
#include "RISCVInstrInfo.h"
|
|
#include "RISCVTargetMachine.h"
|
|
|
|
#include "llvm/CodeGen/LivePhysRegs.h"
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
|
|
|
using namespace llvm;
|
|
|
|
#define RISCV_EXPAND_PSEUDO_NAME "RISCV pseudo instruction expansion pass"
|
|
|
|
namespace {
|
|
|
|
class RISCVExpandPseudo : public MachineFunctionPass {
|
|
public:
|
|
const RISCVInstrInfo *TII;
|
|
static char ID;
|
|
|
|
RISCVExpandPseudo() : MachineFunctionPass(ID) {
|
|
initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry());
|
|
}
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
|
|
|
StringRef getPassName() const override { return RISCV_EXPAND_PSEUDO_NAME; }
|
|
|
|
private:
|
|
bool expandMBB(MachineBasicBlock &MBB);
|
|
bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
MachineBasicBlock::iterator &NextMBBI);
|
|
bool expandAtomicBinOp(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
|
|
bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI);
|
|
bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI,
|
|
AtomicRMWInst::BinOp, bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI);
|
|
bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI, bool IsMasked,
|
|
int Width, MachineBasicBlock::iterator &NextMBBI);
|
|
};
|
|
|
|
char RISCVExpandPseudo::ID = 0;
|
|
|
|
bool RISCVExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
|
|
TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
|
|
bool Modified = false;
|
|
for (auto &MBB : MF)
|
|
Modified |= expandMBB(MBB);
|
|
return Modified;
|
|
}
|
|
|
|
bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
|
|
bool Modified = false;
|
|
|
|
MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
|
|
while (MBBI != E) {
|
|
MachineBasicBlock::iterator NMBBI = std::next(MBBI);
|
|
Modified |= expandMI(MBB, MBBI, NMBBI);
|
|
MBBI = NMBBI;
|
|
}
|
|
|
|
return Modified;
|
|
}
|
|
|
|
bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
|
|
MachineBasicBlock::iterator MBBI,
|
|
MachineBasicBlock::iterator &NextMBBI) {
|
|
switch (MBBI->getOpcode()) {
|
|
case RISCV::PseudoAtomicLoadNand32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicSwap32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadAdd32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadSub32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadNand32:
|
|
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadMax32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadMin32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadUMax32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoMaskedAtomicLoadUMin32:
|
|
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
|
|
NextMBBI);
|
|
case RISCV::PseudoCmpXchg32:
|
|
return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
|
|
case RISCV::PseudoMaskedCmpXchg32:
|
|
return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static unsigned getLRForRMW32(AtomicOrdering Ordering) {
|
|
switch (Ordering) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicOrdering");
|
|
case AtomicOrdering::Monotonic:
|
|
return RISCV::LR_W;
|
|
case AtomicOrdering::Acquire:
|
|
return RISCV::LR_W_AQ;
|
|
case AtomicOrdering::Release:
|
|
return RISCV::LR_W;
|
|
case AtomicOrdering::AcquireRelease:
|
|
return RISCV::LR_W_AQ;
|
|
case AtomicOrdering::SequentiallyConsistent:
|
|
return RISCV::LR_W_AQ_RL;
|
|
}
|
|
}
|
|
|
|
static unsigned getSCForRMW32(AtomicOrdering Ordering) {
|
|
switch (Ordering) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicOrdering");
|
|
case AtomicOrdering::Monotonic:
|
|
return RISCV::SC_W;
|
|
case AtomicOrdering::Acquire:
|
|
return RISCV::SC_W;
|
|
case AtomicOrdering::Release:
|
|
return RISCV::SC_W_RL;
|
|
case AtomicOrdering::AcquireRelease:
|
|
return RISCV::SC_W_RL;
|
|
case AtomicOrdering::SequentiallyConsistent:
|
|
return RISCV::SC_W_AQ_RL;
|
|
}
|
|
}
|
|
|
|
static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
|
|
DebugLoc DL, MachineBasicBlock *ThisMBB,
|
|
MachineBasicBlock *LoopMBB,
|
|
MachineBasicBlock *DoneMBB,
|
|
AtomicRMWInst::BinOp BinOp, int Width) {
|
|
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
|
|
unsigned DestReg = MI.getOperand(0).getReg();
|
|
unsigned ScratchReg = MI.getOperand(1).getReg();
|
|
unsigned AddrReg = MI.getOperand(2).getReg();
|
|
unsigned IncrReg = MI.getOperand(3).getReg();
|
|
AtomicOrdering Ordering =
|
|
static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
|
|
|
|
// .loop:
|
|
// lr.w dest, (addr)
|
|
// binop scratch, dest, val
|
|
// sc.w scratch, scratch, (addr)
|
|
// bnez scratch, loop
|
|
BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
|
|
.addReg(AddrReg);
|
|
switch (BinOp) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicRMW BinOp");
|
|
case AtomicRMWInst::Nand:
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addImm(-1);
|
|
break;
|
|
}
|
|
BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addReg(ScratchReg);
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(ScratchReg)
|
|
.addReg(RISCV::X0)
|
|
.addMBB(LoopMBB);
|
|
}
|
|
|
|
static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
|
|
MachineBasicBlock *MBB, unsigned DestReg,
|
|
unsigned OldValReg, unsigned NewValReg,
|
|
unsigned MaskReg, unsigned ScratchReg) {
|
|
assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
|
|
assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
|
|
assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
|
|
|
|
// We select bits from newval and oldval using:
|
|
// https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
|
|
// r = oldval ^ ((oldval ^ newval) & masktargetdata);
|
|
BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
|
|
.addReg(OldValReg)
|
|
.addReg(NewValReg);
|
|
BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
|
|
.addReg(OldValReg)
|
|
.addReg(ScratchReg);
|
|
}
|
|
|
|
static void doMaskedAtomicBinOpExpansion(
|
|
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
|
|
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
|
|
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
|
|
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
|
|
unsigned DestReg = MI.getOperand(0).getReg();
|
|
unsigned ScratchReg = MI.getOperand(1).getReg();
|
|
unsigned AddrReg = MI.getOperand(2).getReg();
|
|
unsigned IncrReg = MI.getOperand(3).getReg();
|
|
unsigned MaskReg = MI.getOperand(4).getReg();
|
|
AtomicOrdering Ordering =
|
|
static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
|
|
|
|
// .loop:
|
|
// lr.w destreg, (alignedaddr)
|
|
// binop scratch, destreg, incr
|
|
// xor scratch, destreg, scratch
|
|
// and scratch, scratch, masktargetdata
|
|
// xor scratch, destreg, scratch
|
|
// sc.w scratch, scratch, (alignedaddr)
|
|
// bnez scratch, loop
|
|
BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
|
|
.addReg(AddrReg);
|
|
switch (BinOp) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicRMW BinOp");
|
|
case AtomicRMWInst::Xchg:
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
|
|
.addReg(RISCV::X0)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Add:
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Sub:
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
break;
|
|
case AtomicRMWInst::Nand:
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(IncrReg);
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
|
|
.addReg(ScratchReg)
|
|
.addImm(-1);
|
|
break;
|
|
}
|
|
|
|
insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
|
|
ScratchReg);
|
|
|
|
BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addReg(ScratchReg);
|
|
BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(ScratchReg)
|
|
.addReg(RISCV::X0)
|
|
.addMBB(LoopMBB);
|
|
}
|
|
|
|
bool RISCVExpandPseudo::expandAtomicBinOp(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI) {
|
|
MachineInstr &MI = *MBBI;
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
|
|
MachineFunction *MF = MBB.getParent();
|
|
auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
|
|
// Insert new MBBs.
|
|
MF->insert(++MBB.getIterator(), LoopMBB);
|
|
MF->insert(++LoopMBB->getIterator(), DoneMBB);
|
|
|
|
// Set up successors and transfer remaining instructions to DoneMBB.
|
|
LoopMBB->addSuccessor(LoopMBB);
|
|
LoopMBB->addSuccessor(DoneMBB);
|
|
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
|
|
DoneMBB->transferSuccessors(&MBB);
|
|
MBB.addSuccessor(LoopMBB);
|
|
|
|
if (!IsMasked)
|
|
doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
|
|
else
|
|
doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
|
|
Width);
|
|
|
|
NextMBBI = MBB.end();
|
|
MI.eraseFromParent();
|
|
|
|
LivePhysRegs LiveRegs;
|
|
computeAndAddLiveIns(LiveRegs, *LoopMBB);
|
|
computeAndAddLiveIns(LiveRegs, *DoneMBB);
|
|
|
|
return true;
|
|
}
|
|
|
|
static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
|
|
MachineBasicBlock *MBB, unsigned ValReg,
|
|
unsigned ShamtReg) {
|
|
BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
|
|
.addReg(ValReg)
|
|
.addReg(ShamtReg);
|
|
BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
|
|
.addReg(ValReg)
|
|
.addReg(ShamtReg);
|
|
}
|
|
|
|
bool RISCVExpandPseudo::expandAtomicMinMaxOp(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
|
|
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
|
|
MachineBasicBlock::iterator &NextMBBI) {
|
|
assert(IsMasked == true &&
|
|
"Should only need to expand masked atomic max/min");
|
|
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
|
|
|
|
MachineInstr &MI = *MBBI;
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
MachineFunction *MF = MBB.getParent();
|
|
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
|
|
// Insert new MBBs.
|
|
MF->insert(++MBB.getIterator(), LoopHeadMBB);
|
|
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
|
|
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
|
|
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
|
|
|
|
// Set up successors and transfer remaining instructions to DoneMBB.
|
|
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
|
|
LoopHeadMBB->addSuccessor(LoopTailMBB);
|
|
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
|
|
LoopTailMBB->addSuccessor(LoopHeadMBB);
|
|
LoopTailMBB->addSuccessor(DoneMBB);
|
|
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
|
|
DoneMBB->transferSuccessors(&MBB);
|
|
MBB.addSuccessor(LoopHeadMBB);
|
|
|
|
unsigned DestReg = MI.getOperand(0).getReg();
|
|
unsigned Scratch1Reg = MI.getOperand(1).getReg();
|
|
unsigned Scratch2Reg = MI.getOperand(2).getReg();
|
|
unsigned AddrReg = MI.getOperand(3).getReg();
|
|
unsigned IncrReg = MI.getOperand(4).getReg();
|
|
unsigned MaskReg = MI.getOperand(5).getReg();
|
|
bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
|
|
AtomicOrdering Ordering =
|
|
static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());
|
|
|
|
//
|
|
// .loophead:
|
|
// lr.w destreg, (alignedaddr)
|
|
// and scratch2, destreg, mask
|
|
// mv scratch1, destreg
|
|
// [sext scratch2 if signed min/max]
|
|
// ifnochangeneeded scratch2, incr, .looptail
|
|
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
|
|
.addReg(AddrReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
|
|
.addReg(DestReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
|
|
.addReg(DestReg)
|
|
.addImm(0);
|
|
|
|
switch (BinOp) {
|
|
default:
|
|
llvm_unreachable("Unexpected AtomicRMW BinOp");
|
|
case AtomicRMWInst::Max: {
|
|
insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
|
|
.addReg(Scratch2Reg)
|
|
.addReg(IncrReg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
}
|
|
case AtomicRMWInst::Min: {
|
|
insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
|
|
.addReg(IncrReg)
|
|
.addReg(Scratch2Reg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
}
|
|
case AtomicRMWInst::UMax:
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
|
|
.addReg(Scratch2Reg)
|
|
.addReg(IncrReg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
case AtomicRMWInst::UMin:
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
|
|
.addReg(IncrReg)
|
|
.addReg(Scratch2Reg)
|
|
.addMBB(LoopTailMBB);
|
|
break;
|
|
}
|
|
|
|
// .loopifbody:
|
|
// xor scratch1, destreg, incr
|
|
// and scratch1, scratch1, mask
|
|
// xor scratch1, destreg, scratch1
|
|
insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
|
|
MaskReg, Scratch1Reg);
|
|
|
|
// .looptail:
|
|
// sc.w scratch1, scratch1, (addr)
|
|
// bnez scratch1, loop
|
|
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
|
|
.addReg(AddrReg)
|
|
.addReg(Scratch1Reg);
|
|
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(Scratch1Reg)
|
|
.addReg(RISCV::X0)
|
|
.addMBB(LoopHeadMBB);
|
|
|
|
NextMBBI = MBB.end();
|
|
MI.eraseFromParent();
|
|
|
|
LivePhysRegs LiveRegs;
|
|
computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
|
|
computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
|
|
computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
|
|
computeAndAddLiveIns(LiveRegs, *DoneMBB);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool RISCVExpandPseudo::expandAtomicCmpXchg(
|
|
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
|
|
int Width, MachineBasicBlock::iterator &NextMBBI) {
|
|
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
|
|
MachineInstr &MI = *MBBI;
|
|
DebugLoc DL = MI.getDebugLoc();
|
|
MachineFunction *MF = MBB.getParent();
|
|
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
|
|
|
|
// Insert new MBBs.
|
|
MF->insert(++MBB.getIterator(), LoopHeadMBB);
|
|
MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
|
|
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
|
|
|
|
// Set up successors and transfer remaining instructions to DoneMBB.
|
|
LoopHeadMBB->addSuccessor(LoopTailMBB);
|
|
LoopHeadMBB->addSuccessor(DoneMBB);
|
|
LoopTailMBB->addSuccessor(DoneMBB);
|
|
LoopTailMBB->addSuccessor(LoopHeadMBB);
|
|
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
|
|
DoneMBB->transferSuccessors(&MBB);
|
|
MBB.addSuccessor(LoopHeadMBB);
|
|
|
|
unsigned DestReg = MI.getOperand(0).getReg();
|
|
unsigned ScratchReg = MI.getOperand(1).getReg();
|
|
unsigned AddrReg = MI.getOperand(2).getReg();
|
|
unsigned CmpValReg = MI.getOperand(3).getReg();
|
|
unsigned NewValReg = MI.getOperand(4).getReg();
|
|
AtomicOrdering Ordering =
|
|
static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
|
|
|
|
if (!IsMasked) {
|
|
// .loophead:
|
|
// lr.w dest, (addr)
|
|
// bne dest, cmpval, done
|
|
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
|
|
.addReg(AddrReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(DestReg)
|
|
.addReg(CmpValReg)
|
|
.addMBB(DoneMBB);
|
|
// .looptail:
|
|
// sc.w scratch, newval, (addr)
|
|
// bnez scratch, loophead
|
|
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addReg(NewValReg);
|
|
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(ScratchReg)
|
|
.addReg(RISCV::X0)
|
|
.addMBB(LoopHeadMBB);
|
|
} else {
|
|
// .loophead:
|
|
// lr.w dest, (addr)
|
|
// and scratch, dest, mask
|
|
// bne scratch, cmpval, done
|
|
unsigned MaskReg = MI.getOperand(5).getReg();
|
|
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
|
|
.addReg(AddrReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
|
|
.addReg(DestReg)
|
|
.addReg(MaskReg);
|
|
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(ScratchReg)
|
|
.addReg(CmpValReg)
|
|
.addMBB(DoneMBB);
|
|
|
|
// .looptail:
|
|
// xor scratch, dest, newval
|
|
// and scratch, scratch, mask
|
|
// xor scratch, dest, scratch
|
|
// sc.w scratch, scratch, (adrr)
|
|
// bnez scratch, loophead
|
|
insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
|
|
MaskReg, ScratchReg);
|
|
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
|
|
.addReg(AddrReg)
|
|
.addReg(ScratchReg);
|
|
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
|
|
.addReg(ScratchReg)
|
|
.addReg(RISCV::X0)
|
|
.addMBB(LoopHeadMBB);
|
|
}
|
|
|
|
NextMBBI = MBB.end();
|
|
MI.eraseFromParent();
|
|
|
|
LivePhysRegs LiveRegs;
|
|
computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
|
|
computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
|
|
computeAndAddLiveIns(LiveRegs, *DoneMBB);
|
|
|
|
return true;
|
|
}
|
|
|
|
} // end of anonymous namespace
|
|
|
|
INITIALIZE_PASS(RISCVExpandPseudo, "riscv-expand-pseudo",
|
|
RISCV_EXPAND_PSEUDO_NAME, false, false)
|
|
namespace llvm {
|
|
|
|
FunctionPass *createRISCVExpandPseudoPass() { return new RISCVExpandPseudo(); }
|
|
|
|
} // end of namespace llvm
|