RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-22 18:54:02 +01:00
Code Issues Projects Releases Wiki Activity

[SystemZ] Reimplement the i8/i16 compare-and-swap logic.

Browse Source 
Even though the implementation in emitAtomicCmpSwapW() was correct, it made
Valgrind report an error. Instead of using a RISBG on CmpVal, an LL[CH]R can
be made on the OldVal, and the problem is avoided.

Review: Ulrich Weigand

Differential Revision: https://reviews.llvm.org/D97604
This commit is contained in:
Jonas Paulsson 2021-02-26 18:55:44 -06:00
parent 620203451b
commit d57759f053
5 changed files with 53 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
lib/Target/SystemZ/SystemZISelLowering.cpp
View File

@ -4056,7 +4056,10 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
// emitAtomicCmpSwapW() will zero extend the result (original value).
SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
DAG.getValueType(NarrowVT));
DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
return SDValue();
@ -7578,7 +7581,6 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
MachineBasicBlock *
SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
MachineBasicBlock *MBB) const {
MachineFunction &MF = *MBB->getParent();
const SystemZInstrInfo *TII =
static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
@ -7588,7 +7590,7 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
Register Dest = MI.getOperand(0).getReg();
MachineOperand Base = earlyUseOperand(MI.getOperand(1));
int64_t Disp = MI.getOperand(2).getImm();
Register OrigCmpVal = MI.getOperand(3).getReg();
Register CmpVal = MI.getOperand(3).getReg();
Register OrigSwapVal = MI.getOperand(4).getReg();
Register BitShift = MI.getOperand(5).getReg();
Register NegBitShift = MI.getOperand(6).getReg();
@ -7597,19 +7599,19 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
// Get the right opcodes for the displacement.
// Get the right opcodes for the displacement and zero-extension.
unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
assert(LOpcode && CSOpcode && "Displacement out of range");
// Create virtual registers for temporary results.
Register OrigOldVal = MRI.createVirtualRegister(RC);
Register OldVal = MRI.createVirtualRegister(RC);
Register CmpVal = MRI.createVirtualRegister(RC);
Register SwapVal = MRI.createVirtualRegister(RC);
Register StoreVal = MRI.createVirtualRegister(RC);
Register OldValRot = MRI.createVirtualRegister(RC);
Register RetryOldVal = MRI.createVirtualRegister(RC);
Register RetryCmpVal = MRI.createVirtualRegister(RC);
Register RetrySwapVal = MRI.createVirtualRegister(RC);
// Insert 2 basic blocks for the loop.
@ -7631,34 +7633,32 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
// LoopMBB:
// %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
// %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ]
// %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
// %Dest = RLL %OldVal, BitSize(%BitShift)
// %OldValRot = RLL %OldVal, BitSize(%BitShift)
// ^^ The low BitSize bits contain the field
// of interest.
// %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0
// %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
// ^^ Replace the upper 32-BitSize bits of the
// comparison value with those that we loaded,
// so that we can use a full word comparison.
// CR %Dest, %RetryCmpVal
// swap value with those that we loaded and rotated.
// %Dest = LL[CH] %OldValRot
// CR %Dest, %CmpVal
// JNE DoneMBB
// # Fall through to SetMBB
MBB = LoopMBB;
BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
.addReg(OrigOldVal).addMBB(StartMBB)
.addReg(RetryOldVal).addMBB(SetMBB);
BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
.addReg(OrigCmpVal).addMBB(StartMBB)
.addReg(RetryCmpVal).addMBB(SetMBB);
BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
.addReg(OrigSwapVal).addMBB(StartMBB)
.addReg(RetrySwapVal).addMBB(SetMBB);
BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
.addReg(OldVal).addReg(BitShift).addImm(BitSize);
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
.addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
.addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);
BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
.addReg(OldValRot);
BuildMI(MBB, DL, TII->get(SystemZ::CR))
.addReg(Dest).addReg(RetryCmpVal);
.addReg(Dest).addReg(CmpVal);
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
.addImm(SystemZ::CCMASK_ICMP)
.addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
@ -7666,17 +7666,12 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
MBB->addSuccessor(SetMBB);
// SetMBB:
// %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0
// ^^ Replace the upper 32-BitSize bits of the new
// value with those that we loaded.
// %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
// %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
// ^^ Rotate the new field to its proper position.
// %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base)
// %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
// JNE LoopMBB
// # fall through to ExitMMB
MBB = SetMBB;
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
.addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
.addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)

5
lib/Target/SystemZ/SystemZISelLowering.h
View File

@ -550,7 +550,10 @@ public:
unsigned Depth) const override;
ISD::NodeType getExtendForAtomicOps() const override {
return ISD::ANY_EXTEND;
return ISD::ZERO_EXTEND;
}
ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
return ISD::ZERO_EXTEND;
}
bool supportSwiftError() const override {

23
test/CodeGen/SystemZ/cmpxchg-01.ll
View File

@ -15,11 +15,12 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llcr %r4, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll %r2, [[OLD]], 8(%r3)
; CHECK-MAIN: risbg %r4, %r2, 32, 55, 0
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, %r2, 32, 55, 0
; CHECK-MAIN: llcr %r2, %r2
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@ -48,7 +49,6 @@ define i8 @f2(i8 *%src) {
;
; CHECK-SHIFT-LABEL: f2:
; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88
; CHECK-SHIFT: risbg
; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0
; CHECK-SHIFT: br %r14
%pair = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst
@ -62,12 +62,13 @@ define i32 @f3(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llcr %r2, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r3)
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: cr [[TMP]], %r4
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, [[TMP]], 32, 55, 0
; CHECK-MAIN: llcr [[TMP2:%r[0-9]+]], [[TMP]]
; CHECK-MAIN: cr [[TMP2]], %r2
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@ -98,12 +99,13 @@ declare void @g()
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llcr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2)
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: llcr [[TMP]], [[TMP]]
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@ -136,12 +138,13 @@ exit:
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llcr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2)
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: llcr [[TMP]], [[TMP]]
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]

23
test/CodeGen/SystemZ/cmpxchg-02.ll
View File

@ -15,11 +15,12 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llhr %r4, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll %r2, [[OLD]], 16(%r3)
; CHECK-MAIN: risbg %r4, %r2, 32, 47, 0
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, %r2, 32, 47, 0
; CHECK-MAIN: llhr %r2, %r2
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@ -48,7 +49,6 @@ define i16 @f2(i16 *%src) {
;
; CHECK-SHIFT-LABEL: f2:
; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88
; CHECK-SHIFT: risbg
; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0
; CHECK-SHIFT: br %r14
%pair = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst
@ -62,12 +62,13 @@ define i32 @f3(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r3, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llhr %r2, %r4
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r3)
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: cr [[TMP]], %r4
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r5, [[TMP]], 32, 47, 0
; CHECK-MAIN: llhr %r14, %r14
; CHECK-MAIN: cr [[TMP]], %r2
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@ -97,12 +98,13 @@ declare void @g()
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llhr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2)
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: llhr %r14, %r14
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]
@ -135,12 +137,13 @@ exit:
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
; CHECK-MAIN-DAG: sll %r2, 3
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
; CHECK-MAIN-DAG: llhr %r3, %r3
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2)
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: llhr %r14, %r14
; CHECK-MAIN: cr [[TMP]], %r3
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}})
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
; CHECK-MAIN: jl [[LOOP]]

5
test/CodeGen/SystemZ/cmpxchg-05.ll
View File

@ -6,7 +6,7 @@
; CHECK: crjlh
; CHECK-NOT: llcr
; CHECK-NOT: cr
; CHECK: llgcr %r2, [[RES:%r[0-9]+]]
; CHECK: llgfr %r2, [[RES:%r[0-9]+]]
; CHECK-NOT: llcr
; CHECK-NOT: cr
define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) {
@ -19,7 +19,7 @@ define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) {
; CHECK: crjlh
; CHECK-NOT: llhr
; CHECK-NOT: cr
; CHECK: llghr %r2, [[RES:%r[0-9]+]]
; CHECK: llgfr %r2, [[RES:%r[0-9]+]]
; CHECK-NOT: llhr
; CHECK-NOT: cr
define zeroext i16 @f2(i16* nocapture, i16 zeroext, i16 zeroext) {
@ -53,4 +53,3 @@ define signext i16 @f4(i16* nocapture, i16 signext, i16 signext) {
%res = extractvalue { i16, i1 } %cx, 0
ret i16 %res
}