diff --git a/lib/Target/SystemZ/SystemZISelLowering.cpp b/lib/Target/SystemZ/SystemZISelLowering.cpp index 94a441b0345..430837ebc59 100644 --- a/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -4056,7 +4056,10 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1), SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ); - DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0)); + // emitAtomicCmpSwapW() will zero extend the result (original value). + SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0), + DAG.getValueType(NarrowVT)); + DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal); DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success); DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2)); return SDValue(); @@ -7578,7 +7581,6 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( MachineBasicBlock * SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, MachineBasicBlock *MBB) const { - MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = static_cast(Subtarget.getInstrInfo()); @@ -7588,7 +7590,7 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, Register Dest = MI.getOperand(0).getReg(); MachineOperand Base = earlyUseOperand(MI.getOperand(1)); int64_t Disp = MI.getOperand(2).getImm(); - Register OrigCmpVal = MI.getOperand(3).getReg(); + Register CmpVal = MI.getOperand(3).getReg(); Register OrigSwapVal = MI.getOperand(4).getReg(); Register BitShift = MI.getOperand(5).getReg(); Register NegBitShift = MI.getOperand(6).getReg(); @@ -7597,19 +7599,19 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; - // Get the right opcodes for the displacement. + // Get the right opcodes for the displacement and zero-extension. unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); + unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR; assert(LOpcode && CSOpcode && "Displacement out of range"); // Create virtual registers for temporary results. Register OrigOldVal = MRI.createVirtualRegister(RC); Register OldVal = MRI.createVirtualRegister(RC); - Register CmpVal = MRI.createVirtualRegister(RC); Register SwapVal = MRI.createVirtualRegister(RC); Register StoreVal = MRI.createVirtualRegister(RC); + Register OldValRot = MRI.createVirtualRegister(RC); Register RetryOldVal = MRI.createVirtualRegister(RC); - Register RetryCmpVal = MRI.createVirtualRegister(RC); Register RetrySwapVal = MRI.createVirtualRegister(RC); // Insert 2 basic blocks for the loop. @@ -7631,34 +7633,32 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, // LoopMBB: // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] - // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] - // %Dest = RLL %OldVal, BitSize(%BitShift) + // %OldValRot = RLL %OldVal, BitSize(%BitShift) // ^^ The low BitSize bits contain the field // of interest. - // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 + // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0 // ^^ Replace the upper 32-BitSize bits of the - // comparison value with those that we loaded, - // so that we can use a full word comparison. - // CR %Dest, %RetryCmpVal + // swap value with those that we loaded and rotated. + // %Dest = LL[CH] %OldValRot + // CR %Dest, %CmpVal // JNE DoneMBB // # Fall through to SetMBB MBB = LoopMBB; BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) .addReg(OrigOldVal).addMBB(StartMBB) .addReg(RetryOldVal).addMBB(SetMBB); - BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) - .addReg(OrigCmpVal).addMBB(StartMBB) - .addReg(RetryCmpVal).addMBB(SetMBB); BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) .addReg(OrigSwapVal).addMBB(StartMBB) .addReg(RetrySwapVal).addMBB(SetMBB); - BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) + BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot) .addReg(OldVal).addReg(BitShift).addImm(BitSize); - BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) - .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) + .addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0); + BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest) + .addReg(OldValRot); BuildMI(MBB, DL, TII->get(SystemZ::CR)) - .addReg(Dest).addReg(RetryCmpVal); + .addReg(Dest).addReg(CmpVal); BuildMI(MBB, DL, TII->get(SystemZ::BRC)) .addImm(SystemZ::CCMASK_ICMP) .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); @@ -7666,17 +7666,12 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, MBB->addSuccessor(SetMBB); // SetMBB: - // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 - // ^^ Replace the upper 32-BitSize bits of the new - // value with those that we loaded. - // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) + // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) // ^^ Rotate the new field to its proper position. - // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) + // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base) // JNE LoopMBB // # fall through to ExitMMB MBB = SetMBB; - BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) - .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) diff --git a/lib/Target/SystemZ/SystemZISelLowering.h b/lib/Target/SystemZ/SystemZISelLowering.h index 78a0e55f9b7..a9f6cd3ae8a 100644 --- a/lib/Target/SystemZ/SystemZISelLowering.h +++ b/lib/Target/SystemZ/SystemZISelLowering.h @@ -550,7 +550,10 @@ public: unsigned Depth) const override; ISD::NodeType getExtendForAtomicOps() const override { - return ISD::ANY_EXTEND; + return ISD::ZERO_EXTEND; + } + ISD::NodeType getExtendForAtomicCmpSwapArg() const override { + return ISD::ZERO_EXTEND; } bool supportSwiftError() const override { diff --git a/test/CodeGen/SystemZ/cmpxchg-01.ll b/test/CodeGen/SystemZ/cmpxchg-01.ll index 82c67811fb9..29dbfb2b051 100644 --- a/test/CodeGen/SystemZ/cmpxchg-01.ll +++ b/test/CodeGen/SystemZ/cmpxchg-01.ll @@ -15,11 +15,12 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) { ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r3, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llcr %r4, %r4 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll %r2, [[OLD]], 8(%r3) -; CHECK-MAIN: risbg %r4, %r2, 32, 55, 0 -; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]] ; CHECK-MAIN: risbg %r5, %r2, 32, 55, 0 +; CHECK-MAIN: llcr %r2, %r2 +; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]] ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] @@ -48,7 +49,6 @@ define i8 @f2(i8 *%src) { ; ; CHECK-SHIFT-LABEL: f2: ; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88 -; CHECK-SHIFT: risbg ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0 ; CHECK-SHIFT: br %r14 %pair = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst @@ -62,12 +62,13 @@ define i32 @f3(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) { ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r3, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llcr %r2, %r4 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r3) -; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 -; CHECK-MAIN: cr [[TMP]], %r4 -; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] ; CHECK-MAIN: risbg %r5, [[TMP]], 32, 55, 0 +; CHECK-MAIN: llcr [[TMP2:%r[0-9]+]], [[TMP]] +; CHECK-MAIN: cr [[TMP2]], %r2 +; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] @@ -98,12 +99,13 @@ declare void @g() ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r2, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llcr %r3, %r3 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2) -; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0 +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 +; CHECK-MAIN: llcr [[TMP]], [[TMP]] ; CHECK-MAIN: cr [[TMP]], %r3 ; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] -; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] @@ -136,12 +138,13 @@ exit: ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r2, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llcr %r3, %r3 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2) -; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0 +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 +; CHECK-MAIN: llcr [[TMP]], [[TMP]] ; CHECK-MAIN: cr [[TMP]], %r3 ; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] -; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] diff --git a/test/CodeGen/SystemZ/cmpxchg-02.ll b/test/CodeGen/SystemZ/cmpxchg-02.ll index 6e266a1308c..f1522a235d7 100644 --- a/test/CodeGen/SystemZ/cmpxchg-02.ll +++ b/test/CodeGen/SystemZ/cmpxchg-02.ll @@ -15,11 +15,12 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) { ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r3, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llhr %r4, %r4 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll %r2, [[OLD]], 16(%r3) -; CHECK-MAIN: risbg %r4, %r2, 32, 47, 0 -; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]] ; CHECK-MAIN: risbg %r5, %r2, 32, 47, 0 +; CHECK-MAIN: llhr %r2, %r2 +; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]] ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] @@ -48,7 +49,6 @@ define i16 @f2(i16 *%src) { ; ; CHECK-SHIFT-LABEL: f2: ; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88 -; CHECK-SHIFT: risbg ; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0 ; CHECK-SHIFT: br %r14 %pair = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst @@ -62,12 +62,13 @@ define i32 @f3(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) { ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r3, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llhr %r2, %r4 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r3) -; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 -; CHECK-MAIN: cr [[TMP]], %r4 -; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] ; CHECK-MAIN: risbg %r5, [[TMP]], 32, 47, 0 +; CHECK-MAIN: llhr %r14, %r14 +; CHECK-MAIN: cr [[TMP]], %r2 +; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] @@ -97,12 +98,13 @@ declare void @g() ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r2, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llhr %r3, %r3 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2) -; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0 +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 +; CHECK-MAIN: llhr %r14, %r14 ; CHECK-MAIN: cr [[TMP]], %r3 ; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] -; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] @@ -135,12 +137,13 @@ exit: ; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}} ; CHECK-MAIN-DAG: sll %r2, 3 ; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]]) +; CHECK-MAIN-DAG: llhr %r3, %r3 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]: ; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2) -; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0 +; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 +; CHECK-MAIN: llhr %r14, %r14 ; CHECK-MAIN: cr [[TMP]], %r3 ; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]] -; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}}) ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]]) ; CHECK-MAIN: jl [[LOOP]] diff --git a/test/CodeGen/SystemZ/cmpxchg-05.ll b/test/CodeGen/SystemZ/cmpxchg-05.ll index ecfe2dabeaf..9a6d6c9d848 100644 --- a/test/CodeGen/SystemZ/cmpxchg-05.ll +++ b/test/CodeGen/SystemZ/cmpxchg-05.ll @@ -6,7 +6,7 @@ ; CHECK: crjlh ; CHECK-NOT: llcr ; CHECK-NOT: cr -; CHECK: llgcr %r2, [[RES:%r[0-9]+]] +; CHECK: llgfr %r2, [[RES:%r[0-9]+]] ; CHECK-NOT: llcr ; CHECK-NOT: cr define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) { @@ -19,7 +19,7 @@ define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) { ; CHECK: crjlh ; CHECK-NOT: llhr ; CHECK-NOT: cr -; CHECK: llghr %r2, [[RES:%r[0-9]+]] +; CHECK: llgfr %r2, [[RES:%r[0-9]+]] ; CHECK-NOT: llhr ; CHECK-NOT: cr define zeroext i16 @f2(i16* nocapture, i16 zeroext, i16 zeroext) { @@ -53,4 +53,3 @@ define signext i16 @f4(i16* nocapture, i16 signext, i16 signext) { %res = extractvalue { i16, i1 } %cx, 0 ret i16 %res } -