mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-22 02:33:06 +01:00
[AArch64] Out-of-line atomics (-moutline-atomics) implementation.
This patch implements out of line atomics for LSE deployment mechanism. Details how it works can be found in llvm/docs/Atomics.rst Options -moutline-atomics and -mno-outline-atomics to enable and disable it were added to clang driver. This is clang and llvm part of out-of-line atomics interface, library part is already supported by libgcc. Compiler-rt support is provided in separate patch. Differential Revision: https://reviews.llvm.org/D91157
This commit is contained in:
Pavel Iliin
parent
5c4c695a34
commit
2529cb73ff
@ -621,3 +621,23 @@ fence on either side of a normal load or store.)
|
||||
There's also, somewhat separately, the possibility to lower ``ATOMIC_FENCE`` to
|
||||
``__sync_synchronize()``. This may happen or not happen independent of all the
|
||||
above, controlled purely by ``setOperationAction(ISD::ATOMIC_FENCE, ...)``.
|
||||
|
||||
On AArch64, a variant of the __sync_* routines is used which contain the memory
|
||||
order as part of the function name. These routines may determine at runtime
|
||||
whether the single-instruction atomic operations which were introduced as part
|
||||
of AArch64 Large System Extensions "LSE" instruction set are available, or if
|
||||
it needs to fall back to an LL/SC loop. The following helper functions are
|
||||
implemented in both ``compiler-rt`` and ``libgcc`` libraries
|
||||
(``N`` is one of 1, 2, 4, 8, and ``M`` is one of 1, 2, 4, 8 and 16, and
|
||||
``ORDER`` is one of 'relax', 'acq', 'rel', 'acq_rel')::
|
||||
|
||||
iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)
|
||||
iN __aarch64_swpN_ORDER(iN val, iN *ptr)
|
||||
iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)
|
||||
iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)
|
||||
iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)
|
||||
iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)
|
||||
|
||||
Please note, if LSE instruction set is specified for AArch64 target then
|
||||
out-of-line atomics calls are not generated and single-instruction atomic
|
||||
operations are used in place.
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
#define LLVM_CODEGEN_RUNTIMELIBCALLS_H
|
||||
|
||||
#include "llvm/CodeGen/ValueTypes.h"
|
||||
#include "llvm/Support/AtomicOrdering.h"
|
||||
|
||||
namespace llvm {
|
||||
namespace RTLIB {
|
||||
@ -60,6 +61,10 @@ namespace RTLIB {
|
||||
/// UNKNOWN_LIBCALL if there is none.
|
||||
Libcall getSYNC(unsigned Opc, MVT VT);
|
||||
|
||||
/// Return the outline atomics value for the given opcode, atomic ordering
|
||||
/// and type, or UNKNOWN_LIBCALL if there is none.
|
||||
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
|
||||
|
||||
/// getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return
|
||||
/// MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given element size or
|
||||
/// UNKNOW_LIBCALL if there is none.
|
||||
|
||||
@ -545,6 +545,23 @@ HANDLE_LIBCALL(ATOMIC_FETCH_NAND_4, "__atomic_fetch_nand_4")
|
||||
HANDLE_LIBCALL(ATOMIC_FETCH_NAND_8, "__atomic_fetch_nand_8")
|
||||
HANDLE_LIBCALL(ATOMIC_FETCH_NAND_16, "__atomic_fetch_nand_16")
|
||||
|
||||
// Out-of-line atomics libcalls
|
||||
#define HLCALLS(A, N) \
|
||||
HANDLE_LIBCALL(A##N##_RELAX, nullptr) \
|
||||
HANDLE_LIBCALL(A##N##_ACQ, nullptr) \
|
||||
HANDLE_LIBCALL(A##N##_REL, nullptr) \
|
||||
HANDLE_LIBCALL(A##N##_ACQ_REL, nullptr)
|
||||
#define HLCALL5(A) \
|
||||
HLCALLS(A, 1) HLCALLS(A, 2) HLCALLS(A, 4) HLCALLS(A, 8) HLCALLS(A, 16)
|
||||
HLCALL5(OUTLINE_ATOMIC_CAS)
|
||||
HLCALL5(OUTLINE_ATOMIC_SWP)
|
||||
HLCALL5(OUTLINE_ATOMIC_LDADD)
|
||||
HLCALL5(OUTLINE_ATOMIC_LDSET)
|
||||
HLCALL5(OUTLINE_ATOMIC_LDCLR)
|
||||
HLCALL5(OUTLINE_ATOMIC_LDEOR)
|
||||
#undef HLCALLS
|
||||
#undef HLCALL5
|
||||
|
||||
// Stack Protector Fail
|
||||
HANDLE_LIBCALL(STACKPROTECTOR_CHECK_FAIL, "__stack_chk_fail")
|
||||
|
||||
|
||||
@ -4052,12 +4052,23 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
|
||||
case ISD::ATOMIC_LOAD_UMAX:
|
||||
case ISD::ATOMIC_CMP_SWAP: {
|
||||
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
|
||||
RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
|
||||
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
|
||||
|
||||
AtomicOrdering Order = cast<AtomicSDNode>(Node)->getOrdering();
|
||||
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
|
||||
EVT RetVT = Node->getValueType(0);
|
||||
SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end());
|
||||
TargetLowering::MakeLibCallOptions CallOptions;
|
||||
SmallVector<SDValue, 4> Ops;
|
||||
if (TLI.getLibcallName(LC)) {
|
||||
// If outline atomic available, prepare its arguments and expand.
|
||||
Ops.append(Node->op_begin() + 2, Node->op_end());
|
||||
Ops.push_back(Node->getOperand(1));
|
||||
|
||||
} else {
|
||||
LC = RTLIB::getSYNC(Opc, VT);
|
||||
assert(LC != RTLIB::UNKNOWN_LIBCALL &&
|
||||
"Unexpected atomic op or value type!");
|
||||
// Arguments for expansion to sync libcall
|
||||
Ops.append(Node->op_begin() + 1, Node->op_end());
|
||||
}
|
||||
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
|
||||
Ops, CallOptions,
|
||||
SDLoc(Node),
|
||||
|
||||
@ -2169,12 +2169,22 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
|
||||
std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
|
||||
unsigned Opc = Node->getOpcode();
|
||||
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
|
||||
RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
|
||||
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
|
||||
|
||||
AtomicOrdering order = cast<AtomicSDNode>(Node)->getOrdering();
|
||||
// Lower to outline atomic libcall if outline atomics enabled,
|
||||
// or to sync libcall otherwise
|
||||
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
|
||||
EVT RetVT = Node->getValueType(0);
|
||||
SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end());
|
||||
TargetLowering::MakeLibCallOptions CallOptions;
|
||||
SmallVector<SDValue, 4> Ops;
|
||||
if (TLI.getLibcallName(LC)) {
|
||||
Ops.append(Node->op_begin() + 2, Node->op_end());
|
||||
Ops.push_back(Node->getOperand(1));
|
||||
} else {
|
||||
LC = RTLIB::getSYNC(Opc, VT);
|
||||
assert(LC != RTLIB::UNKNOWN_LIBCALL &&
|
||||
"Unexpected atomic op or value type!");
|
||||
Ops.append(Node->op_begin() + 1, Node->op_end());
|
||||
}
|
||||
return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
|
||||
Node->getOperand(0));
|
||||
}
|
||||
|
||||
@ -448,6 +448,83 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
|
||||
return UNKNOWN_LIBCALL;
|
||||
}
|
||||
|
||||
RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
|
||||
MVT VT) {
|
||||
unsigned ModeN, ModelN;
|
||||
switch (VT.SimpleTy) {
|
||||
case MVT::i8:
|
||||
ModeN = 0;
|
||||
break;
|
||||
case MVT::i16:
|
||||
ModeN = 1;
|
||||
break;
|
||||
case MVT::i32:
|
||||
ModeN = 2;
|
||||
break;
|
||||
case MVT::i64:
|
||||
ModeN = 3;
|
||||
break;
|
||||
case MVT::i128:
|
||||
ModeN = 4;
|
||||
break;
|
||||
default:
|
||||
return UNKNOWN_LIBCALL;
|
||||
}
|
||||
|
||||
switch (Order) {
|
||||
case AtomicOrdering::Monotonic:
|
||||
ModelN = 0;
|
||||
break;
|
||||
case AtomicOrdering::Acquire:
|
||||
ModelN = 1;
|
||||
break;
|
||||
case AtomicOrdering::Release:
|
||||
ModelN = 2;
|
||||
break;
|
||||
case AtomicOrdering::AcquireRelease:
|
||||
case AtomicOrdering::SequentiallyConsistent:
|
||||
ModelN = 3;
|
||||
break;
|
||||
default:
|
||||
return UNKNOWN_LIBCALL;
|
||||
}
|
||||
|
||||
#define LCALLS(A, B) \
|
||||
{ A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
|
||||
#define LCALL5(A) \
|
||||
LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
|
||||
switch (Opc) {
|
||||
case ISD::ATOMIC_CMP_SWAP: {
|
||||
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
|
||||
return LC[ModeN][ModelN];
|
||||
}
|
||||
case ISD::ATOMIC_SWAP: {
|
||||
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
|
||||
return LC[ModeN][ModelN];
|
||||
}
|
||||
case ISD::ATOMIC_LOAD_ADD: {
|
||||
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
|
||||
return LC[ModeN][ModelN];
|
||||
}
|
||||
case ISD::ATOMIC_LOAD_OR: {
|
||||
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
|
||||
return LC[ModeN][ModelN];
|
||||
}
|
||||
case ISD::ATOMIC_LOAD_CLR: {
|
||||
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
|
||||
return LC[ModeN][ModelN];
|
||||
}
|
||||
case ISD::ATOMIC_LOAD_XOR: {
|
||||
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
|
||||
return LC[ModeN][ModelN];
|
||||
}
|
||||
default:
|
||||
return UNKNOWN_LIBCALL;
|
||||
}
|
||||
#undef LCALLS
|
||||
#undef LCALL5
|
||||
}
|
||||
|
||||
RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {
|
||||
#define OP_TO_LIBCALL(Name, Enum) \
|
||||
case Name: \
|
||||
|
||||
@ -61,6 +61,9 @@ def FeatureRAS : SubtargetFeature<"ras", "HasRAS", "true",
|
||||
def FeatureLSE : SubtargetFeature<"lse", "HasLSE", "true",
|
||||
"Enable ARMv8.1 Large System Extension (LSE) atomic instructions">;
|
||||
|
||||
def FeatureOutlineAtomics : SubtargetFeature<"outline-atomics", "OutlineAtomics", "true",
|
||||
"Enable out of line atomics to support LSE instructions">;
|
||||
|
||||
def FeatureRDM : SubtargetFeature<"rdm", "HasRDM", "true",
|
||||
"Enable ARMv8.1 Rounding Double Multiply Add/Subtract instructions">;
|
||||
|
||||
|
||||
@ -662,6 +662,57 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
|
||||
setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
|
||||
|
||||
// Generate outline atomics library calls only if LSE was not specified for
|
||||
// subtarget
|
||||
if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
|
||||
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
|
||||
setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
|
||||
#define LCALLNAMES(A, B, N) \
|
||||
setLibcallName(A##N##_RELAX, #B #N "_relax"); \
|
||||
setLibcallName(A##N##_ACQ, #B #N "_acq"); \
|
||||
setLibcallName(A##N##_REL, #B #N "_rel"); \
|
||||
setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
|
||||
#define LCALLNAME4(A, B) \
|
||||
LCALLNAMES(A, B, 1) \
|
||||
LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
|
||||
#define LCALLNAME5(A, B) \
|
||||
LCALLNAMES(A, B, 1) \
|
||||
LCALLNAMES(A, B, 2) \
|
||||
LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
|
||||
LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
|
||||
LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
|
||||
LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
|
||||
LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
|
||||
LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
|
||||
LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
|
||||
#undef LCALLNAMES
|
||||
#undef LCALLNAME4
|
||||
#undef LCALLNAME5
|
||||
}
|
||||
|
||||
// 128-bit loads and stores can be done without expanding
|
||||
setOperationAction(ISD::LOAD, MVT::i128, Custom);
|
||||
setOperationAction(ISD::STORE, MVT::i128, Custom);
|
||||
@ -10078,7 +10129,7 @@ SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
|
||||
SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
|
||||
SelectionDAG &DAG) const {
|
||||
auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
|
||||
if (!Subtarget.hasLSE())
|
||||
if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics())
|
||||
return SDValue();
|
||||
|
||||
// LSE has an atomic load-add instruction, but not a load-sub.
|
||||
@ -10095,7 +10146,7 @@ SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
|
||||
SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
|
||||
SelectionDAG &DAG) const {
|
||||
auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
|
||||
if (!Subtarget.hasLSE())
|
||||
if (!Subtarget.hasLSE() && !Subtarget.outlineAtomics())
|
||||
return SDValue();
|
||||
|
||||
// LSE has an atomic load-clear instruction, but not a load-and.
|
||||
@ -15549,7 +15600,7 @@ static void ReplaceCMP_SWAP_128Results(SDNode *N,
|
||||
assert(N->getValueType(0) == MVT::i128 &&
|
||||
"AtomicCmpSwap on types less than 128 should be legal");
|
||||
|
||||
if (Subtarget->hasLSE()) {
|
||||
if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) {
|
||||
// LSE has a 128-bit compare and swap (CASP), but i128 is not a legal type,
|
||||
// so lower it here, wrapped in REG_SEQUENCE and EXTRACT_SUBREG.
|
||||
SDValue Ops[] = {
|
||||
@ -15778,14 +15829,30 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
|
||||
// Nand not supported in LSE.
|
||||
if (AI->getOperation() == AtomicRMWInst::Nand) return AtomicExpansionKind::LLSC;
|
||||
// Leave 128 bits to LLSC.
|
||||
return (Subtarget->hasLSE() && Size < 128) ? AtomicExpansionKind::None : AtomicExpansionKind::LLSC;
|
||||
if (Subtarget->hasLSE() && Size < 128)
|
||||
return AtomicExpansionKind::None;
|
||||
if (Subtarget->outlineAtomics() && Size < 128) {
|
||||
// [U]Min/[U]Max RWM atomics are used in __sync_fetch_ libcalls so far.
|
||||
// Don't outline them unless
|
||||
// (1) high level <atomic> support approved:
|
||||
// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p0493r1.pdf
|
||||
// (2) low level libgcc and compiler-rt support implemented by:
|
||||
// min/max outline atomics helpers
|
||||
if (AI->getOperation() != AtomicRMWInst::Min &&
|
||||
AI->getOperation() != AtomicRMWInst::Max &&
|
||||
AI->getOperation() != AtomicRMWInst::UMin &&
|
||||
AI->getOperation() != AtomicRMWInst::UMax) {
|
||||
return AtomicExpansionKind::None;
|
||||
}
|
||||
}
|
||||
return AtomicExpansionKind::LLSC;
|
||||
}
|
||||
|
||||
TargetLowering::AtomicExpansionKind
|
||||
AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR(
|
||||
AtomicCmpXchgInst *AI) const {
|
||||
// If subtarget has LSE, leave cmpxchg intact for codegen.
|
||||
if (Subtarget->hasLSE())
|
||||
if (Subtarget->hasLSE() || Subtarget->outlineAtomics())
|
||||
return AtomicExpansionKind::None;
|
||||
// At -O0, fast-regalloc cannot cope with the live vregs necessary to
|
||||
// implement cmpxchg without spilling. If the address being exchanged is also
|
||||
|
||||
@ -195,6 +195,7 @@ protected:
|
||||
// Enable 64-bit vectorization in SLP.
|
||||
unsigned MinVectorRegisterBitWidth = 64;
|
||||
|
||||
bool OutlineAtomics = false;
|
||||
bool UseAA = false;
|
||||
bool PredictableSelectIsExpensive = false;
|
||||
bool BalanceFPOps = false;
|
||||
@ -471,6 +472,8 @@ public:
|
||||
|
||||
bool useAA() const override { return UseAA; }
|
||||
|
||||
bool outlineAtomics() const { return OutlineAtomics; }
|
||||
|
||||
bool hasVH() const { return HasVH; }
|
||||
bool hasPAN() const { return HasPAN; }
|
||||
bool hasLOR() const { return HasLOR; }
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s
|
||||
; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
@var = global i128 0
|
||||
|
||||
define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas16_acq
|
||||
; CHECK-LABEL: val_compare_and_swap:
|
||||
; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
|
||||
; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]]
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck -enable-var-scope %s
|
||||
; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck -enable-var-scope %s -check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas4_acq
|
||||
; CHECK-LABEL: val_compare_and_swap:
|
||||
; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0
|
||||
; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:
|
||||
@ -19,6 +21,7 @@ define i32 @val_compare_and_swap(i32* %p, i32 %cmp, i32 %new) #0 {
|
||||
}
|
||||
|
||||
define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas4_acq
|
||||
; CHECK-LABEL: val_compare_and_swap_from_load:
|
||||
; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2]
|
||||
; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:
|
||||
@ -40,6 +43,7 @@ define i32 @val_compare_and_swap_from_load(i32* %p, i32 %cmp, i32* %pnew) #0 {
|
||||
}
|
||||
|
||||
define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas4_acq_rel
|
||||
; CHECK-LABEL: val_compare_and_swap_rel:
|
||||
; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0
|
||||
; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:
|
||||
@ -58,6 +62,7 @@ define i32 @val_compare_and_swap_rel(i32* %p, i32 %cmp, i32 %new) #0 {
|
||||
}
|
||||
|
||||
define i64 @val_compare_and_swap_64(i64* %p, i64 %cmp, i64 %new) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas8_relax
|
||||
; CHECK-LABEL: val_compare_and_swap_64:
|
||||
; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0
|
||||
; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:
|
||||
@ -104,6 +109,7 @@ define i64 @fetch_and_nand_64(i64* %p) #0 {
|
||||
}
|
||||
|
||||
define i32 @fetch_and_or(i32* %p) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_ldset4_acq_rel
|
||||
; CHECK-LABEL: fetch_and_or:
|
||||
; CHECK: mov [[OLDVAL_REG:w[0-9]+]], #5
|
||||
; CHECK: [[TRYBB:.?LBB[0-9_]+]]:
|
||||
@ -118,6 +124,7 @@ define i32 @fetch_and_or(i32* %p) #0 {
|
||||
}
|
||||
|
||||
define i64 @fetch_and_or_64(i64* %p) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_ldset8_relax
|
||||
; CHECK: fetch_and_or_64:
|
||||
; CHECK: mov x[[ADDR:[0-9]+]], x0
|
||||
; CHECK: [[TRYBB:.?LBB[0-9_]+]]:
|
||||
@ -332,6 +339,7 @@ define void @atomic_store_relaxed_32(i32* %p, i32 %off32, i32 %val) #0 {
|
||||
}
|
||||
|
||||
define void @atomic_store_relaxed_64(i64* %p, i32 %off32, i64 %val) #0 {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_ldadd4_acq_rel
|
||||
; CHECK-LABEL: atomic_store_relaxed_64:
|
||||
%ptr_unsigned = getelementptr i64, i64* %p, i32 4095
|
||||
store atomic i64 %val, i64* %ptr_unsigned monotonic, align 8
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
; RUN: llc -mtriple=arm64_32-apple-ios7.0 -o - %s | FileCheck %s
|
||||
; RUN: llc -mtriple=arm64_32-apple-ios7.0 -mattr=+outline-atomics -o - %s | FileCheck %s -check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
define i8 @test_load_8(i8* %addr) {
|
||||
; CHECK-LABAL: test_load_8:
|
||||
@ -239,6 +240,7 @@ define i32 @test_stlxr_64(i64* %addr, i64 %val) {
|
||||
}
|
||||
|
||||
define {i8*, i1} @test_cmpxchg_ptr(i8** %addr, i8* %cmp, i8* %new) {
|
||||
; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_ptr:
|
||||
; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
|
||||
; CHECK: ldaxr [[OLD:w[0-9]+]], [x0]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,6 +1,8 @@
|
||||
; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
|
||||
; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -mattr=+outline-atomics < %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
define i32 @foo(i32* %var, i1 %cond) {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_ldadd4_relax
|
||||
; CHECK-LABEL: foo:
|
||||
br i1 %cond, label %atomic_ver, label %simple_ver
|
||||
simple_ver:
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s
|
||||
; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG
|
||||
; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics < %s | FileCheck %s --check-prefix=OUTLINE_ATOMICS
|
||||
|
||||
|
||||
; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
|
||||
@ -14,6 +15,14 @@
|
||||
|
||||
define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_add_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw add i8* @var8, i8 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -34,6 +43,14 @@ define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_add_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw add i16* @var16, i16 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -54,6 +71,14 @@ define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_add_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw add i32* @var32, i32 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -74,6 +99,14 @@ define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_add_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_relax
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw add i64* @var64, i64 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -94,6 +127,15 @@ define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_sub_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: neg w0, w0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd1_relax
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw sub i8* @var8, i8 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -114,6 +156,15 @@ define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_sub_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: neg w0, w0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd2_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw sub i16* @var16, i16 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -134,6 +185,15 @@ define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_sub_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: neg w0, w0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd4_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw sub i32* @var32, i32 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -154,6 +214,15 @@ define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_sub_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: neg x0, x0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw sub i64* @var64, i64 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -174,6 +243,15 @@ define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_and_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: mvn w0, w0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr1_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw and i8* @var8, i8 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -194,6 +272,15 @@ define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_and_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: mvn w0, w0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr2_relax
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw and i16* @var16, i16 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -214,6 +301,15 @@ define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_and_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: mvn w0, w0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw and i32* @var32, i32 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -234,6 +330,15 @@ define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_and_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: mvn x0, x0
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldclr8_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw and i64* @var64, i64 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -254,6 +359,14 @@ define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_or_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw or i8* @var8, i8 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -274,6 +387,14 @@ define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_or_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset2_relax
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw or i16* @var16, i16 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -294,6 +415,14 @@ define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_or_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset4_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw or i32* @var32, i32 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -314,6 +443,14 @@ define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_or_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldset8_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw or i64* @var64, i64 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -334,6 +471,14 @@ define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xor_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor1_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xor i8* @var8, i8 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -354,6 +499,14 @@ define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xor_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor2_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xor i16* @var16, i16 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -374,6 +527,14 @@ define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xor_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xor i32* @var32, i32 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -394,6 +555,14 @@ define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xor_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_ldeor8_relax
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xor i64* @var64, i64 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -414,6 +583,14 @@ define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xchg_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp1_relax
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xchg i8* @var8, i8 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -433,6 +610,14 @@ define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xchg_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp2_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -453,6 +638,14 @@ define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
|
||||
define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xchg_i32:
|
||||
; CHECK: mov {{[xw]}}8, w[[OLD:[0-9]+]]
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp4_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xchg i32* @var32, i32 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -470,6 +663,14 @@ define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_xchg_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x1, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x1, x1, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_swp8_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw xchg i64* @var64, i64 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -490,6 +691,21 @@ define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_min_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB24_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxrb w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: sxtb w8, w10
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxtb
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, le
|
||||
; OUTLINE_ATOMICS-NEXT: stxrb w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB24_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw min i8* @var8, i8 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -514,6 +730,21 @@ define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_min_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB25_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxrh w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: sxth w8, w10
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxth
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, le
|
||||
; OUTLINE_ATOMICS-NEXT: stlxrh w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB25_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw min i16* @var16, i16 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -539,6 +770,20 @@ define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_min_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB26_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxr w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, le
|
||||
; OUTLINE_ATOMICS-NEXT: stxr w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB26_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw min i32* @var32, i32 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -563,6 +808,20 @@ define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_min_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB27_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxr x8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp x8, x0
|
||||
; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, le
|
||||
; OUTLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB27_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov x0, x8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw min i64* @var64, i64 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -587,6 +846,21 @@ define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_max_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB28_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxrb w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: sxtb w8, w10
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxtb
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, gt
|
||||
; OUTLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB28_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw max i8* @var8, i8 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -612,6 +886,21 @@ define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_max_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB29_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxrh w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: sxth w8, w10
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, sxth
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w10, w0, gt
|
||||
; OUTLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB29_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw max i16* @var16, i16 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -637,6 +926,20 @@ define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_max_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB30_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxr w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, gt
|
||||
; OUTLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB30_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw max i32* @var32, i32 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -661,6 +964,20 @@ define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_max_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB31_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxr x8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp x8, x0
|
||||
; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, gt
|
||||
; OUTLINE_ATOMICS-NEXT: stxr w11, x10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB31_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov x0, x8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw max i64* @var64, i64 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -685,6 +1002,20 @@ define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umin_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB32_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxrb w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxtb
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, ls
|
||||
; OUTLINE_ATOMICS-NEXT: stxrb w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB32_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umin i8* @var8, i8 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -709,6 +1040,20 @@ define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umin_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB33_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxrh w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxth
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, ls
|
||||
; OUTLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB33_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umin i16* @var16, i16 %offset acquire
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -733,6 +1078,20 @@ define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umin_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB34_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxr w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, ls
|
||||
; OUTLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB34_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umin i32* @var32, i32 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -757,6 +1116,20 @@ define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umin_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB35_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxr x8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp x8, x0
|
||||
; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, ls
|
||||
; OUTLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB35_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov x0, x8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umin i64* @var64, i64 %offset acq_rel
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -781,6 +1154,20 @@ define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umax_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB36_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxrb w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxtb
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, hi
|
||||
; OUTLINE_ATOMICS-NEXT: stlxrb w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB36_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umax i8* @var8, i8 %offset acq_rel
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -805,6 +1192,20 @@ define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
|
||||
|
||||
define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umax_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB37_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxrh w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0, uxth
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, hi
|
||||
; OUTLINE_ATOMICS-NEXT: stxrh w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB37_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umax i16* @var16, i16 %offset monotonic
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
|
||||
@ -829,6 +1230,20 @@ define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
|
||||
|
||||
define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umax_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB38_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldaxr w8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp w8, w0
|
||||
; OUTLINE_ATOMICS-NEXT: csel w10, w8, w0, hi
|
||||
; OUTLINE_ATOMICS-NEXT: stlxr w11, w10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB38_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov w0, w8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umax i32* @var32, i32 %offset seq_cst
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
|
||||
@ -853,6 +1268,20 @@ define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
|
||||
|
||||
define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_umax_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x9, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x9, x9, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: .LBB39_1: // %atomicrmw.start
|
||||
; OUTLINE_ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1
|
||||
; OUTLINE_ATOMICS-NEXT: ldxr x8, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cmp x8, x0
|
||||
; OUTLINE_ATOMICS-NEXT: csel x10, x8, x0, hi
|
||||
; OUTLINE_ATOMICS-NEXT: stlxr w11, x10, [x9]
|
||||
; OUTLINE_ATOMICS-NEXT: cbnz w11, .LBB39_1
|
||||
; OUTLINE_ATOMICS-NEXT: // %bb.2: // %atomicrmw.end
|
||||
; OUTLINE_ATOMICS-NEXT: mov x0, x8
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%old = atomicrmw umax i64* @var64, i64 %offset release
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
|
||||
@ -877,6 +1306,14 @@ define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
|
||||
|
||||
define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
|
||||
; CHECK-LABEL: test_atomic_cmpxchg_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x2, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas1_acq
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
|
||||
%old = extractvalue { i8, i1 } %pair, 0
|
||||
|
||||
@ -902,6 +1339,14 @@ define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
|
||||
|
||||
define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
|
||||
; CHECK-LABEL: test_atomic_cmpxchg_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x2, var16
|
||||
; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var16
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas2_acq_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
|
||||
%old = extractvalue { i16, i1 } %pair, 0
|
||||
|
||||
@ -927,6 +1372,14 @@ define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
|
||||
|
||||
define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
|
||||
; CHECK-LABEL: test_atomic_cmpxchg_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x2, var32
|
||||
; OUTLINE_ATOMICS-NEXT: add x2, x2, :lo12:var32
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas4_rel
|
||||
; OUTLINE_ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
|
||||
%old = extractvalue { i32, i1 } %pair, 0
|
||||
|
||||
@ -950,6 +1403,16 @@ define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
|
||||
|
||||
define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
|
||||
; CHECK-LABEL: test_atomic_cmpxchg_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x19, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x19, x19, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: mov x2, x19
|
||||
; OUTLINE_ATOMICS-NEXT: bl __aarch64_cas8_relax
|
||||
; OUTLINE_ATOMICS-NEXT: str x0, [x19]
|
||||
; OUTLINE_ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
|
||||
%old = extractvalue { i64, i1 } %pair, 0
|
||||
|
||||
@ -977,6 +1440,11 @@ define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_monotonic_i8() nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_monotonic_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var8
|
||||
; OUTLINE_ATOMICS-NEXT: ldrb w0, [x8, :lo12:var8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%val = load atomic i8, i8* @var8 monotonic, align 1
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp x[[HIADDR:[0-9]+]], var8
|
||||
@ -988,6 +1456,10 @@ define i8 @test_atomic_load_monotonic_i8() nounwind {
|
||||
|
||||
define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_regoff_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: ldrb w0, [x0, x1]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%addr_int = add i64 %base, %off
|
||||
%addr = inttoptr i64 %addr_int to i8*
|
||||
|
||||
@ -1001,6 +1473,12 @@ define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
|
||||
|
||||
define i8 @test_atomic_load_acquire_i8() nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_acquire_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_acquire_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: ldarb w0, [x8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%val = load atomic i8, i8* @var8 acquire, align 1
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
|
||||
@ -1014,6 +1492,12 @@ define i8 @test_atomic_load_acquire_i8() nounwind {
|
||||
|
||||
define i8 @test_atomic_load_seq_cst_i8() nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_seq_cst_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_seq_cst_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: ldarb w0, [x8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%val = load atomic i8, i8* @var8 seq_cst, align 1
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[HIADDR:x[0-9]+]], var8
|
||||
@ -1027,6 +1511,11 @@ define i8 @test_atomic_load_seq_cst_i8() nounwind {
|
||||
|
||||
define i16 @test_atomic_load_monotonic_i16() nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_monotonic_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var16
|
||||
; OUTLINE_ATOMICS-NEXT: ldrh w0, [x8, :lo12:var16]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%val = load atomic i16, i16* @var16 monotonic, align 2
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp x[[HIADDR:[0-9]+]], var16
|
||||
@ -1039,6 +1528,10 @@ define i16 @test_atomic_load_monotonic_i16() nounwind {
|
||||
|
||||
define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_regoff_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: ldr w0, [x0, x1]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%addr_int = add i64 %base, %off
|
||||
%addr = inttoptr i64 %addr_int to i32*
|
||||
|
||||
@ -1052,6 +1545,12 @@ define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind
|
||||
|
||||
define i64 @test_atomic_load_seq_cst_i64() nounwind {
|
||||
; CHECK-LABEL: test_atomic_load_seq_cst_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_load_seq_cst_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: ldar x0, [x8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%val = load atomic i64, i64* @var64 seq_cst, align 8
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[HIADDR:x[0-9]+]], var64
|
||||
@ -1065,6 +1564,11 @@ define i64 @test_atomic_load_seq_cst_i64() nounwind {
|
||||
|
||||
define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_monotonic_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var8
|
||||
; OUTLINE_ATOMICS-NEXT: strb w0, [x8, :lo12:var8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
store atomic i8 %val, i8* @var8 monotonic, align 1
|
||||
; CHECK: adrp x[[HIADDR:[0-9]+]], var8
|
||||
; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
|
||||
@ -1074,7 +1578,10 @@ define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
|
||||
|
||||
define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
|
||||
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_regoff_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: strb w2, [x0, x1]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%addr_int = add i64 %base, %off
|
||||
%addr = inttoptr i64 %addr_int to i8*
|
||||
|
||||
@ -1085,6 +1592,12 @@ define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val)
|
||||
}
|
||||
define void @test_atomic_store_release_i8(i8 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_release_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_release_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: stlrb w0, [x8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
store atomic i8 %val, i8* @var8 release, align 1
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[HIADDR:x[0-9]+]], var8
|
||||
@ -1098,6 +1611,12 @@ define void @test_atomic_store_release_i8(i8 %val) nounwind {
|
||||
|
||||
define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_seq_cst_i8:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_seq_cst_i8:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var8
|
||||
; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var8
|
||||
; OUTLINE_ATOMICS-NEXT: stlrb w0, [x8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
store atomic i8 %val, i8* @var8 seq_cst, align 1
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[HIADDR:x[0-9]+]], var8
|
||||
@ -1112,6 +1631,11 @@ define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
|
||||
|
||||
define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_monotonic_i16:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_i16:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var16
|
||||
; OUTLINE_ATOMICS-NEXT: strh w0, [x8, :lo12:var16]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
store atomic i16 %val, i16* @var16 monotonic, align 2
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp x[[HIADDR:[0-9]+]], var16
|
||||
@ -1123,7 +1647,10 @@ define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
|
||||
|
||||
define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
|
||||
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_regoff_i32:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: str w2, [x0, x1]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
%addr_int = add i64 %base, %off
|
||||
%addr = inttoptr i64 %addr_int to i32*
|
||||
|
||||
@ -1137,6 +1664,12 @@ define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %va
|
||||
|
||||
define void @test_atomic_store_release_i64(i64 %val) nounwind {
|
||||
; CHECK-LABEL: test_atomic_store_release_i64:
|
||||
; OUTLINE_ATOMICS-LABEL: test_atomic_store_release_i64:
|
||||
; OUTLINE_ATOMICS: // %bb.0:
|
||||
; OUTLINE_ATOMICS-NEXT: adrp x8, var64
|
||||
; OUTLINE_ATOMICS-NEXT: add x8, x8, :lo12:var64
|
||||
; OUTLINE_ATOMICS-NEXT: stlr x0, [x8]
|
||||
; OUTLINE_ATOMICS-NEXT: ret
|
||||
store atomic i64 %val, i64* @var64 release, align 8
|
||||
; CHECK-NOT: dmb
|
||||
; CHECK: adrp [[HIADDR:x[0-9]+]], var64
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
; RUN: llc -verify-machineinstrs -mtriple=aarch64-linux-gnu -O0 -fast-isel=0 -global-isel=false %s -o - | FileCheck -enable-var-scope %s
|
||||
; RUN: llc -verify-machineinstrs -mtriple=aarch64-linux-gnu -O0 -fast-isel=0 -global-isel=false -mattr=+outline-atomics %s -o - | FileCheck -enable-var-scope %s --check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
define { i8, i1 } @test_cmpxchg_8(i8* %addr, i8 %desired, i8 %new) nounwind {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas1_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_8:
|
||||
; CHECK: mov [[ADDR:x[0-9]+]], x0
|
||||
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
|
||||
@ -17,6 +19,7 @@ define { i8, i1 } @test_cmpxchg_8(i8* %addr, i8 %desired, i8 %new) nounwind {
|
||||
}
|
||||
|
||||
define { i16, i1 } @test_cmpxchg_16(i16* %addr, i16 %desired, i16 %new) nounwind {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas2_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_16:
|
||||
; CHECK: mov [[ADDR:x[0-9]+]], x0
|
||||
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
|
||||
@ -33,6 +36,7 @@ define { i16, i1 } @test_cmpxchg_16(i16* %addr, i16 %desired, i16 %new) nounwind
|
||||
}
|
||||
|
||||
define { i32, i1 } @test_cmpxchg_32(i32* %addr, i32 %desired, i32 %new) nounwind {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas4_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_32:
|
||||
; CHECK: mov [[ADDR:x[0-9]+]], x0
|
||||
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
|
||||
@ -49,6 +53,7 @@ define { i32, i1 } @test_cmpxchg_32(i32* %addr, i32 %desired, i32 %new) nounwind
|
||||
}
|
||||
|
||||
define { i64, i1 } @test_cmpxchg_64(i64* %addr, i64 %desired, i64 %new) nounwind {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas8_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_64:
|
||||
; CHECK: mov [[ADDR:x[0-9]+]], x0
|
||||
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
|
||||
@ -65,6 +70,7 @@ define { i64, i1 } @test_cmpxchg_64(i64* %addr, i64 %desired, i64 %new) nounwind
|
||||
}
|
||||
|
||||
define { i128, i1 } @test_cmpxchg_128(i128* %addr, i128 %desired, i128 %new) nounwind {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas16_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_128:
|
||||
; CHECK: mov [[ADDR:x[0-9]+]], x0
|
||||
; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
|
||||
@ -86,6 +92,7 @@ define { i128, i1 } @test_cmpxchg_128(i128* %addr, i128 %desired, i128 %new) nou
|
||||
; was false.
|
||||
@var128 = global i128 0
|
||||
define {i128, i1} @test_cmpxchg_128_unsplit(i128* %addr) {
|
||||
; OUTLINE-ATOMICS: bl __aarch64_cas16_acq_rel
|
||||
; CHECK-LABEL: test_cmpxchg_128_unsplit:
|
||||
; CHECK: mov [[ADDR:x[0-9]+]], x0
|
||||
; CHECK: add x[[VAR128:[0-9]+]], {{x[0-9]+}}, :lo12:var128
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
; RUN: llc -mtriple=aarch64-apple-ios7.0 -o - %s | FileCheck %s
|
||||
; RUN: llc -mtriple=aarch64-apple-ios7.0 -mattr=+outline-atomics -o - %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) {
|
||||
; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel
|
||||
; CHECK-LABEL: test_return:
|
||||
|
||||
; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
|
||||
@ -27,6 +29,7 @@ define i32 @test_return(i32* %p, i32 %oldval, i32 %newval) {
|
||||
}
|
||||
|
||||
define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) {
|
||||
; OUTLINE-ATOMICS: bl ___aarch64_cas1_acq_rel
|
||||
; CHECK-LABEL: test_return_bool:
|
||||
|
||||
; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
|
||||
@ -55,6 +58,7 @@ define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) {
|
||||
}
|
||||
|
||||
define void @test_conditional(i32* %p, i32 %oldval, i32 %newval) {
|
||||
; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel
|
||||
; CHECK-LABEL: test_conditional:
|
||||
|
||||
; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
|
||||
@ -92,6 +96,7 @@ declare void @bar()
|
||||
declare void @baz()
|
||||
|
||||
define i1 @test_conditional2(i32 %a, i32 %b, i32* %c) {
|
||||
; OUTLINE-ATOMICS: bl ___aarch64_cas4_acq_rel
|
||||
; CHECK-LABEL: test_conditional2:
|
||||
; CHECK: [[LOOP:LBB[0-9]+_[0-9]+]]:
|
||||
; CHECK: ldaxr [[LOADED:w[0-9]+]], [x19]
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
; RUN: llc -mtriple arm64-apple-ios -mattr=+lse %s -o - | FileCheck %s
|
||||
; RUN: llc -mtriple arm64-apple-ios -mattr=+lse -mattr=+outline-atomics %s -o - | FileCheck %s
|
||||
|
||||
; Only "even,even+1" pairs are valid for CASP instructions. Make sure LLVM
|
||||
; doesn't allocate odd ones and that it can copy them around properly. N.b. we
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
||||
; RUN: opt -S -mtriple=aarch64-- -atomic-expand %s | FileCheck %s
|
||||
; RUN: opt -S -mtriple=aarch64-- -mattr=+outline-atomics -atomic-expand %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS
|
||||
|
||||
define void @atomic_swap_f16(half* %ptr, half %val) nounwind {
|
||||
; CHECK-LABEL: @atomic_swap_f16(
|
||||
@ -15,6 +16,10 @@ define void @atomic_swap_f16(half* %ptr, half %val) nounwind {
|
||||
; CHECK-NEXT: br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
|
||||
; CHECK: atomicrmw.end:
|
||||
; CHECK-NEXT: ret void
|
||||
;
|
||||
; OUTLINE-ATOMICS-LABEL: @atomic_swap_f16(
|
||||
; OUTLINE-ATOMICS-NEXT: [[T1:%.*]] = atomicrmw xchg half* [[PTR:%.*]], half [[VAL:%.*]] acquire
|
||||
; OUTLINE-ATOMICS-NEXT: ret void
|
||||
;
|
||||
%t1 = atomicrmw xchg half* %ptr, half %val acquire
|
||||
ret void
|
||||
@ -34,6 +39,10 @@ define void @atomic_swap_f32(float* %ptr, float %val) nounwind {
|
||||
; CHECK-NEXT: br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
|
||||
; CHECK: atomicrmw.end:
|
||||
; CHECK-NEXT: ret void
|
||||
;
|
||||
; OUTLINE-ATOMICS-LABEL: @atomic_swap_f32(
|
||||
; OUTLINE-ATOMICS-NEXT: [[T1:%.*]] = atomicrmw xchg float* [[PTR:%.*]], float [[VAL:%.*]] acquire
|
||||
; OUTLINE-ATOMICS-NEXT: ret void
|
||||
;
|
||||
%t1 = atomicrmw xchg float* %ptr, float %val acquire
|
||||
ret void
|
||||
@ -51,6 +60,10 @@ define void @atomic_swap_f64(double* %ptr, double %val) nounwind {
|
||||
; CHECK-NEXT: br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
|
||||
; CHECK: atomicrmw.end:
|
||||
; CHECK-NEXT: ret void
|
||||
;
|
||||
; OUTLINE-ATOMICS-LABEL: @atomic_swap_f64(
|
||||
; OUTLINE-ATOMICS-NEXT: [[T1:%.*]] = atomicrmw xchg double* [[PTR:%.*]], double [[VAL:%.*]] acquire
|
||||
; OUTLINE-ATOMICS-NEXT: ret void
|
||||
;
|
||||
%t1 = atomicrmw xchg double* %ptr, double %val acquire
|
||||
ret void
|
||||
|
||||
Loading…
Reference in New Issue
Block a user