From a9e95965678dd33d17ab27751b3ca0d95ef73ccf Mon Sep 17 00:00:00 2001 From: Eli Friedman Date: Wed, 21 Jul 2021 11:21:21 -0700 Subject: [PATCH] [AArch64] Regenerate and add more tests for i128 atomics. Generating these tests unfortunately means a lot of junk, but it's hard to write/update these tests by hand. Added tests focus on atomic orderings for cmpxchg. Actually writing out these tests showed some potentially dubious results; we should probably consider using casp for 128-bit atomic load/store/rmw. --- .../AArch64/GlobalISel/arm64-atomic-128.ll | 534 +++++++++++++++--- test/CodeGen/AArch64/arm64-atomic-128.ll | 500 +++++++++++----- 2 files changed, 839 insertions(+), 195 deletions(-) diff --git a/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll b/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll index da82f2447ef..b5b49e86f15 100644 --- a/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll +++ b/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=CHECK-LLSC-O1 ; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=apple-a13 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=CHECK-CAS-O1 ; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -O0 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefix=CHECK-LLSC-O0 @@ -6,44 +7,82 @@ define void @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { ; CHECK-LLSC-O1-LABEL: val_compare_and_swap: -; CHECK-LLSC-O1: ldaxp {{x[0-9]+}}, {{x[0-9]+}}, [x0] -; [... LOTS of stuff that is generic IR unrelated to atomic operations ...] -; CHECK-LLSC-O1: stxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0] +; CHECK-LLSC-O1: // %bb.0: +; CHECK-LLSC-O1-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cmp x8, x2 +; CHECK-LLSC-O1-NEXT: cset w10, ne +; CHECK-LLSC-O1-NEXT: cmp x9, x3 +; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O1-NEXT: cbz w10, .LBB0_3 +; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stxp w10, x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB0_1 +; CHECK-LLSC-O1-NEXT: b .LBB0_4 +; CHECK-LLSC-O1-NEXT: .LBB0_3: // in Loop: Header=BB0_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stxp w10, x4, x5, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB0_1 +; CHECK-LLSC-O1-NEXT: .LBB0_4: +; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8 +; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9 +; CHECK-LLSC-O1-NEXT: str q0, [x0] +; CHECK-LLSC-O1-NEXT: ret ; ; CHECK-CAS-O1-LABEL: val_compare_and_swap: -; CHECK-CAS-O1: caspa x2, x3, x4, x5, [x0] -; CHECK-CAS-O1: mov v[[OLD:[0-9]+]].d[0], x2 -; CHECK-CAS-O1: mov v[[OLD]].d[1], x3 -; CHECK-CAS-O1: str q[[OLD]], [x0] - +; CHECK-CAS-O1: // %bb.0: +; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: caspa x2, x3, x4, x5, [x0] +; CHECK-CAS-O1-NEXT: mov v0.d[0], x2 +; CHECK-CAS-O1-NEXT: mov v0.d[1], x3 +; CHECK-CAS-O1-NEXT: str q0, [x0] +; CHECK-CAS-O1-NEXT: ret +; ; CHECK-LLSC-O0-LABEL: val_compare_and_swap: -; CHECK-LLSC-O0: .LBB0_1: -; CHECK-LLSC-O0: ldaxp [[OLD_LO:x[0-9]+]], [[OLD_HI:x[0-9]+]], [x0] -; CHECK-LLSC-O0: cmp [[OLD_LO]], x2 -; CHECK-LLSC-O0: cset [[EQUAL_TMP:w[0-9]+]], ne -; CHECK-LLSC-O0: cmp [[OLD_HI]], x3 -; CHECK-LLSC-O0: cinc [[EQUAL:w[0-9]+]], [[EQUAL_TMP]], ne -; CHECK-LLSC-O0: cbnz [[EQUAL]], .LBB0_3 -; CHECK-LLSC-O0: stxp [[STATUS:w[0-9]+]], x4, x5, [x0] -; CHECK-LLSC-O0: cbnz [[STATUS]], .LBB0_1 -; CHECK-LLSC-O0: .LBB0_3: -; CHECK-LLSC-O0: mov v[[OLD:[0-9]+]].d[0], [[OLD_LO]] -; CHECK-LLSC-O0: mov v[[OLD]].d[1], [[OLD_HI]] -; CHECK-LLSC-O0: str q[[OLD]], [x0] - - +; CHECK-LLSC-O0: // %bb.0: +; CHECK-LLSC-O0-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cmp x9, x2 +; CHECK-LLSC-O0-NEXT: cset w10, ne +; CHECK-LLSC-O0-NEXT: cmp x8, x3 +; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB0_3 +; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stxp w10, x4, x5, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB0_1 +; CHECK-LLSC-O0-NEXT: b .LBB0_4 +; CHECK-LLSC-O0-NEXT: .LBB0_3: // in Loop: Header=BB0_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stxp w10, x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB0_1 +; CHECK-LLSC-O0-NEXT: .LBB0_4: +; CHECK-LLSC-O0-NEXT: // implicit-def: $q0 +; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9 +; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8 +; CHECK-LLSC-O0-NEXT: str q0, [x0] +; CHECK-LLSC-O0-NEXT: ret +; ; CHECK-CAS-O0-LABEL: val_compare_and_swap: -; CHECK-CAS-O0: str x3, [sp, #[[SLOT:[0-9]+]]] -; CHECK-CAS-O0: mov [[NEW_HI_TMP:x[0-9]+]], x5 -; CHECK-CAS-O0: ldr [[DESIRED_HI_TMP:x[0-9]+]], [sp, #[[SLOT]]] -; CHECK-CAS-O0: mov [[DESIRED_HI:x[0-9]+]], [[DESIRED_HI_TMP]] -; CHECK-CAS-O0: mov [[NEW_HI:x[0-9]+]], [[NEW_HI_TMP]] -; CHECK-CAS-O0: caspa x2, [[DESIRED_HI]], x4, [[NEW_HI]], [x0] -; CHECK-CAS-O0: mov [[OLD_LO:x[0-9]+]], x2 -; CHECK-CAS-O0: mov [[OLD_HI:x[0-9]+]], x3 -; CHECK-CAS-O0: mov v[[OLD:[0-9]+]].d[0], [[OLD_LO]] -; CHECK-CAS-O0: mov v[[OLD]].d[1], [[OLD_HI]] -; CHECK-CAS-O0: str q[[OLD]], [x0] +; CHECK-CAS-O0: // %bb.0: +; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16 +; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: mov x1, x5 +; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload +; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3 +; CHECK-CAS-O0-NEXT: mov x3, x5 +; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5 +; CHECK-CAS-O0-NEXT: mov x5, x1 +; CHECK-CAS-O0-NEXT: caspa x2, x3, x4, x5, [x0] +; CHECK-CAS-O0-NEXT: mov x9, x2 +; CHECK-CAS-O0-NEXT: mov x8, x3 +; CHECK-CAS-O0-NEXT: // implicit-def: $q0 +; CHECK-CAS-O0-NEXT: mov v0.d[0], x9 +; CHECK-CAS-O0-NEXT: mov v0.d[1], x8 +; CHECK-CAS-O0-NEXT: str q0, [x0] +; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: ret %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire %val = extractvalue { i128, i1 } %pair, 0 @@ -53,20 +92,82 @@ define void @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { define void @val_compare_and_swap_monotonic_seqcst(i128* %p, i128 %oldval, i128 %newval) { ; CHECK-LLSC-O1-LABEL: val_compare_and_swap_monotonic_seqcst: -; CHECK-LLSC-O1: ldaxp {{x[0-9]+}}, {{x[0-9]+}}, [x0] -; [... LOTS of stuff that is generic IR unrelated to atomic operations ...] -; CHECK-LLSC-O1: stlxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0] +; CHECK-LLSC-O1: // %bb.0: +; CHECK-LLSC-O1-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cmp x8, x2 +; CHECK-LLSC-O1-NEXT: cset w10, ne +; CHECK-LLSC-O1-NEXT: cmp x9, x3 +; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O1-NEXT: cbz w10, .LBB1_3 +; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stlxp w10, x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB1_1 +; CHECK-LLSC-O1-NEXT: b .LBB1_4 +; CHECK-LLSC-O1-NEXT: .LBB1_3: // in Loop: Header=BB1_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB1_1 +; CHECK-LLSC-O1-NEXT: .LBB1_4: +; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8 +; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9 +; CHECK-LLSC-O1-NEXT: str q0, [x0] +; CHECK-LLSC-O1-NEXT: ret ; ; CHECK-CAS-O1-LABEL: val_compare_and_swap_monotonic_seqcst: -; CHECK-CAS-O1: caspal x2, x3, x4, x5, [x0] - +; CHECK-CAS-O1: // %bb.0: +; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0] +; CHECK-CAS-O1-NEXT: mov v0.d[0], x2 +; CHECK-CAS-O1-NEXT: mov v0.d[1], x3 +; CHECK-CAS-O1-NEXT: str q0, [x0] +; CHECK-CAS-O1-NEXT: ret +; ; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic_seqcst: -; CHECK-LLSC-O0: .LBB1_1: -; CHECK-LLSC-O0: ldaxp -; CHECK-LLSC-O0: stlxp - +; CHECK-LLSC-O0: // %bb.0: +; CHECK-LLSC-O0-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cmp x9, x2 +; CHECK-LLSC-O0-NEXT: cset w10, ne +; CHECK-LLSC-O0-NEXT: cmp x8, x3 +; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB1_3 +; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stlxp w10, x4, x5, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB1_1 +; CHECK-LLSC-O0-NEXT: b .LBB1_4 +; CHECK-LLSC-O0-NEXT: .LBB1_3: // in Loop: Header=BB1_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stlxp w10, x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB1_1 +; CHECK-LLSC-O0-NEXT: .LBB1_4: +; CHECK-LLSC-O0-NEXT: // implicit-def: $q0 +; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9 +; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8 +; CHECK-LLSC-O0-NEXT: str q0, [x0] +; CHECK-LLSC-O0-NEXT: ret +; ; CHECK-CAS-O0-LABEL: val_compare_and_swap_monotonic_seqcst: -; CHECK-CAS-O0: caspal +; CHECK-CAS-O0: // %bb.0: +; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16 +; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: mov x1, x5 +; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload +; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3 +; CHECK-CAS-O0-NEXT: mov x3, x5 +; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5 +; CHECK-CAS-O0-NEXT: mov x5, x1 +; CHECK-CAS-O0-NEXT: caspal x2, x3, x4, x5, [x0] +; CHECK-CAS-O0-NEXT: mov x9, x2 +; CHECK-CAS-O0-NEXT: mov x8, x3 +; CHECK-CAS-O0-NEXT: // implicit-def: $q0 +; CHECK-CAS-O0-NEXT: mov v0.d[0], x9 +; CHECK-CAS-O0-NEXT: mov v0.d[1], x8 +; CHECK-CAS-O0-NEXT: str q0, [x0] +; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: ret %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval monotonic seq_cst %val = extractvalue { i128, i1 } %pair, 0 @@ -76,20 +177,82 @@ define void @val_compare_and_swap_monotonic_seqcst(i128* %p, i128 %oldval, i128 define void @val_compare_and_swap_release_acquire(i128* %p, i128 %oldval, i128 %newval) { ; CHECK-LLSC-O1-LABEL: val_compare_and_swap_release_acquire: -; CHECK-LLSC-O1: ldaxp {{x[0-9]+}}, {{x[0-9]+}}, [x0] -; [... LOTS of stuff that is generic IR unrelated to atomic operations ...] -; CHECK-LLSC-O1: stlxp {{w[0-9]+}}, {{x[0-9]+}}, {{x[0-9]+}}, [x0] +; CHECK-LLSC-O1: // %bb.0: +; CHECK-LLSC-O1-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cmp x8, x2 +; CHECK-LLSC-O1-NEXT: cset w10, ne +; CHECK-LLSC-O1-NEXT: cmp x9, x3 +; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O1-NEXT: cbz w10, .LBB2_3 +; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stlxp w10, x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB2_1 +; CHECK-LLSC-O1-NEXT: b .LBB2_4 +; CHECK-LLSC-O1-NEXT: .LBB2_3: // in Loop: Header=BB2_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB2_1 +; CHECK-LLSC-O1-NEXT: .LBB2_4: +; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8 +; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9 +; CHECK-LLSC-O1-NEXT: str q0, [x0] +; CHECK-LLSC-O1-NEXT: ret ; ; CHECK-CAS-O1-LABEL: val_compare_and_swap_release_acquire: -; CHECK-CAS-O1: caspal x2, x3, x4, x5, [x0] - +; CHECK-CAS-O1: // %bb.0: +; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0] +; CHECK-CAS-O1-NEXT: mov v0.d[0], x2 +; CHECK-CAS-O1-NEXT: mov v0.d[1], x3 +; CHECK-CAS-O1-NEXT: str q0, [x0] +; CHECK-CAS-O1-NEXT: ret +; ; CHECK-LLSC-O0-LABEL: val_compare_and_swap_release_acquire: -; CHECK-LLSC-O0: .LBB2_1: -; CHECK-LLSC-O0: ldaxp -; CHECK-LLSC-O0: stlxp - +; CHECK-LLSC-O0: // %bb.0: +; CHECK-LLSC-O0-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cmp x9, x2 +; CHECK-LLSC-O0-NEXT: cset w10, ne +; CHECK-LLSC-O0-NEXT: cmp x8, x3 +; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB2_3 +; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stlxp w10, x4, x5, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB2_1 +; CHECK-LLSC-O0-NEXT: b .LBB2_4 +; CHECK-LLSC-O0-NEXT: .LBB2_3: // in Loop: Header=BB2_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stlxp w10, x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB2_1 +; CHECK-LLSC-O0-NEXT: .LBB2_4: +; CHECK-LLSC-O0-NEXT: // implicit-def: $q0 +; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9 +; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8 +; CHECK-LLSC-O0-NEXT: str q0, [x0] +; CHECK-LLSC-O0-NEXT: ret +; ; CHECK-CAS-O0-LABEL: val_compare_and_swap_release_acquire: -; CHECK-CAS-O0: caspal +; CHECK-CAS-O0: // %bb.0: +; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16 +; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: mov x1, x5 +; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload +; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3 +; CHECK-CAS-O0-NEXT: mov x3, x5 +; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5 +; CHECK-CAS-O0-NEXT: mov x5, x1 +; CHECK-CAS-O0-NEXT: caspal x2, x3, x4, x5, [x0] +; CHECK-CAS-O0-NEXT: mov x9, x2 +; CHECK-CAS-O0-NEXT: mov x8, x3 +; CHECK-CAS-O0-NEXT: // implicit-def: $q0 +; CHECK-CAS-O0-NEXT: mov v0.d[0], x9 +; CHECK-CAS-O0-NEXT: mov v0.d[1], x8 +; CHECK-CAS-O0-NEXT: str q0, [x0] +; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: ret %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire %val = extractvalue { i128, i1 } %pair, 0 @@ -97,22 +260,257 @@ define void @val_compare_and_swap_release_acquire(i128* %p, i128 %oldval, i128 % ret void } +define void @val_compare_and_swap_monotonic(i128* %p, i128 %oldval, i128 %newval) { +; CHECK-LLSC-O1-LABEL: val_compare_and_swap_monotonic: +; CHECK-LLSC-O1: // %bb.0: +; CHECK-LLSC-O1-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O1-NEXT: ldaxp x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cmp x8, x2 +; CHECK-LLSC-O1-NEXT: cset w10, ne +; CHECK-LLSC-O1-NEXT: cmp x9, x3 +; CHECK-LLSC-O1-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O1-NEXT: cbz w10, .LBB3_3 +; CHECK-LLSC-O1-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stlxp w10, x8, x9, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB3_1 +; CHECK-LLSC-O1-NEXT: b .LBB3_4 +; CHECK-LLSC-O1-NEXT: .LBB3_3: // in Loop: Header=BB3_1 Depth=1 +; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0] +; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB3_1 +; CHECK-LLSC-O1-NEXT: .LBB3_4: +; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8 +; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9 +; CHECK-LLSC-O1-NEXT: str q0, [x0] +; CHECK-LLSC-O1-NEXT: ret +; +; CHECK-CAS-O1-LABEL: val_compare_and_swap_monotonic: +; CHECK-CAS-O1: // %bb.0: +; CHECK-CAS-O1-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0] +; CHECK-CAS-O1-NEXT: mov v0.d[0], x2 +; CHECK-CAS-O1-NEXT: mov v0.d[1], x3 +; CHECK-CAS-O1-NEXT: str q0, [x0] +; CHECK-CAS-O1-NEXT: ret +; +; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic: +; CHECK-LLSC-O0: // %bb.0: +; CHECK-LLSC-O0-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O0-NEXT: ldaxp x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cmp x9, x2 +; CHECK-LLSC-O0-NEXT: cset w10, ne +; CHECK-LLSC-O0-NEXT: cmp x8, x3 +; CHECK-LLSC-O0-NEXT: cinc w10, w10, ne +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB3_3 +; CHECK-LLSC-O0-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stlxp w10, x4, x5, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB3_1 +; CHECK-LLSC-O0-NEXT: b .LBB3_4 +; CHECK-LLSC-O0-NEXT: .LBB3_3: // in Loop: Header=BB3_1 Depth=1 +; CHECK-LLSC-O0-NEXT: stlxp w10, x9, x8, [x0] +; CHECK-LLSC-O0-NEXT: cbnz w10, .LBB3_1 +; CHECK-LLSC-O0-NEXT: .LBB3_4: +; CHECK-LLSC-O0-NEXT: // implicit-def: $q0 +; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9 +; CHECK-LLSC-O0-NEXT: mov v0.d[1], x8 +; CHECK-LLSC-O0-NEXT: str q0, [x0] +; CHECK-LLSC-O0-NEXT: ret +; +; CHECK-CAS-O0-LABEL: val_compare_and_swap_monotonic: +; CHECK-CAS-O0: // %bb.0: +; CHECK-CAS-O0-NEXT: sub sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 16 +; CHECK-CAS-O0-NEXT: str x3, [sp, #8] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: mov x1, x5 +; CHECK-CAS-O0-NEXT: ldr x5, [sp, #8] // 8-byte Folded Reload +; CHECK-CAS-O0-NEXT: // kill: def $x2 killed $x2 def $x2_x3 +; CHECK-CAS-O0-NEXT: mov x3, x5 +; CHECK-CAS-O0-NEXT: // kill: def $x4 killed $x4 def $x4_x5 +; CHECK-CAS-O0-NEXT: mov x5, x1 +; CHECK-CAS-O0-NEXT: caspal x2, x3, x4, x5, [x0] +; CHECK-CAS-O0-NEXT: mov x9, x2 +; CHECK-CAS-O0-NEXT: mov x8, x3 +; CHECK-CAS-O0-NEXT: // implicit-def: $q0 +; CHECK-CAS-O0-NEXT: mov v0.d[0], x9 +; CHECK-CAS-O0-NEXT: mov v0.d[1], x8 +; CHECK-CAS-O0-NEXT: str q0, [x0] +; CHECK-CAS-O0-NEXT: add sp, sp, #16 // =16 +; CHECK-CAS-O0-NEXT: ret + %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release acquire + %val = extractvalue { i128, i1 } %pair, 0 + store i128 %val, i128* %p + ret void +} + define void @atomic_load_relaxed(i64, i64, i128* %p, i128* %p2) { -; CHECK-LLSC-O1-LABEL: atomic_load_relaxed -; CHECK-LLSC-O1: ldxp -; CHECK-LLSC-O1: stxp +; CHECK-LLSC-O1-LABEL: atomic_load_relaxed: +; CHECK-LLSC-O1: // %bb.0: +; CHECK-LLSC-O1-NEXT: mov w8, #64 +; CHECK-LLSC-O1-NEXT: sub x9, x8, #64 // =64 +; CHECK-LLSC-O1-NEXT: .LBB4_1: // %atomicrmw.start +; CHECK-LLSC-O1-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O1-NEXT: ldxp x11, x10, [x2] +; CHECK-LLSC-O1-NEXT: sub x12, x8, #64 // =64 +; CHECK-LLSC-O1-NEXT: tst wzr, #0x1 +; CHECK-LLSC-O1-NEXT: lsl x13, x10, x8 +; CHECK-LLSC-O1-NEXT: lsr x14, x10, x9 +; CHECK-LLSC-O1-NEXT: lsl x10, x10, x12 +; CHECK-LLSC-O1-NEXT: csel x10, x14, x10, ne +; CHECK-LLSC-O1-NEXT: csel x13, x13, xzr, ne +; CHECK-LLSC-O1-NEXT: csel x10, xzr, x10, ne +; CHECK-LLSC-O1-NEXT: orr x11, x11, x13 +; CHECK-LLSC-O1-NEXT: lsl x13, x10, x9 +; CHECK-LLSC-O1-NEXT: lsr x12, x10, x12 +; CHECK-LLSC-O1-NEXT: orr x13, x13, x11, lsr #0 +; CHECK-LLSC-O1-NEXT: tst wzr, #0x1 +; CHECK-LLSC-O1-NEXT: csel x12, x13, x12, ne +; CHECK-LLSC-O1-NEXT: csel x12, x11, x12, ne +; CHECK-LLSC-O1-NEXT: stxp w13, x11, x12, [x2] +; CHECK-LLSC-O1-NEXT: cbnz w13, .LBB4_1 +; CHECK-LLSC-O1-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-LLSC-O1-NEXT: mov v0.d[0], x11 +; CHECK-LLSC-O1-NEXT: mov v0.d[1], x10 +; CHECK-LLSC-O1-NEXT: str q0, [x3] +; CHECK-LLSC-O1-NEXT: ret +; +; CHECK-CAS-O1-LABEL: atomic_load_relaxed: +; CHECK-CAS-O1: // %bb.0: +; CHECK-CAS-O1-NEXT: mov w8, #64 +; CHECK-CAS-O1-NEXT: sub x9, x8, #64 // =64 +; CHECK-CAS-O1-NEXT: .LBB4_1: // %atomicrmw.start +; CHECK-CAS-O1-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-CAS-O1-NEXT: ldxp x11, x10, [x2] +; CHECK-CAS-O1-NEXT: sub x12, x8, #64 // =64 +; CHECK-CAS-O1-NEXT: lsl x13, x10, x8 +; CHECK-CAS-O1-NEXT: lsr x14, x10, x9 +; CHECK-CAS-O1-NEXT: lsl x10, x10, x12 +; CHECK-CAS-O1-NEXT: tst wzr, #0x1 +; CHECK-CAS-O1-NEXT: csel x13, x13, xzr, ne +; CHECK-CAS-O1-NEXT: csel x10, x14, x10, ne +; CHECK-CAS-O1-NEXT: csel x10, xzr, x10, ne +; CHECK-CAS-O1-NEXT: orr x11, x11, x13 +; CHECK-CAS-O1-NEXT: lsl x13, x10, x9 +; CHECK-CAS-O1-NEXT: orr x13, x13, x11, lsr #0 +; CHECK-CAS-O1-NEXT: lsr x12, x10, x12 +; CHECK-CAS-O1-NEXT: tst wzr, #0x1 +; CHECK-CAS-O1-NEXT: csel x12, x13, x12, ne +; CHECK-CAS-O1-NEXT: csel x12, x11, x12, ne +; CHECK-CAS-O1-NEXT: stxp w13, x11, x12, [x2] +; CHECK-CAS-O1-NEXT: cbnz w13, .LBB4_1 +; CHECK-CAS-O1-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-CAS-O1-NEXT: mov v0.d[0], x11 +; CHECK-CAS-O1-NEXT: mov v0.d[1], x10 +; CHECK-CAS-O1-NEXT: str q0, [x3] +; CHECK-CAS-O1-NEXT: ret +; +; CHECK-LLSC-O0-LABEL: atomic_load_relaxed: +; CHECK-LLSC-O0: // %bb.0: +; CHECK-LLSC-O0-NEXT: sub sp, sp, #64 // =64 +; CHECK-LLSC-O0-NEXT: .cfi_def_cfa_offset 64 +; CHECK-LLSC-O0-NEXT: str x2, [sp, #48] // 8-byte Folded Spill +; CHECK-LLSC-O0-NEXT: str x3, [sp, #56] // 8-byte Folded Spill +; CHECK-LLSC-O0-NEXT: .LBB4_1: // %atomicrmw.start +; CHECK-LLSC-O0-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-LLSC-O0-NEXT: ldr x11, [sp, #48] // 8-byte Folded Reload +; CHECK-LLSC-O0-NEXT: ldxp x9, x15, [x11] +; CHECK-LLSC-O0-NEXT: mov x12, xzr +; CHECK-LLSC-O0-NEXT: mov w8, #64 +; CHECK-LLSC-O0-NEXT: // kill: def $x8 killed $w8 +; CHECK-LLSC-O0-NEXT: mov w10, #64 +; CHECK-LLSC-O0-NEXT: // kill: def $x10 killed $w10 +; CHECK-LLSC-O0-NEXT: str x10, [sp, #8] // 8-byte Folded Spill +; CHECK-LLSC-O0-NEXT: subs x16, x10, #64 // =64 +; CHECK-LLSC-O0-NEXT: subs x13, x8, #64 // =64 +; CHECK-LLSC-O0-NEXT: lsl x14, x15, x10 +; CHECK-LLSC-O0-NEXT: lsr x13, x15, x13 +; CHECK-LLSC-O0-NEXT: orr x13, x13, x12 +; CHECK-LLSC-O0-NEXT: lsl x15, x15, x16 +; CHECK-LLSC-O0-NEXT: subs x16, x10, #64 // =64 +; CHECK-LLSC-O0-NEXT: csel x14, x14, x12, lo +; CHECK-LLSC-O0-NEXT: subs x16, x10, #64 // =64 +; CHECK-LLSC-O0-NEXT: csel x13, x13, x15, lo +; CHECK-LLSC-O0-NEXT: subs x15, x10, #0 // =0 +; CHECK-LLSC-O0-NEXT: csel x13, x12, x13, eq +; CHECK-LLSC-O0-NEXT: orr x9, x9, x14 +; CHECK-LLSC-O0-NEXT: orr x12, x12, x13 +; CHECK-LLSC-O0-NEXT: // implicit-def: $q0 +; CHECK-LLSC-O0-NEXT: mov v0.d[0], x9 +; CHECK-LLSC-O0-NEXT: str q0, [sp, #16] // 16-byte Folded Spill +; CHECK-LLSC-O0-NEXT: mov v0.d[1], x12 +; CHECK-LLSC-O0-NEXT: str q0, [sp, #32] // 16-byte Folded Spill +; CHECK-LLSC-O0-NEXT: subs x13, x10, #64 // =64 +; CHECK-LLSC-O0-NEXT: subs x8, x8, #64 // =64 +; CHECK-LLSC-O0-NEXT: lsl x8, x12, x8 +; CHECK-LLSC-O0-NEXT: orr x8, x8, x9, lsr #0 +; CHECK-LLSC-O0-NEXT: lsr x12, x12, x13 +; CHECK-LLSC-O0-NEXT: subs x13, x10, #64 // =64 +; CHECK-LLSC-O0-NEXT: csel x8, x8, x12, lo +; CHECK-LLSC-O0-NEXT: subs x10, x10, #0 // =0 +; CHECK-LLSC-O0-NEXT: csel x10, x9, x8, eq +; CHECK-LLSC-O0-NEXT: stxp w8, x9, x10, [x11] +; CHECK-LLSC-O0-NEXT: cbnz w8, .LBB4_1 +; CHECK-LLSC-O0-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-LLSC-O0-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload +; CHECK-LLSC-O0-NEXT: ldr x8, [sp, #56] // 8-byte Folded Reload +; CHECK-LLSC-O0-NEXT: str q0, [x8] +; CHECK-LLSC-O0-NEXT: add sp, sp, #64 // =64 +; CHECK-LLSC-O0-NEXT: ret +; +; CHECK-CAS-O0-LABEL: atomic_load_relaxed: +; CHECK-CAS-O0: // %bb.0: +; CHECK-CAS-O0-NEXT: sub sp, sp, #64 // =64 +; CHECK-CAS-O0-NEXT: .cfi_def_cfa_offset 64 +; CHECK-CAS-O0-NEXT: str x2, [sp, #48] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: str x3, [sp, #56] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: .LBB4_1: // %atomicrmw.start +; CHECK-CAS-O0-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-CAS-O0-NEXT: ldr x11, [sp, #48] // 8-byte Folded Reload +; CHECK-CAS-O0-NEXT: ldxp x9, x15, [x11] +; CHECK-CAS-O0-NEXT: mov x12, #0 +; CHECK-CAS-O0-NEXT: mov w8, #64 +; CHECK-CAS-O0-NEXT: // kill: def $x8 killed $w8 +; CHECK-CAS-O0-NEXT: mov w10, #64 +; CHECK-CAS-O0-NEXT: // kill: def $x10 killed $w10 +; CHECK-CAS-O0-NEXT: str x10, [sp, #8] // 8-byte Folded Spill +; CHECK-CAS-O0-NEXT: subs x16, x10, #64 // =64 +; CHECK-CAS-O0-NEXT: subs x13, x8, #64 // =64 +; CHECK-CAS-O0-NEXT: lsl x14, x15, x10 +; CHECK-CAS-O0-NEXT: lsr x13, x15, x13 +; CHECK-CAS-O0-NEXT: orr x13, x13, x12 +; CHECK-CAS-O0-NEXT: lsl x15, x15, x16 +; CHECK-CAS-O0-NEXT: subs x16, x10, #64 // =64 +; CHECK-CAS-O0-NEXT: csel x14, x14, x12, lo +; CHECK-CAS-O0-NEXT: subs x16, x10, #64 // =64 +; CHECK-CAS-O0-NEXT: csel x13, x13, x15, lo +; CHECK-CAS-O0-NEXT: subs x15, x10, #0 // =0 +; CHECK-CAS-O0-NEXT: csel x13, x12, x13, eq +; CHECK-CAS-O0-NEXT: orr x9, x9, x14 +; CHECK-CAS-O0-NEXT: orr x12, x12, x13 +; CHECK-CAS-O0-NEXT: // implicit-def: $q0 +; CHECK-CAS-O0-NEXT: mov v0.d[0], x9 +; CHECK-CAS-O0-NEXT: str q0, [sp, #16] // 16-byte Folded Spill +; CHECK-CAS-O0-NEXT: mov v0.d[1], x12 +; CHECK-CAS-O0-NEXT: str q0, [sp, #32] // 16-byte Folded Spill +; CHECK-CAS-O0-NEXT: subs x13, x10, #64 // =64 +; CHECK-CAS-O0-NEXT: subs x8, x8, #64 // =64 +; CHECK-CAS-O0-NEXT: lsl x8, x12, x8 +; CHECK-CAS-O0-NEXT: orr x8, x8, x9, lsr #0 +; CHECK-CAS-O0-NEXT: lsr x12, x12, x13 +; CHECK-CAS-O0-NEXT: subs x13, x10, #64 // =64 +; CHECK-CAS-O0-NEXT: csel x8, x8, x12, lo +; CHECK-CAS-O0-NEXT: subs x10, x10, #0 // =0 +; CHECK-CAS-O0-NEXT: csel x10, x9, x8, eq +; CHECK-CAS-O0-NEXT: stxp w8, x9, x10, [x11] +; CHECK-CAS-O0-NEXT: cbnz w8, .LBB4_1 +; CHECK-CAS-O0-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-CAS-O0-NEXT: ldr q0, [sp, #32] // 16-byte Folded Reload +; CHECK-CAS-O0-NEXT: ldr x8, [sp, #56] // 8-byte Folded Reload +; CHECK-CAS-O0-NEXT: str q0, [x8] +; CHECK-CAS-O0-NEXT: add sp, sp, #64 // =64 +; CHECK-CAS-O0-NEXT: ret -; CHECK-LLSC-O0-LABEL: atomic_load_relaxed -; CHECK-LLSC-O0: ldxp -; CHECK-LLSC-O0: stxp - -; CHECK-CAS-O1-LABEL: atomic_load_relaxed -; CHECK-CAS-O1: ldxp -; CHECK-CAS-O1: stxp - -; CHECK-CAS-O0-LABEL: atomic_load_relaxed -; CHECK-CAS-O0: ldxp -; CHECK-CAS-O0: stxp %r = load atomic i128, i128* %p monotonic, align 16 store i128 %r, i128* %p2 ret void diff --git a/test/CodeGen/AArch64/arm64-atomic-128.ll b/test/CodeGen/AArch64/arm64-atomic-128.ll index e78a6af63c2..89ae80a7100 100644 --- a/test/CodeGen/AArch64/arm64-atomic-128.ll +++ b/test/CodeGen/AArch64/arm64-atomic-128.ll @@ -1,42 +1,232 @@ -; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s -; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefix=OUTLINE-ATOMICS +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s -check-prefixes=CHECK,NOOUTLINE +; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefixes=CHECK,OUTLINE +; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+lse | FileCheck %s -check-prefixes=CHECK,LSE @var = global i128 0 define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { -; OUTLINE-ATOMICS: bl __aarch64_cas16_acq -; CHECK-LABEL: val_compare_and_swap: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]] -; CHECK: cmp -; CHECK: cset -; CHECK: cmp -; CHECK: cinc [[MISMATCH:w[0-9]+]] -; CHECK: cbz [[MISMATCH]], [[EQUAL:.LBB[0-9]+_[0-9]+]] -; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], [[RESULTLO]], [[RESULTHI]], [x[[ADDR]]] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] -; CHECK: b [[DONE:.LBB[0-9]+_[0-9]+]] -; CHECK: [[EQUAL]]: -; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] -; CHECK: [[DONE]]: +; NOOUTLINE-LABEL: val_compare_and_swap: +; NOOUTLINE: // %bb.0: +; NOOUTLINE-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1 +; NOOUTLINE-NEXT: ldaxp x8, x1, [x0] +; NOOUTLINE-NEXT: cmp x8, x2 +; NOOUTLINE-NEXT: cset w9, ne +; NOOUTLINE-NEXT: cmp x1, x3 +; NOOUTLINE-NEXT: cinc w9, w9, ne +; NOOUTLINE-NEXT: cbz w9, .LBB0_3 +; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1 +; NOOUTLINE-NEXT: stxp w9, x8, x1, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB0_1 +; NOOUTLINE-NEXT: b .LBB0_4 +; NOOUTLINE-NEXT: .LBB0_3: // in Loop: Header=BB0_1 Depth=1 +; NOOUTLINE-NEXT: stxp w9, x4, x5, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB0_1 +; NOOUTLINE-NEXT: .LBB0_4: +; NOOUTLINE-NEXT: mov x0, x8 +; NOOUTLINE-NEXT: ret +; +; OUTLINE-LABEL: val_compare_and_swap: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: .cfi_def_cfa_offset 16 +; OUTLINE-NEXT: .cfi_offset w30, -16 +; OUTLINE-NEXT: mov x1, x3 +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov x0, x2 +; OUTLINE-NEXT: mov x2, x4 +; OUTLINE-NEXT: mov x3, x5 +; OUTLINE-NEXT: mov x4, x8 +; OUTLINE-NEXT: bl __aarch64_cas16_acq +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret +; +; LSE-LABEL: val_compare_and_swap: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; LSE-NEXT: caspa x2, x3, x4, x5, [x0] +; LSE-NEXT: mov x0, x2 +; LSE-NEXT: mov x1, x3 +; LSE-NEXT: ret %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire %val = extractvalue { i128, i1 } %pair, 0 ret i128 %val } +define i128 @val_compare_and_swap_seqcst(i128* %p, i128 %oldval, i128 %newval) { +; NOOUTLINE-LABEL: val_compare_and_swap_seqcst: +; NOOUTLINE: // %bb.0: +; NOOUTLINE-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1 +; NOOUTLINE-NEXT: ldaxp x8, x1, [x0] +; NOOUTLINE-NEXT: cmp x8, x2 +; NOOUTLINE-NEXT: cset w9, ne +; NOOUTLINE-NEXT: cmp x1, x3 +; NOOUTLINE-NEXT: cinc w9, w9, ne +; NOOUTLINE-NEXT: cbz w9, .LBB1_3 +; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1 +; NOOUTLINE-NEXT: stlxp w9, x8, x1, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB1_1 +; NOOUTLINE-NEXT: b .LBB1_4 +; NOOUTLINE-NEXT: .LBB1_3: // in Loop: Header=BB1_1 Depth=1 +; NOOUTLINE-NEXT: stlxp w9, x4, x5, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB1_1 +; NOOUTLINE-NEXT: .LBB1_4: +; NOOUTLINE-NEXT: mov x0, x8 +; NOOUTLINE-NEXT: ret +; +; OUTLINE-LABEL: val_compare_and_swap_seqcst: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: .cfi_def_cfa_offset 16 +; OUTLINE-NEXT: .cfi_offset w30, -16 +; OUTLINE-NEXT: mov x1, x3 +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov x0, x2 +; OUTLINE-NEXT: mov x2, x4 +; OUTLINE-NEXT: mov x3, x5 +; OUTLINE-NEXT: mov x4, x8 +; OUTLINE-NEXT: bl __aarch64_cas16_acq_rel +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret +; +; LSE-LABEL: val_compare_and_swap_seqcst: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; LSE-NEXT: caspal x2, x3, x4, x5, [x0] +; LSE-NEXT: mov x0, x2 +; LSE-NEXT: mov x1, x3 +; LSE-NEXT: ret + %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval seq_cst seq_cst + %val = extractvalue { i128, i1 } %pair, 0 + ret i128 %val +} + +define i128 @val_compare_and_swap_release(i128* %p, i128 %oldval, i128 %newval) { +; NOOUTLINE-LABEL: val_compare_and_swap_release: +; NOOUTLINE: // %bb.0: +; NOOUTLINE-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1 +; NOOUTLINE-NEXT: ldxp x8, x1, [x0] +; NOOUTLINE-NEXT: cmp x8, x2 +; NOOUTLINE-NEXT: cset w9, ne +; NOOUTLINE-NEXT: cmp x1, x3 +; NOOUTLINE-NEXT: cinc w9, w9, ne +; NOOUTLINE-NEXT: cbz w9, .LBB2_3 +; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1 +; NOOUTLINE-NEXT: stlxp w9, x8, x1, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB2_1 +; NOOUTLINE-NEXT: b .LBB2_4 +; NOOUTLINE-NEXT: .LBB2_3: // in Loop: Header=BB2_1 Depth=1 +; NOOUTLINE-NEXT: stlxp w9, x4, x5, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB2_1 +; NOOUTLINE-NEXT: .LBB2_4: +; NOOUTLINE-NEXT: mov x0, x8 +; NOOUTLINE-NEXT: ret +; +; OUTLINE-LABEL: val_compare_and_swap_release: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: .cfi_def_cfa_offset 16 +; OUTLINE-NEXT: .cfi_offset w30, -16 +; OUTLINE-NEXT: mov x1, x3 +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov x0, x2 +; OUTLINE-NEXT: mov x2, x4 +; OUTLINE-NEXT: mov x3, x5 +; OUTLINE-NEXT: mov x4, x8 +; OUTLINE-NEXT: bl __aarch64_cas16_rel +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret +; +; LSE-LABEL: val_compare_and_swap_release: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; LSE-NEXT: caspl x2, x3, x4, x5, [x0] +; LSE-NEXT: mov x0, x2 +; LSE-NEXT: mov x1, x3 +; LSE-NEXT: ret + %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval release monotonic + %val = extractvalue { i128, i1 } %pair, 0 + ret i128 %val +} + +define i128 @val_compare_and_swap_monotonic(i128* %p, i128 %oldval, i128 %newval) { +; NOOUTLINE-LABEL: val_compare_and_swap_monotonic: +; NOOUTLINE: // %bb.0: +; NOOUTLINE-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1 +; NOOUTLINE-NEXT: ldxp x8, x1, [x0] +; NOOUTLINE-NEXT: cmp x8, x2 +; NOOUTLINE-NEXT: cset w9, ne +; NOOUTLINE-NEXT: cmp x1, x3 +; NOOUTLINE-NEXT: cinc w9, w9, ne +; NOOUTLINE-NEXT: cbz w9, .LBB3_3 +; NOOUTLINE-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1 +; NOOUTLINE-NEXT: stxp w9, x8, x1, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB3_1 +; NOOUTLINE-NEXT: b .LBB3_4 +; NOOUTLINE-NEXT: .LBB3_3: // in Loop: Header=BB3_1 Depth=1 +; NOOUTLINE-NEXT: stxp w9, x4, x5, [x0] +; NOOUTLINE-NEXT: cbnz w9, .LBB3_1 +; NOOUTLINE-NEXT: .LBB3_4: +; NOOUTLINE-NEXT: mov x0, x8 +; NOOUTLINE-NEXT: ret +; +; OUTLINE-LABEL: val_compare_and_swap_monotonic: +; OUTLINE: // %bb.0: +; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; OUTLINE-NEXT: .cfi_def_cfa_offset 16 +; OUTLINE-NEXT: .cfi_offset w30, -16 +; OUTLINE-NEXT: mov x1, x3 +; OUTLINE-NEXT: mov x8, x0 +; OUTLINE-NEXT: mov x0, x2 +; OUTLINE-NEXT: mov x2, x4 +; OUTLINE-NEXT: mov x3, x5 +; OUTLINE-NEXT: mov x4, x8 +; OUTLINE-NEXT: bl __aarch64_cas16_relax +; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; OUTLINE-NEXT: ret +; +; LSE-LABEL: val_compare_and_swap_monotonic: +; LSE: // %bb.0: +; LSE-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3 +; LSE-NEXT: // kill: def $x4 killed $x4 killed $x4_x5 def $x4_x5 +; LSE-NEXT: // kill: def $x2 killed $x2 killed $x2_x3 def $x2_x3 +; LSE-NEXT: casp x2, x3, x4, x5, [x0] +; LSE-NEXT: mov x0, x2 +; LSE-NEXT: mov x1, x3 +; LSE-NEXT: ret + %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval monotonic monotonic + %val = extractvalue { i128, i1 } %pair, 0 + ret i128 %val +} + define void @fetch_and_nand(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_nand: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK-DAG: and [[TMP_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 -; CHECK-DAG: and [[TMP_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 -; CHECK-DAG: mvn [[SCRATCH_REGLO:x[0-9]+]], [[TMP_REGLO]] -; CHECK-DAG: mvn [[SCRATCH_REGHI:x[0-9]+]], [[TMP_REGHI]] -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB4_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldxp x9, x8, [x0] +; CHECK-NEXT: and x10, x9, x2 +; CHECK-NEXT: and x11, x8, x3 +; CHECK-NEXT: mvn x11, x11 +; CHECK-NEXT: mvn x10, x10 +; CHECK-NEXT: stlxp w12, x10, x11, [x0] +; CHECK-NEXT: cbnz w12, .LBB4_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw nand i128* %p, i128 %bits release store i128 %val, i128* @var, align 16 ret void @@ -44,14 +234,20 @@ define void @fetch_and_nand(i128* %p, i128 %bits) { define void @fetch_and_or(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_or: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK-DAG: orr [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 -; CHECK-DAG: orr [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB5_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: orr x10, x8, x3 +; CHECK-NEXT: orr x11, x9, x2 +; CHECK-NEXT: stlxp w12, x11, x10, [x0] +; CHECK-NEXT: cbnz w12, .LBB5_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw or i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -59,14 +255,20 @@ define void @fetch_and_or(i128* %p, i128 %bits) { define void @fetch_and_add(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_add: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK: adds [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 -; CHECK: adcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB6_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: adds x10, x9, x2 +; CHECK-NEXT: adcs x11, x8, x3 +; CHECK-NEXT: stlxp w12, x10, x11, [x0] +; CHECK-NEXT: cbnz w12, .LBB6_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw add i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -74,14 +276,20 @@ define void @fetch_and_add(i128* %p, i128 %bits) { define void @fetch_and_sub(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_sub: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK: subs [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 -; CHECK: sbcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB7_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: subs x10, x9, x2 +; CHECK-NEXT: sbcs x11, x8, x3 +; CHECK-NEXT: stlxp w12, x10, x11, [x0] +; CHECK-NEXT: cbnz w12, .LBB7_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw sub i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -89,20 +297,26 @@ define void @fetch_and_sub(i128* %p, i128 %bits) { define void @fetch_and_min(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_min: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK: cmp [[DEST_REGLO]], x2 -; CHECK: cset [[LOCMP:w[0-9]+]], ls -; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 -; CHECK: cset [[HICMP:w[0-9]+]], le -; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq -; CHECK: cmp [[CMP]], #0 -; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne -; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB8_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: cmp x9, x2 +; CHECK-NEXT: cset w10, ls +; CHECK-NEXT: cmp x8, x3 +; CHECK-NEXT: cset w11, le +; CHECK-NEXT: csel w10, w10, w11, eq +; CHECK-NEXT: cmp w10, #0 // =0 +; CHECK-NEXT: csel x10, x8, x3, ne +; CHECK-NEXT: csel x11, x9, x2, ne +; CHECK-NEXT: stlxp w12, x11, x10, [x0] +; CHECK-NEXT: cbnz w12, .LBB8_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw min i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -110,20 +324,26 @@ define void @fetch_and_min(i128* %p, i128 %bits) { define void @fetch_and_max(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_max: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK: cmp [[DEST_REGLO]], x2 -; CHECK: cset [[LOCMP:w[0-9]+]], hi -; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 -; CHECK: cset [[HICMP:w[0-9]+]], gt -; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq -; CHECK: cmp [[CMP]], #0 -; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne -; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB9_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: cmp x9, x2 +; CHECK-NEXT: cset w10, hi +; CHECK-NEXT: cmp x8, x3 +; CHECK-NEXT: cset w11, gt +; CHECK-NEXT: csel w10, w10, w11, eq +; CHECK-NEXT: cmp w10, #0 // =0 +; CHECK-NEXT: csel x10, x8, x3, ne +; CHECK-NEXT: csel x11, x9, x2, ne +; CHECK-NEXT: stlxp w12, x11, x10, [x0] +; CHECK-NEXT: cbnz w12, .LBB9_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw max i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -131,20 +351,26 @@ define void @fetch_and_max(i128* %p, i128 %bits) { define void @fetch_and_umin(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_umin: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK: cmp [[DEST_REGLO]], x2 -; CHECK: cset [[LOCMP:w[0-9]+]], ls -; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 -; CHECK: cset [[HICMP:w[0-9]+]], ls -; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq -; CHECK: cmp [[CMP]], #0 -; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne -; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB10_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: cmp x9, x2 +; CHECK-NEXT: cset w10, ls +; CHECK-NEXT: cmp x8, x3 +; CHECK-NEXT: cset w11, ls +; CHECK-NEXT: csel w10, w10, w11, eq +; CHECK-NEXT: cmp w10, #0 // =0 +; CHECK-NEXT: csel x10, x8, x3, ne +; CHECK-NEXT: csel x11, x9, x2, ne +; CHECK-NEXT: stlxp w12, x11, x10, [x0] +; CHECK-NEXT: cbnz w12, .LBB10_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw umin i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -152,20 +378,26 @@ define void @fetch_and_umin(i128* %p, i128 %bits) { define void @fetch_and_umax(i128* %p, i128 %bits) { ; CHECK-LABEL: fetch_and_umax: -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] -; CHECK: cmp [[DEST_REGLO]], x2 -; CHECK: cset [[LOCMP:w[0-9]+]], hi -; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 -; CHECK: cset [[HICMP:w[0-9]+]], hi -; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq -; CHECK: cmp [[CMP]], #0 -; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne -; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne -; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] -; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB11_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x9, x8, [x0] +; CHECK-NEXT: cmp x9, x2 +; CHECK-NEXT: cset w10, hi +; CHECK-NEXT: cmp x8, x3 +; CHECK-NEXT: cset w11, hi +; CHECK-NEXT: csel w10, w10, w11, eq +; CHECK-NEXT: cmp w10, #0 // =0 +; CHECK-NEXT: csel x10, x8, x3, ne +; CHECK-NEXT: csel x11, x9, x2, ne +; CHECK-NEXT: stlxp w12, x11, x10, [x0] +; CHECK-NEXT: cbnz w12, .LBB11_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: adrp x10, :got:var +; CHECK-NEXT: ldr x10, [x10, :got_lo12:var] +; CHECK-NEXT: stp x9, x8, [x10] +; CHECK-NEXT: ret -; CHECK-DAG: stp [[DEST_REGLO]], [[DEST_REGHI]] %val = atomicrmw umax i128* %p, i128 %bits seq_cst store i128 %val, i128* @var, align 16 ret void @@ -173,21 +405,29 @@ define void @fetch_and_umax(i128* %p, i128 %bits) { define i128 @atomic_load_seq_cst(i128* %p) { ; CHECK-LABEL: atomic_load_seq_cst: -; CHECK-NOT: dmb -; CHECK-LABEL: ldaxp -; CHECK-NOT: dmb +; CHECK: // %bb.0: +; CHECK-NEXT: mov x8, x0 +; CHECK-NEXT: .LBB12_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp x0, x1, [x8] +; CHECK-NEXT: stlxp w9, x0, x1, [x8] +; CHECK-NEXT: cbnz w9, .LBB12_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: ret %r = load atomic i128, i128* %p seq_cst, align 16 ret i128 %r } define i128 @atomic_load_relaxed(i64, i64, i128* %p) { ; CHECK-LABEL: atomic_load_relaxed: -; CHECK-NOT: dmb -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldxp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x2] -; CHECK-NEXT: stxp [[SUCCESS:w[0-9]+]], [[LO]], [[HI]], [x2] -; CHECK: cbnz [[SUCCESS]], [[LABEL]] -; CHECK-NOT: dmb +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB13_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldxp x0, x1, [x2] +; CHECK-NEXT: stxp w8, x0, x1, [x2] +; CHECK-NEXT: cbnz w8, .LBB13_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: ret %r = load atomic i128, i128* %p monotonic, align 16 ret i128 %r } @@ -195,36 +435,42 @@ define i128 @atomic_load_relaxed(i64, i64, i128* %p) { define void @atomic_store_seq_cst(i128 %in, i128* %p) { ; CHECK-LABEL: atomic_store_seq_cst: -; CHECK-NOT: dmb -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldaxp xzr, [[IGNORED:x[0-9]+]], [x2] -; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] -; CHECK: cbnz [[SUCCESS]], [[LABEL]] -; CHECK-NOT: dmb +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB14_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldaxp xzr, x8, [x2] +; CHECK-NEXT: stlxp w8, x0, x1, [x2] +; CHECK-NEXT: cbnz w8, .LBB14_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: ret store atomic i128 %in, i128* %p seq_cst, align 16 ret void } define void @atomic_store_release(i128 %in, i128* %p) { ; CHECK-LABEL: atomic_store_release: -; CHECK-NOT: dmb -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldxp xzr, [[IGNORED:x[0-9]+]], [x2] -; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] -; CHECK: cbnz [[SUCCESS]], [[LABEL]] -; CHECK-NOT: dmb +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB15_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldxp xzr, x8, [x2] +; CHECK-NEXT: stlxp w8, x0, x1, [x2] +; CHECK-NEXT: cbnz w8, .LBB15_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: ret store atomic i128 %in, i128* %p release, align 16 ret void } define void @atomic_store_relaxed(i128 %in, i128* %p) { ; CHECK-LABEL: atomic_store_relaxed: -; CHECK-NOT: dmb -; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: -; CHECK: ldxp xzr, [[IGNORED:x[0-9]+]], [x2] -; CHECK: stxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] -; CHECK: cbnz [[SUCCESS]], [[LABEL]] -; CHECK-NOT: dmb +; CHECK: // %bb.0: +; CHECK-NEXT: .LBB16_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: ldxp xzr, x8, [x2] +; CHECK-NEXT: stxp w8, x0, x1, [x2] +; CHECK-NEXT: cbnz w8, .LBB16_1 +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: ret store atomic i128 %in, i128* %p unordered, align 16 ret void }