[llvm-branch-commits] [llvm] AArch64: Use consistent atomicrmw expansion for FP operations (PR #103702)

Thu Aug 22 13:06:32 PDT 2024

https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/103702

>From 054a7adb6696d0dcf61f10fc75d659ad94948d0b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 14 Aug 2024 00:43:03 +0400
Subject: [PATCH 1/4] AArch64: Use consistent atomicrmw expansion for FP
 operations

Use LLSC or cmpxchg in the same cases as for the unsupported
integer operations. This required some fixups to the LLSC
implementatation to deal with the fp128 case.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  30 +-
 llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll   | 392 ++++------------
 llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll   | 418 +++++-------------
 llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll   | 418 +++++-------------
 llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll   | 392 ++++------------
 5 files changed, 420 insertions(+), 1230 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8c2f85657ff87e..c1fe1054bcdcfa 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27079,9 +27079,6 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
   assert(Size <= 128 && "AtomicExpandPass should've handled larger sizes.");
 
-  if (AI->isFloatingPointOperation())
-    return AtomicExpansionKind::CmpXChg;
-
   bool CanUseLSE128 = Subtarget->hasLSE128() && Size == 128 &&
                       (AI->getOperation() == AtomicRMWInst::Xchg ||
                        AI->getOperation() == AtomicRMWInst::Or ||
@@ -27091,7 +27088,8 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
 
   // Nand is not supported in LSE.
   // Leave 128 bits to LLSC or CmpXChg.
-  if (AI->getOperation() != AtomicRMWInst::Nand && Size < 128) {
+  if (AI->getOperation() != AtomicRMWInst::Nand && Size < 128 &&
+      !AI->isFloatingPointOperation()) {
     if (Subtarget->hasLSE())
       return AtomicExpansionKind::None;
     if (Subtarget->outlineAtomics()) {
@@ -27164,10 +27162,14 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
 
     Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
     Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
-    Lo = Builder.CreateZExt(Lo, ValueTy, "lo64");
-    Hi = Builder.CreateZExt(Hi, ValueTy, "hi64");
-    return Builder.CreateOr(
-        Lo, Builder.CreateShl(Hi, ConstantInt::get(ValueTy, 64)), "val64");
+
+    auto *Int128Ty = Type::getInt128Ty(Builder.getContext());
+    Lo = Builder.CreateZExt(Lo, Int128Ty, "lo64");
+    Hi = Builder.CreateZExt(Hi, Int128Ty, "hi64");
+
+    Value *Or = Builder.CreateOr(
+        Lo, Builder.CreateShl(Hi, ConstantInt::get(Int128Ty, 64)), "val64");
+    return Builder.CreateBitCast(Or, ValueTy);
   }
 
   Type *Tys[] = { Addr->getType() };
@@ -27178,8 +27180,8 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
   const DataLayout &DL = M->getDataLayout();
   IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
   CallInst *CI = Builder.CreateCall(Ldxr, Addr);
-  CI->addParamAttr(
-      0, Attribute::get(Builder.getContext(), Attribute::ElementType, ValueTy));
+  CI->addParamAttr(0, Attribute::get(Builder.getContext(),
+                                     Attribute::ElementType, IntEltTy));
   Value *Trunc = Builder.CreateTrunc(CI, IntEltTy);
 
   return Builder.CreateBitCast(Trunc, ValueTy);
@@ -27205,9 +27207,13 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder,
         IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
     Function *Stxr = Intrinsic::getDeclaration(M, Int);
     Type *Int64Ty = Type::getInt64Ty(M->getContext());
+    Type *Int128Ty = Type::getInt128Ty(M->getContext());
+
+    Value *CastVal = Builder.CreateBitCast(Val, Int128Ty);
 
-    Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo");
-    Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi");
+    Value *Lo = Builder.CreateTrunc(CastVal, Int64Ty, "lo");
+    Value *Hi =
+        Builder.CreateTrunc(Builder.CreateLShr(CastVal, 64), Int64Ty, "hi");
     return Builder.CreateCall(Stxr, {Lo, Hi, Addr});
   }
 
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
index 4ca421449ee33e..37c258ff33f16a 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
@@ -7,33 +7,17 @@ define half @test_atomicrmw_fadd_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align2:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB0_2
 ; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB0_5
-; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fadd s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB0_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB0_3
-; NOLSE-NEXT:    b .LBB0_1
-; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB0_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -108,33 +92,17 @@ define half @test_atomicrmw_fadd_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_f16_seq_cst_align4:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB1_2
 ; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB1_5
-; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fadd s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB1_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB1_3
-; NOLSE-NEXT:    b .LBB1_1
-; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB1_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -211,19 +179,12 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB2_2
 ; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB2_5
-; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fadd s2, s2, s1
@@ -232,21 +193,9 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB2_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB2_3
-; NOLSE-NEXT:    b .LBB2_1
-; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB2_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -325,19 +274,12 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB3_2
 ; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB3_5
-; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fadd s2, s2, s1
@@ -346,21 +288,9 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB3_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB3_3
-; NOLSE-NEXT:    b .LBB3_1
-; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB3_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -436,31 +366,15 @@ define bfloat @test_atomicrmw_fadd_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 define float @test_atomicrmw_fadd_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align4:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr s1, [x0]
-; NOLSE-NEXT:    b .LBB4_2
 ; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
-; NOLSE-NEXT:    fmov s1, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB4_5
-; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s1, w8
 ; NOLSE-NEXT:    fadd s2, s1, s0
-; NOLSE-NEXT:    fmov w9, s1
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB4_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB4_3
-; NOLSE-NEXT:    b .LBB4_1
-; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB4_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov s0, s1
 ; NOLSE-NEXT:    ret
 ;
@@ -523,31 +437,15 @@ define float @test_atomicrmw_fadd_f32_seq_cst_align4(ptr %ptr, float %value) #0
 define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB5_2
 ; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB5_5
-; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fadd d2, d1, d0
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB5_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB5_3
-; NOLSE-NEXT:    b .LBB5_1
-; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB5_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -610,54 +508,26 @@ define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fadd_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #96
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #80
+; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
-; NOLSE-NEXT:    stp x12, x13, [sp, #32]
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    ldr q1, [sp, #32]
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    b.eq .LBB6_6
-; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x19]
+; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    stp x8, x9, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; NOLSE-NEXT:    bl __addtf3
-; NOLSE-NEXT:    str q0, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
-; NOLSE-NEXT:    str q0, [sp, #64]
-; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
-; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x19]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB6_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    str q0, [sp, #32]
+; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w10, .LBB6_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    add sp, sp, #80
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16:
@@ -748,35 +618,19 @@ define fp128 @test_atomicrmw_fadd_fp128_seq_cst_align16(ptr %ptr, fp128 %value)
 define <2 x half> @test_atomicrmw_fadd_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    fcvtl v1.4s, v0.4h
-; NOLSE-NEXT:    ldr s0, [x0]
-; NOLSE-NEXT:    b .LBB7_2
+; NOLSE-NEXT:    fcvtl v0.4s, v0.4h
 ; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB7_5
-; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
-; NOLSE-NEXT:    fcvtl v2.4s, v0.4h
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fadd v2.4s, v2.4s, v1.4s
-; NOLSE-NEXT:    fcvtn v2.4h, v2.4s
-; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB7_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB7_3
-; NOLSE-NEXT:    b .LBB7_1
-; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s1, w8
+; NOLSE-NEXT:    fcvtl v1.4s, v1.4h
+; NOLSE-NEXT:    fadd v1.4s, v1.4s, v0.4s
+; NOLSE-NEXT:    fcvtn v1.4h, v1.4s
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    stlxr w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB7_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x8
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fadd_v2f16_seq_cst_align4:
@@ -867,38 +721,22 @@ define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    movi v1.4s, #1
 ; NOLSE-NEXT:    movi v2.4s, #127, msl #8
-; NOLSE-NEXT:    shll v3.4s, v0.4h, #16
-; NOLSE-NEXT:    ldr s0, [x0]
-; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:    shll v0.4s, v0.4h, #16
 ; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB8_5
-; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
-; NOLSE-NEXT:    shll v4.4s, v0.4h, #16
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fadd v4.4s, v4.4s, v3.4s
-; NOLSE-NEXT:    ushr v5.4s, v4.4s, #16
-; NOLSE-NEXT:    and v5.16b, v5.16b, v1.16b
-; NOLSE-NEXT:    add v4.4s, v5.4s, v4.4s
-; NOLSE-NEXT:    addhn v4.4h, v4.4s, v2.4s
-; NOLSE-NEXT:    fmov w8, s4
-; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB8_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB8_3
-; NOLSE-NEXT:    b .LBB8_1
-; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s3, w8
+; NOLSE-NEXT:    shll v3.4s, v3.4h, #16
+; NOLSE-NEXT:    fadd v3.4s, v3.4s, v0.4s
+; NOLSE-NEXT:    ushr v4.4s, v3.4s, #16
+; NOLSE-NEXT:    and v4.16b, v4.16b, v1.16b
+; NOLSE-NEXT:    add v3.4s, v4.4s, v3.4s
+; NOLSE-NEXT:    addhn v3.4h, v3.4s, v2.4s
+; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    stlxr w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB8_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x8
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fadd_v2bf16_seq_cst_align4:
@@ -984,31 +822,15 @@ define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 define <2 x float> @test_atomicrmw_fadd_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_v2f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB9_2
 ; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB9_5
-; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fadd v2.2s, v1.2s, v0.2s
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB9_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB9_3
-; NOLSE-NEXT:    b .LBB9_1
-; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB9_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -1086,43 +908,17 @@ define <2 x float> @test_atomicrmw_fadd_v2f32_seq_cst_align8(ptr %ptr, <2 x floa
 define <2 x double> @test_atomicrmw_fadd_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_v2f64_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    b .LBB10_2
 ; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x12
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    mov v1.d[1], x13
-; NOLSE-NEXT:    b.eq .LBB10_6
-; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x0]
+; NOLSE-NEXT:    fmov d1, x8
+; NOLSE-NEXT:    mov v1.d[1], x9
 ; NOLSE-NEXT:    fadd v2.2d, v1.2d, v0.2d
-; NOLSE-NEXT:    mov x9, v1.d[1]
-; NOLSE-NEXT:    fmov x11, d1
 ; NOLSE-NEXT:    mov x8, v2.d[1]
-; NOLSE-NEXT:    fmov x10, d2
-; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x0]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB10_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    fmov x9, d2
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB10_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    mov v0.16b, v1.16b
 ; NOLSE-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
index 7a1e395bb80629..bd310a2cc0cc3b 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
@@ -7,33 +7,17 @@ define half @test_atomicrmw_fmax_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align2:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB0_2
 ; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB0_5
-; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fmaxnm s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB0_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB0_3
-; NOLSE-NEXT:    b .LBB0_1
-; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB0_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -108,33 +92,17 @@ define half @test_atomicrmw_fmax_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_f16_seq_cst_align4:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB1_2
 ; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB1_5
-; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fmaxnm s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB1_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB1_3
-; NOLSE-NEXT:    b .LBB1_1
-; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB1_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -211,19 +179,12 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB2_2
 ; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB2_5
-; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fmaxnm s2, s2, s1
@@ -232,21 +193,9 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB2_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB2_3
-; NOLSE-NEXT:    b .LBB2_1
-; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB2_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -325,19 +274,12 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB3_2
 ; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB3_5
-; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fmaxnm s2, s2, s1
@@ -346,21 +288,9 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB3_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB3_3
-; NOLSE-NEXT:    b .LBB3_1
-; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB3_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -436,31 +366,15 @@ define bfloat @test_atomicrmw_fmax_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 define float @test_atomicrmw_fmax_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align4:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr s1, [x0]
-; NOLSE-NEXT:    b .LBB4_2
 ; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
-; NOLSE-NEXT:    fmov s1, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB4_5
-; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s1, w8
 ; NOLSE-NEXT:    fmaxnm s2, s1, s0
-; NOLSE-NEXT:    fmov w9, s1
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB4_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB4_3
-; NOLSE-NEXT:    b .LBB4_1
-; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB4_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov s0, s1
 ; NOLSE-NEXT:    ret
 ;
@@ -523,31 +437,15 @@ define float @test_atomicrmw_fmax_f32_seq_cst_align4(ptr %ptr, float %value) #0
 define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB5_2
 ; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB5_5
-; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fmaxnm d2, d1, d0
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB5_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB5_3
-; NOLSE-NEXT:    b .LBB5_1
-; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB5_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -610,54 +508,26 @@ define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fmax_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #96
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #80
+; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
-; NOLSE-NEXT:    stp x12, x13, [sp, #32]
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    ldr q1, [sp, #32]
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    b.eq .LBB6_6
-; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x19]
+; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    stp x8, x9, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; NOLSE-NEXT:    bl fmaxl
-; NOLSE-NEXT:    str q0, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
-; NOLSE-NEXT:    str q0, [sp, #64]
-; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
-; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x19]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB6_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    str q0, [sp, #32]
+; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w10, .LBB6_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    add sp, sp, #80
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16:
@@ -750,41 +620,25 @@ define <2 x half> @test_atomicrmw_fmax_v2f16_seq_cst_align4(ptr %ptr, <2 x half>
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; NOLSE-NEXT:    mov h1, v0.h[1]
-; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    fcvt s0, h0
 ; NOLSE-NEXT:    fcvt s1, h1
-; NOLSE-NEXT:    b .LBB7_2
 ; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB7_5
-; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
-; NOLSE-NEXT:    mov h3, v0.h[1]
-; NOLSE-NEXT:    fcvt s4, h0
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s2, w8
+; NOLSE-NEXT:    mov h3, v2.h[1]
+; NOLSE-NEXT:    fcvt s2, h2
 ; NOLSE-NEXT:    fcvt s3, h3
-; NOLSE-NEXT:    fmaxnm s4, s4, s2
+; NOLSE-NEXT:    fmaxnm s2, s2, s0
 ; NOLSE-NEXT:    fmaxnm s3, s3, s1
-; NOLSE-NEXT:    fcvt h4, s4
+; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fcvt h3, s3
-; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
-; NOLSE-NEXT:    fmov w8, s4
-; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB7_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB7_3
-; NOLSE-NEXT:    b .LBB7_1
-; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    mov v2.h[1], v3.h[0]
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    stlxr w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB7_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x8
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmax_v2f16_seq_cst_align4:
@@ -885,58 +739,42 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 ; NOLSE-NEXT:    mov h1, v0.h[1]
 ; NOLSE-NEXT:    fmov w10, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr s0, [x0]
 ; NOLSE-NEXT:    lsl w10, w10, #16
 ; NOLSE-NEXT:    fmov w9, s1
-; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    fmov s1, w10
 ; NOLSE-NEXT:    lsl w9, w9, #16
-; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9
-; NOLSE-NEXT:    b.eq .LBB8_5
-; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
-; NOLSE-NEXT:    mov h3, v0.h[1]
-; NOLSE-NEXT:    fmov w10, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w9, [x0]
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    mov h3, v2.h[1]
+; NOLSE-NEXT:    fmov w11, s2
+; NOLSE-NEXT:    lsl w11, w11, #16
+; NOLSE-NEXT:    fmov w10, s3
+; NOLSE-NEXT:    fmov s3, w11
 ; NOLSE-NEXT:    lsl w10, w10, #16
-; NOLSE-NEXT:    fmov w9, s3
-; NOLSE-NEXT:    fmov s4, w10
-; NOLSE-NEXT:    lsl w9, w9, #16
-; NOLSE-NEXT:    fmaxnm s4, s4, s2
-; NOLSE-NEXT:    fmov s3, w9
 ; NOLSE-NEXT:    fmaxnm s3, s3, s1
-; NOLSE-NEXT:    fmov w10, s4
+; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    fmaxnm s2, s2, s0
+; NOLSE-NEXT:    fmov w11, s3
+; NOLSE-NEXT:    ubfx w13, w11, #16, #1
+; NOLSE-NEXT:    add w11, w11, w8
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:    add w11, w13, w11
+; NOLSE-NEXT:    lsr w11, w11, #16
 ; NOLSE-NEXT:    ubfx w12, w10, #16, #1
 ; NOLSE-NEXT:    add w10, w10, w8
-; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    fmov s3, w11
 ; NOLSE-NEXT:    add w10, w12, w10
 ; NOLSE-NEXT:    lsr w10, w10, #16
-; NOLSE-NEXT:    ubfx w11, w9, #16, #1
-; NOLSE-NEXT:    add w9, w9, w8
-; NOLSE-NEXT:    fmov s4, w10
-; NOLSE-NEXT:    add w9, w11, w9
-; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s3, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
-; NOLSE-NEXT:    fmov w10, s4
-; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9
-; NOLSE-NEXT:    b.ne .LBB8_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB8_3
-; NOLSE-NEXT:    b .LBB8_1
-; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    mov v3.h[1], v2.h[0]
+; NOLSE-NEXT:    fmov w10, s3
+; NOLSE-NEXT:    stlxr w11, w10, [x0]
+; NOLSE-NEXT:    cbnz w11, .LBB8_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x9
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmax_v2bf16_seq_cst_align4:
@@ -1044,31 +882,15 @@ define <2 x bfloat> @test_atomicrmw_fmax_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_v2f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB9_2
 ; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB9_5
-; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fmaxnm v2.2s, v1.2s, v0.2s
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB9_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB9_3
-; NOLSE-NEXT:    b .LBB9_1
-; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB9_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -1146,43 +968,17 @@ define <2 x float> @test_atomicrmw_fmax_v2f32_seq_cst_align8(ptr %ptr, <2 x floa
 define <2 x double> @test_atomicrmw_fmax_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_v2f64_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    b .LBB10_2
 ; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x12
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    mov v1.d[1], x13
-; NOLSE-NEXT:    b.eq .LBB10_6
-; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x0]
+; NOLSE-NEXT:    fmov d1, x8
+; NOLSE-NEXT:    mov v1.d[1], x9
 ; NOLSE-NEXT:    fmaxnm v2.2d, v1.2d, v0.2d
-; NOLSE-NEXT:    mov x9, v1.d[1]
-; NOLSE-NEXT:    fmov x11, d1
 ; NOLSE-NEXT:    mov x8, v2.d[1]
-; NOLSE-NEXT:    fmov x10, d2
-; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x0]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB10_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    fmov x9, d2
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB10_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    mov v0.16b, v1.16b
 ; NOLSE-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
index 2ec7c0d2ffa39f..8b348a85d861c0 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
@@ -7,33 +7,17 @@ define half @test_atomicrmw_fmin_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align2:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB0_2
 ; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB0_5
-; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fminnm s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB0_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB0_3
-; NOLSE-NEXT:    b .LBB0_1
-; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB0_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -108,33 +92,17 @@ define half @test_atomicrmw_fmin_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_f16_seq_cst_align4:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB1_2
 ; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB1_5
-; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fminnm s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB1_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB1_3
-; NOLSE-NEXT:    b .LBB1_1
-; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB1_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -211,19 +179,12 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB2_2
 ; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB2_5
-; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fminnm s2, s2, s1
@@ -232,21 +193,9 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB2_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB2_3
-; NOLSE-NEXT:    b .LBB2_1
-; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB2_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -325,19 +274,12 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB3_2
 ; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB3_5
-; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fminnm s2, s2, s1
@@ -346,21 +288,9 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB3_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB3_3
-; NOLSE-NEXT:    b .LBB3_1
-; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB3_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -436,31 +366,15 @@ define bfloat @test_atomicrmw_fmin_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 define float @test_atomicrmw_fmin_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align4:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr s1, [x0]
-; NOLSE-NEXT:    b .LBB4_2
 ; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
-; NOLSE-NEXT:    fmov s1, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB4_5
-; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s1, w8
 ; NOLSE-NEXT:    fminnm s2, s1, s0
-; NOLSE-NEXT:    fmov w9, s1
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB4_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB4_3
-; NOLSE-NEXT:    b .LBB4_1
-; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB4_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov s0, s1
 ; NOLSE-NEXT:    ret
 ;
@@ -523,31 +437,15 @@ define float @test_atomicrmw_fmin_f32_seq_cst_align4(ptr %ptr, float %value) #0
 define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB5_2
 ; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB5_5
-; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fminnm d2, d1, d0
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB5_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB5_3
-; NOLSE-NEXT:    b .LBB5_1
-; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB5_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -610,54 +508,26 @@ define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fmin_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #96
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #80
+; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
-; NOLSE-NEXT:    stp x12, x13, [sp, #32]
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    ldr q1, [sp, #32]
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    b.eq .LBB6_6
-; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x19]
+; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    stp x8, x9, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; NOLSE-NEXT:    bl fminl
-; NOLSE-NEXT:    str q0, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
-; NOLSE-NEXT:    str q0, [sp, #64]
-; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
-; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x19]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB6_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    str q0, [sp, #32]
+; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w10, .LBB6_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    add sp, sp, #80
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16:
@@ -750,41 +620,25 @@ define <2 x half> @test_atomicrmw_fmin_v2f16_seq_cst_align4(ptr %ptr, <2 x half>
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; NOLSE-NEXT:    mov h1, v0.h[1]
-; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    ldr s0, [x0]
+; NOLSE-NEXT:    fcvt s0, h0
 ; NOLSE-NEXT:    fcvt s1, h1
-; NOLSE-NEXT:    b .LBB7_2
 ; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB7_5
-; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
-; NOLSE-NEXT:    mov h3, v0.h[1]
-; NOLSE-NEXT:    fcvt s4, h0
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s2, w8
+; NOLSE-NEXT:    mov h3, v2.h[1]
+; NOLSE-NEXT:    fcvt s2, h2
 ; NOLSE-NEXT:    fcvt s3, h3
-; NOLSE-NEXT:    fminnm s4, s4, s2
+; NOLSE-NEXT:    fminnm s2, s2, s0
 ; NOLSE-NEXT:    fminnm s3, s3, s1
-; NOLSE-NEXT:    fcvt h4, s4
+; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fcvt h3, s3
-; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
-; NOLSE-NEXT:    fmov w8, s4
-; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB7_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB7_3
-; NOLSE-NEXT:    b .LBB7_1
-; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    mov v2.h[1], v3.h[0]
+; NOLSE-NEXT:    fmov w9, s2
+; NOLSE-NEXT:    stlxr w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB7_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x8
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmin_v2f16_seq_cst_align4:
@@ -885,58 +739,42 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 ; NOLSE-NEXT:    mov h1, v0.h[1]
 ; NOLSE-NEXT:    fmov w10, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr s0, [x0]
 ; NOLSE-NEXT:    lsl w10, w10, #16
 ; NOLSE-NEXT:    fmov w9, s1
-; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    fmov s1, w10
 ; NOLSE-NEXT:    lsl w9, w9, #16
-; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9
-; NOLSE-NEXT:    b.eq .LBB8_5
-; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
-; NOLSE-NEXT:    mov h3, v0.h[1]
-; NOLSE-NEXT:    fmov w10, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w9, [x0]
+; NOLSE-NEXT:    fmov s2, w9
+; NOLSE-NEXT:    mov h3, v2.h[1]
+; NOLSE-NEXT:    fmov w11, s2
+; NOLSE-NEXT:    lsl w11, w11, #16
+; NOLSE-NEXT:    fmov w10, s3
+; NOLSE-NEXT:    fmov s3, w11
 ; NOLSE-NEXT:    lsl w10, w10, #16
-; NOLSE-NEXT:    fmov w9, s3
-; NOLSE-NEXT:    fmov s4, w10
-; NOLSE-NEXT:    lsl w9, w9, #16
-; NOLSE-NEXT:    fminnm s4, s4, s2
-; NOLSE-NEXT:    fmov s3, w9
 ; NOLSE-NEXT:    fminnm s3, s3, s1
-; NOLSE-NEXT:    fmov w10, s4
+; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    fminnm s2, s2, s0
+; NOLSE-NEXT:    fmov w11, s3
+; NOLSE-NEXT:    ubfx w13, w11, #16, #1
+; NOLSE-NEXT:    add w11, w11, w8
+; NOLSE-NEXT:    fmov w10, s2
+; NOLSE-NEXT:    add w11, w13, w11
+; NOLSE-NEXT:    lsr w11, w11, #16
 ; NOLSE-NEXT:    ubfx w12, w10, #16, #1
 ; NOLSE-NEXT:    add w10, w10, w8
-; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    fmov s3, w11
 ; NOLSE-NEXT:    add w10, w12, w10
 ; NOLSE-NEXT:    lsr w10, w10, #16
-; NOLSE-NEXT:    ubfx w11, w9, #16, #1
-; NOLSE-NEXT:    add w9, w9, w8
-; NOLSE-NEXT:    fmov s4, w10
-; NOLSE-NEXT:    add w9, w11, w9
-; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s3, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    mov v4.h[1], v3.h[0]
-; NOLSE-NEXT:    fmov w10, s4
-; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9
-; NOLSE-NEXT:    b.ne .LBB8_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB8_3
-; NOLSE-NEXT:    b .LBB8_1
-; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    fmov s2, w10
+; NOLSE-NEXT:    mov v3.h[1], v2.h[0]
+; NOLSE-NEXT:    fmov w10, s3
+; NOLSE-NEXT:    stlxr w11, w10, [x0]
+; NOLSE-NEXT:    cbnz w11, .LBB8_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x9
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmin_v2bf16_seq_cst_align4:
@@ -1044,31 +882,15 @@ define <2 x bfloat> @test_atomicrmw_fmin_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_v2f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB9_2
 ; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB9_5
-; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fminnm v2.2s, v1.2s, v0.2s
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB9_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB9_3
-; NOLSE-NEXT:    b .LBB9_1
-; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB9_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -1146,43 +968,17 @@ define <2 x float> @test_atomicrmw_fmin_v2f32_seq_cst_align8(ptr %ptr, <2 x floa
 define <2 x double> @test_atomicrmw_fmin_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_v2f64_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    b .LBB10_2
 ; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x12
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    mov v1.d[1], x13
-; NOLSE-NEXT:    b.eq .LBB10_6
-; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x0]
+; NOLSE-NEXT:    fmov d1, x8
+; NOLSE-NEXT:    mov v1.d[1], x9
 ; NOLSE-NEXT:    fminnm v2.2d, v1.2d, v0.2d
-; NOLSE-NEXT:    mov x9, v1.d[1]
-; NOLSE-NEXT:    fmov x11, d1
 ; NOLSE-NEXT:    mov x8, v2.d[1]
-; NOLSE-NEXT:    fmov x10, d2
-; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x0]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB10_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    fmov x9, d2
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB10_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    mov v0.16b, v1.16b
 ; NOLSE-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
index d799642451362a..16d404e3ab1b8e 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
@@ -7,33 +7,17 @@ define half @test_atomicrmw_fsub_f16_seq_cst_align2(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align2:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB0_2
 ; NOLSE-NEXT:  .LBB0_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB0_5
-; NOLSE-NEXT:  .LBB0_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB0_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fsub s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB0_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB0_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB0_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB0_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB0_3
-; NOLSE-NEXT:    b .LBB0_1
-; NOLSE-NEXT:  .LBB0_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB0_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -108,33 +92,17 @@ define half @test_atomicrmw_fsub_f16_seq_cst_align4(ptr %ptr, half %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_f16_seq_cst_align4:
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    fcvt s1, h0
-; NOLSE-NEXT:    ldr h0, [x0]
-; NOLSE-NEXT:    b .LBB1_2
 ; NOLSE-NEXT:  .LBB1_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB1_5
-; NOLSE-NEXT:  .LBB1_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB1_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w8, [x0]
+; NOLSE-NEXT:    fmov s0, w8
 ; NOLSE-NEXT:    fcvt s2, h0
-; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    fsub s2, s2, s1
 ; NOLSE-NEXT:    fcvt h2, s2
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB1_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB1_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB1_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB1_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB1_3
-; NOLSE-NEXT:    b .LBB1_1
-; NOLSE-NEXT:  .LBB1_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB1_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -211,19 +179,12 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB2_2
 ; NOLSE-NEXT:  .LBB2_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB2_5
-; NOLSE-NEXT:  .LBB2_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB2_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fsub s2, s2, s1
@@ -232,21 +193,9 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align2(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB2_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB2_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB2_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB2_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB2_3
-; NOLSE-NEXT:    b .LBB2_1
-; NOLSE-NEXT:  .LBB2_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB2_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -325,19 +274,12 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 def $s0
 ; NOLSE-NEXT:    fmov w9, s0
 ; NOLSE-NEXT:    mov w8, #32767 // =0x7fff
-; NOLSE-NEXT:    ldr h0, [x0]
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s1, w9
-; NOLSE-NEXT:    b .LBB3_2
 ; NOLSE-NEXT:  .LBB3_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w11
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.eq .LBB3_5
-; NOLSE-NEXT:  .LBB3_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB3_3 Depth 2
-; NOLSE-NEXT:    fmov w9, s0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxrh w9, [x0]
+; NOLSE-NEXT:    fmov s0, w9
 ; NOLSE-NEXT:    lsl w9, w9, #16
 ; NOLSE-NEXT:    fmov s2, w9
 ; NOLSE-NEXT:    fsub s2, s2, s1
@@ -346,21 +288,9 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 ; NOLSE-NEXT:    add w9, w9, w8
 ; NOLSE-NEXT:    add w9, w10, w9
 ; NOLSE-NEXT:    lsr w9, w9, #16
-; NOLSE-NEXT:    fmov s2, w9
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fmov w10, s2
-; NOLSE-NEXT:  .LBB3_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB3_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxrh w11, [x0]
-; NOLSE-NEXT:    cmp w11, w9, uxth
-; NOLSE-NEXT:    b.ne .LBB3_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB3_3 Depth=2
-; NOLSE-NEXT:    stlxrh wzr, w10, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB3_3
-; NOLSE-NEXT:    b .LBB3_1
-; NOLSE-NEXT:  .LBB3_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxrh w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB3_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; NOLSE-NEXT:    ret
 ;
@@ -436,31 +366,15 @@ define bfloat @test_atomicrmw_fsub_bf16_seq_cst_align4(ptr %ptr, bfloat %value)
 define float @test_atomicrmw_fsub_f32_seq_cst_align4(ptr %ptr, float %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align4:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr s1, [x0]
-; NOLSE-NEXT:    b .LBB4_2
 ; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=1
-; NOLSE-NEXT:    fmov s1, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB4_5
-; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB4_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s1, w8
 ; NOLSE-NEXT:    fsub s2, s1, s0
-; NOLSE-NEXT:    fmov w9, s1
 ; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB4_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB4_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB4_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB4_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB4_3
-; NOLSE-NEXT:    b .LBB4_1
-; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, w8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB4_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov s0, s1
 ; NOLSE-NEXT:    ret
 ;
@@ -523,31 +437,15 @@ define float @test_atomicrmw_fsub_f32_seq_cst_align4(ptr %ptr, float %value) #0
 define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB5_2
 ; NOLSE-NEXT:  .LBB5_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB5_5
-; NOLSE-NEXT:  .LBB5_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB5_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fsub d2, d1, d0
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB5_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB5_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB5_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB5_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB5_3
-; NOLSE-NEXT:    b .LBB5_1
-; NOLSE-NEXT:  .LBB5_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB5_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -610,54 +508,26 @@ define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fsub_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #96
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #80
+; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
-; NOLSE-NEXT:    stp x12, x13, [sp, #32]
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    ldr q1, [sp, #32]
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    b.eq .LBB6_6
-; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x19]
+; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    stp x8, x9, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; NOLSE-NEXT:    bl __subtf3
-; NOLSE-NEXT:    str q0, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
-; NOLSE-NEXT:    str q0, [sp, #64]
-; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
-; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x19]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x10
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB6_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w14, .LBB6_3
-; NOLSE-NEXT:    b .LBB6_1
-; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
-; NOLSE-NEXT:    mov v0.16b, v1.16b
-; NOLSE-NEXT:    add sp, sp, #96
+; NOLSE-NEXT:    str q0, [sp, #32]
+; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w10, .LBB6_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; NOLSE-NEXT:    add sp, sp, #80
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16:
@@ -748,35 +618,19 @@ define fp128 @test_atomicrmw_fsub_fp128_seq_cst_align16(ptr %ptr, fp128 %value)
 define <2 x half> @test_atomicrmw_fsub_v2f16_seq_cst_align4(ptr %ptr, <2 x half> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    fcvtl v1.4s, v0.4h
-; NOLSE-NEXT:    ldr s0, [x0]
-; NOLSE-NEXT:    b .LBB7_2
+; NOLSE-NEXT:    fcvtl v0.4s, v0.4h
 ; NOLSE-NEXT:  .LBB7_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB7_5
-; NOLSE-NEXT:  .LBB7_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB7_3 Depth 2
-; NOLSE-NEXT:    fcvtl v2.4s, v0.4h
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fsub v2.4s, v2.4s, v1.4s
-; NOLSE-NEXT:    fcvtn v2.4h, v2.4s
-; NOLSE-NEXT:    fmov w8, s2
-; NOLSE-NEXT:  .LBB7_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB7_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB7_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB7_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB7_3
-; NOLSE-NEXT:    b .LBB7_1
-; NOLSE-NEXT:  .LBB7_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s1, w8
+; NOLSE-NEXT:    fcvtl v1.4s, v1.4h
+; NOLSE-NEXT:    fsub v1.4s, v1.4s, v0.4s
+; NOLSE-NEXT:    fcvtn v1.4h, v1.4s
+; NOLSE-NEXT:    fmov w9, s1
+; NOLSE-NEXT:    stlxr w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB7_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x8
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fsub_v2f16_seq_cst_align4:
@@ -867,38 +721,22 @@ define <2 x bfloat> @test_atomicrmw_fsub_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 ; NOLSE:       // %bb.0:
 ; NOLSE-NEXT:    movi v1.4s, #1
 ; NOLSE-NEXT:    movi v2.4s, #127, msl #8
-; NOLSE-NEXT:    shll v3.4s, v0.4h, #16
-; NOLSE-NEXT:    ldr s0, [x0]
-; NOLSE-NEXT:    b .LBB8_2
+; NOLSE-NEXT:    shll v0.4s, v0.4h, #16
 ; NOLSE-NEXT:  .LBB8_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_2 Depth=1
-; NOLSE-NEXT:    fmov s0, w10
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.eq .LBB8_5
-; NOLSE-NEXT:  .LBB8_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB8_3 Depth 2
-; NOLSE-NEXT:    shll v4.4s, v0.4h, #16
-; NOLSE-NEXT:    fmov w9, s0
-; NOLSE-NEXT:    fsub v4.4s, v4.4s, v3.4s
-; NOLSE-NEXT:    ushr v5.4s, v4.4s, #16
-; NOLSE-NEXT:    and v5.16b, v5.16b, v1.16b
-; NOLSE-NEXT:    add v4.4s, v5.4s, v4.4s
-; NOLSE-NEXT:    addhn v4.4h, v4.4s, v2.4s
-; NOLSE-NEXT:    fmov w8, s4
-; NOLSE-NEXT:  .LBB8_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB8_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr w10, [x0]
-; NOLSE-NEXT:    cmp w10, w9
-; NOLSE-NEXT:    b.ne .LBB8_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB8_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, w8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB8_3
-; NOLSE-NEXT:    b .LBB8_1
-; NOLSE-NEXT:  .LBB8_5: // %atomicrmw.end
-; NOLSE-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr w8, [x0]
+; NOLSE-NEXT:    fmov s3, w8
+; NOLSE-NEXT:    shll v3.4s, v3.4h, #16
+; NOLSE-NEXT:    fsub v3.4s, v3.4s, v0.4s
+; NOLSE-NEXT:    ushr v4.4s, v3.4s, #16
+; NOLSE-NEXT:    and v4.16b, v4.16b, v1.16b
+; NOLSE-NEXT:    add v3.4s, v4.4s, v3.4s
+; NOLSE-NEXT:    addhn v3.4h, v3.4s, v2.4s
+; NOLSE-NEXT:    fmov w9, s3
+; NOLSE-NEXT:    stlxr w10, w9, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB8_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT:    fmov d0, x8
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fsub_v2bf16_seq_cst_align4:
@@ -984,31 +822,15 @@ define <2 x bfloat> @test_atomicrmw_fsub_v2bf16_seq_cst_align4(ptr %ptr, <2 x bf
 define <2 x float> @test_atomicrmw_fsub_v2f32_seq_cst_align8(ptr %ptr, <2 x float> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_v2f32_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr d1, [x0]
-; NOLSE-NEXT:    b .LBB9_2
 ; NOLSE-NEXT:  .LBB9_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x10
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.eq .LBB9_5
-; NOLSE-NEXT:  .LBB9_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB9_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxr x8, [x0]
+; NOLSE-NEXT:    fmov d1, x8
 ; NOLSE-NEXT:    fsub v2.2s, v1.2s, v0.2s
-; NOLSE-NEXT:    fmov x9, d1
 ; NOLSE-NEXT:    fmov x8, d2
-; NOLSE-NEXT:  .LBB9_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB9_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxr x10, [x0]
-; NOLSE-NEXT:    cmp x10, x9
-; NOLSE-NEXT:    b.ne .LBB9_1
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB9_3 Depth=2
-; NOLSE-NEXT:    stlxr wzr, x8, [x0]
-; NOLSE-NEXT:    cbnz wzr, .LBB9_3
-; NOLSE-NEXT:    b .LBB9_1
-; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    stlxr w9, x8, [x0]
+; NOLSE-NEXT:    cbnz w9, .LBB9_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    fmov d0, d1
 ; NOLSE-NEXT:    ret
 ;
@@ -1086,43 +908,17 @@ define <2 x float> @test_atomicrmw_fsub_v2f32_seq_cst_align8(ptr %ptr, <2 x floa
 define <2 x double> @test_atomicrmw_fsub_v2f64_seq_cst_align8(ptr %ptr, <2 x double> %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_v2f64_seq_cst_align8:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    ldr q1, [x0]
-; NOLSE-NEXT:    b .LBB10_2
 ; NOLSE-NEXT:  .LBB10_1: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_2 Depth=1
-; NOLSE-NEXT:    fmov d1, x12
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    ccmp x12, x11, #0, eq
-; NOLSE-NEXT:    mov v1.d[1], x13
-; NOLSE-NEXT:    b.eq .LBB10_6
-; NOLSE-NEXT:  .LBB10_2: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Loop Header: Depth=1
-; NOLSE-NEXT:    // Child Loop BB10_3 Depth 2
+; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT:    ldaxp x8, x9, [x0]
+; NOLSE-NEXT:    fmov d1, x8
+; NOLSE-NEXT:    mov v1.d[1], x9
 ; NOLSE-NEXT:    fsub v2.2d, v1.2d, v0.2d
-; NOLSE-NEXT:    mov x9, v1.d[1]
-; NOLSE-NEXT:    fmov x11, d1
 ; NOLSE-NEXT:    mov x8, v2.d[1]
-; NOLSE-NEXT:    fmov x10, d2
-; NOLSE-NEXT:  .LBB10_3: // %atomicrmw.start
-; NOLSE-NEXT:    // Parent Loop BB10_2 Depth=1
-; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
-; NOLSE-NEXT:    ldaxp x12, x13, [x0]
-; NOLSE-NEXT:    cmp x12, x11
-; NOLSE-NEXT:    cset w14, ne
-; NOLSE-NEXT:    cmp x13, x9
-; NOLSE-NEXT:    cinc w14, w14, ne
-; NOLSE-NEXT:    cbz w14, .LBB10_5
-; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x12, x13, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_5: // %atomicrmw.start
-; NOLSE-NEXT:    // in Loop: Header=BB10_3 Depth=2
-; NOLSE-NEXT:    stlxp w14, x10, x8, [x0]
-; NOLSE-NEXT:    cbnz w14, .LBB10_3
-; NOLSE-NEXT:    b .LBB10_1
-; NOLSE-NEXT:  .LBB10_6: // %atomicrmw.end
+; NOLSE-NEXT:    fmov x9, d2
+; NOLSE-NEXT:    stlxp w10, x9, x8, [x0]
+; NOLSE-NEXT:    cbnz w10, .LBB10_1
+; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
 ; NOLSE-NEXT:    mov v0.16b, v1.16b
 ; NOLSE-NEXT:    ret
 ;

>From 473f51bcee7437f7dabd9aef636e82c477e9db7d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 15 Aug 2024 18:36:20 +0400
Subject: [PATCH 2/4] Remove wrong comment about fp exceptions

---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c1fe1054bcdcfa..1883576e3e453c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27071,9 +27071,6 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
 // However, with the LSE instructions (or outline-atomics mode, which provides
 // library routines in place of the LSE-instructions), we can directly emit many
 // operations instead.
-//
-// Floating-point operations are always emitted to a cmpxchg loop, because they
-// may trigger a trap which aborts an LLSC sequence.
 TargetLowering::AtomicExpansionKind
 AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();

>From 12ca53f299c9ce7df689c6b424811f87ab8605c4 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Fri, 16 Aug 2024 14:26:55 +0400
Subject: [PATCH 3/4] Check if libcall

---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 24 ++++++-
 llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll   | 62 ++++++++++++++-----
 llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll   | 62 ++++++++++++++-----
 llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll   | 62 ++++++++++++++-----
 llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll   | 62 ++++++++++++++-----
 5 files changed, 202 insertions(+), 70 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1883576e3e453c..2a27e30a70e023 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27067,13 +27067,33 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
                              : AtomicExpansionKind::LLSC;
 }
 
+// Return true if the atomic operation expansion will lower to use a library
+// call, and is thus ineligible to use an LLSC expansion.
+static bool rmwOpMayLowerToLibcall(const AtomicRMWInst *RMW) {
+  if (!RMW->isFloatingPointOperation())
+    return false;
+  switch (RMW->getType()->getScalarType()->getTypeID()) {
+  case Type::FloatTyID:
+  case Type::DoubleTyID:
+  case Type::HalfTyID:
+  case Type::BFloatTyID:
+    return false;
+  default:
+    // fp128 will emit library calls.
+    return true;
+  }
+
+  llvm_unreachable("covered type switch");
+}
+
 // The "default" for integer RMW operations is to expand to an LL/SC loop.
 // However, with the LSE instructions (or outline-atomics mode, which provides
 // library routines in place of the LSE-instructions), we can directly emit many
 // operations instead.
 TargetLowering::AtomicExpansionKind
 AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
-  unsigned Size = AI->getType()->getPrimitiveSizeInBits();
+  Type *Ty = AI->getType();
+  unsigned Size = Ty->getPrimitiveSizeInBits();
   assert(Size <= 128 && "AtomicExpandPass should've handled larger sizes.");
 
   bool CanUseLSE128 = Subtarget->hasLSE128() && Size == 128 &&
@@ -27112,7 +27132,7 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   // succeed. So at -O0 lower this operation to a CAS loop. Also worthwhile if
   // we have a single CAS instruction that can replace the loop.
   if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None ||
-      Subtarget->hasLSE())
+      Subtarget->hasLSE() || rmwOpMayLowerToLibcall(AI))
     return AtomicExpansionKind::CmpXChg;
 
   return AtomicExpansionKind::LLSC;
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
index 37c258ff33f16a..3b6f6981f5265b 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fadd.ll
@@ -508,26 +508,54 @@ define double @test_atomicrmw_fadd_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fadd_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #80
-; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT:    ldaxp x8, x9, [x19]
-; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    stp x8, x9, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #48]
 ; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; NOLSE-NEXT:    bl __addtf3
-; NOLSE-NEXT:    str q0, [sp, #32]
-; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
-; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w10, .LBB6_1
-; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; NOLSE-NEXT:    add sp, sp, #80
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fadd_fp128_seq_cst_align16:
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
index bd310a2cc0cc3b..ebcc9e13331faa 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmax.ll
@@ -508,26 +508,54 @@ define double @test_atomicrmw_fmax_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fmax_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #80
-; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT:    ldaxp x8, x9, [x19]
-; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    stp x8, x9, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #48]
 ; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; NOLSE-NEXT:    bl fmaxl
-; NOLSE-NEXT:    str q0, [sp, #32]
-; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
-; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w10, .LBB6_1
-; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; NOLSE-NEXT:    add sp, sp, #80
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmax_fp128_seq_cst_align16:
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
index 8b348a85d861c0..6001c728dbeff5 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fmin.ll
@@ -508,26 +508,54 @@ define double @test_atomicrmw_fmin_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fmin_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #80
-; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT:    ldaxp x8, x9, [x19]
-; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    stp x8, x9, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #48]
 ; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; NOLSE-NEXT:    bl fminl
-; NOLSE-NEXT:    str q0, [sp, #32]
-; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
-; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w10, .LBB6_1
-; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; NOLSE-NEXT:    add sp, sp, #80
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fmin_fp128_seq_cst_align16:
diff --git a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
index 16d404e3ab1b8e..16a65218fb929d 100644
--- a/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AArch64/atomicrmw-fsub.ll
@@ -508,26 +508,54 @@ define double @test_atomicrmw_fsub_f32_seq_cst_align8(ptr %ptr, double %value) #
 define fp128 @test_atomicrmw_fsub_fp128_seq_cst_align16(ptr %ptr, fp128 %value) #0 {
 ; NOLSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16:
 ; NOLSE:       // %bb.0:
-; NOLSE-NEXT:    sub sp, sp, #80
-; NOLSE-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; NOLSE-NEXT:    sub sp, sp, #96
+; NOLSE-NEXT:    ldr q1, [x0]
+; NOLSE-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; NOLSE-NEXT:    mov x19, x0
-; NOLSE-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
-; NOLSE-NEXT:    // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT:    ldaxp x8, x9, [x19]
-; NOLSE-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; NOLSE-NEXT:    stp x8, x9, [sp, #48]
-; NOLSE-NEXT:    ldr q0, [sp, #48]
 ; NOLSE-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; NOLSE-NEXT:    b .LBB6_2
+; NOLSE-NEXT:  .LBB6_1: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_2 Depth=1
+; NOLSE-NEXT:    stp x12, x13, [sp, #32]
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    ldr q1, [sp, #32]
+; NOLSE-NEXT:    ccmp x12, x11, #0, eq
+; NOLSE-NEXT:    b.eq .LBB6_6
+; NOLSE-NEXT:  .LBB6_2: // %atomicrmw.start
+; NOLSE-NEXT:    // =>This Loop Header: Depth=1
+; NOLSE-NEXT:    // Child Loop BB6_3 Depth 2
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; NOLSE-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; NOLSE-NEXT:    bl __subtf3
-; NOLSE-NEXT:    str q0, [sp, #32]
-; NOLSE-NEXT:    ldp x9, x8, [sp, #32]
-; NOLSE-NEXT:    stlxp w10, x9, x8, [x19]
-; NOLSE-NEXT:    cbnz w10, .LBB6_1
-; NOLSE-NEXT:  // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; NOLSE-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; NOLSE-NEXT:    add sp, sp, #80
+; NOLSE-NEXT:    str q0, [sp, #48]
+; NOLSE-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; NOLSE-NEXT:    ldp x9, x8, [sp, #48]
+; NOLSE-NEXT:    str q0, [sp, #64]
+; NOLSE-NEXT:    ldp x11, x10, [sp, #64]
+; NOLSE-NEXT:  .LBB6_3: // %atomicrmw.start
+; NOLSE-NEXT:    // Parent Loop BB6_2 Depth=1
+; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
+; NOLSE-NEXT:    ldaxp x12, x13, [x19]
+; NOLSE-NEXT:    cmp x12, x11
+; NOLSE-NEXT:    cset w14, ne
+; NOLSE-NEXT:    cmp x13, x10
+; NOLSE-NEXT:    cinc w14, w14, ne
+; NOLSE-NEXT:    cbz w14, .LBB6_5
+; NOLSE-NEXT:  // %bb.4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x12, x13, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_5: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB6_3 Depth=2
+; NOLSE-NEXT:    stlxp w14, x9, x8, [x19]
+; NOLSE-NEXT:    cbnz w14, .LBB6_3
+; NOLSE-NEXT:    b .LBB6_1
+; NOLSE-NEXT:  .LBB6_6: // %atomicrmw.end
+; NOLSE-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; NOLSE-NEXT:    mov v0.16b, v1.16b
+; NOLSE-NEXT:    add sp, sp, #96
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_atomicrmw_fsub_fp128_seq_cst_align16:

>From 5bed50cf488bc942231c49a97c7c999e451e88f3 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 22 Aug 2024 23:55:03 +0400
Subject: [PATCH 4/4] Check soft float

---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2a27e30a70e023..fbbfe2c8cf4e62 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27069,7 +27069,8 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
 
 // Return true if the atomic operation expansion will lower to use a library
 // call, and is thus ineligible to use an LLSC expansion.
-static bool rmwOpMayLowerToLibcall(const AtomicRMWInst *RMW) {
+static bool rmwOpMayLowerToLibcall(const AArch64Subtarget &Subtarget,
+                                   const AtomicRMWInst *RMW) {
   if (!RMW->isFloatingPointOperation())
     return false;
   switch (RMW->getType()->getScalarType()->getTypeID()) {
@@ -27077,7 +27078,8 @@ static bool rmwOpMayLowerToLibcall(const AtomicRMWInst *RMW) {
   case Type::DoubleTyID:
   case Type::HalfTyID:
   case Type::BFloatTyID:
-    return false;
+    // Will use soft float
+    return !Subtarget.hasFPARMv8();
   default:
     // fp128 will emit library calls.
     return true;
@@ -27132,7 +27134,7 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   // succeed. So at -O0 lower this operation to a CAS loop. Also worthwhile if
   // we have a single CAS instruction that can replace the loop.
   if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None ||
-      Subtarget->hasLSE() || rmwOpMayLowerToLibcall(AI))
+      Subtarget->hasLSE() || rmwOpMayLowerToLibcall(*Subtarget, AI))
     return AtomicExpansionKind::CmpXChg;
 
   return AtomicExpansionKind::LLSC;

