[llvm] [SDAG][AArch64] Fold extract from pext to use status flags (PR #206443)

Benjamin Maxwell via llvm-commits llvm-commits at lists.llvm.org
Thu Jul 2 10:10:03 PDT 2026


https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/206443

>From 204975c8b01a493a3c19204ea69c4e50925d766b Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 29 Jun 2026 09:45:26 +0000
Subject: [PATCH 1/3] Precommit tests

---
 .../CodeGen/AArch64/sve2p1-while-pn-folds.ll  | 232 ++++++++++++++++++
 1 file changed, 232 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll

diff --git a/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll b/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
new file mode 100644
index 0000000000000..580f1c413cd6f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
@@ -0,0 +1,232 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-unknown -mattr=+sve2p1 -o - < %s | FileCheck %s -check-prefixes=CHECK,CHECK-SVE
+; RUN: llc -mtriple=aarch64-linux-unknown -mattr=+sme2 -force-streaming -o - < %s | FileCheck %s -check-prefixes=CHECK,CHECK-SME
+
+; Tests extracting the first lane from the first segment of a
+; predicate-as-counter while folds to a conditional set (CSET) based on the
+; "FIRST_ACTIVE" status flag from the while.
+;
+; %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+; %first.segment = pext(%while, 0) ; predicate extract of segment 0
+; %first.active = extract_elt(%first_segment, 0) ; extract first lane
+;
+; ->
+;
+; %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+; %first.active = cset(%while, FIRST_ACTIVE)
+
+define i1 @whilege_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilege_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.b, pn8[0]
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 16 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilegt_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilegt_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilegt pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.h, pn8[0]
+; CHECK-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 8 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilelt_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelt_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelt pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.s, pn8[0]
+; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 4 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilele_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilele_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilele pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.d, pn8[0]
+; CHECK-NEXT:    mov z0.d, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 2 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilehs_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehs_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehs pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.b, pn8[0]
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 16 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilehi_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilehi_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilehi pn8.h, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.h, pn8[0]
+; CHECK-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 8 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilelo_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.s, pn8[0]
+; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 4 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define i1 @whilels_first_active(i64 %a, i64 %b) {
+; CHECK-LABEL: whilels_first_active:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilels pn8.d, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.d, pn8[0]
+; CHECK-NEXT:    mov z0.d, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 2 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+define void @whilege_first_active_branch(i64 %a, i64 %b) {
+; CHECK-LABEL: whilege_first_active_branch:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.b, pn8[0]
+; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tbz w8, #0, .LBB8_2
+; CHECK-NEXT:  // %bb.1: // %then
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:  .LBB8_2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 16 x i1> %pext, i64 0
+  br i1 %bit, label %then, label %exit
+
+then:
+  tail call void asm sideeffect "", ""()
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @whilelo_first_active_branch(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_first_active_branch:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.s, pn8[0]
+; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    tbnz w8, #0, .LBB9_2
+; CHECK-NEXT:  // %bb.1: // %then
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:  .LBB9_2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 4 x i1> %pext, i64 0
+  br i1 %bit, label %exit, label %then
+
+then:
+  tail call void asm sideeffect "", ""()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Negative test: a nonzero pext offset does not match the first-active fold.
+define i1 @whilelo_pext_nonzero_offset(i64 %a, i64 %b) {
+; CHECK-LABEL: whilelo_pext_nonzero_offset:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
+; CHECK-NEXT:    pext p0.s, pn8[1]
+; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") %while, i32 1)
+  %bit = extractelement <vscale x 4 x i1> %pext, i64 0
+  ret i1 %bit
+}
+
+; Negative test: extracting any element other than lane 0 does not match the first-active fold.
+define i1 @whilelt_extractelement_nonzero_index(i64 %a, i64 %b) {
+; CHECK-SVE-LABEL: whilelt_extractelement_nonzero_index:
+; CHECK-SVE:       // %bb.0:
+; CHECK-SVE-NEXT:    whilelt pn8.h, x0, x1, vlx4
+; CHECK-SVE-NEXT:    pext p0.h, pn8[0]
+; CHECK-SVE-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-SVE-NEXT:    umov w8, v0.h[1]
+; CHECK-SVE-NEXT:    and w0, w8, #0x1
+; CHECK-SVE-NEXT:    ret
+;
+; CHECK-SME-LABEL: whilelt_extractelement_nonzero_index:
+; CHECK-SME:       // %bb.0:
+; CHECK-SME-NEXT:    whilelt pn8.h, x0, x1, vlx4
+; CHECK-SME-NEXT:    pext p0.h, pn8[0]
+; CHECK-SME-NEXT:    mov z0.h, p0/z, #1 // =0x1
+; CHECK-SME-NEXT:    mov z0.h, z0.h[1]
+; CHECK-SME-NEXT:    fmov w8, s0
+; CHECK-SME-NEXT:    and w0, w8, #0x1
+; CHECK-SME-NEXT:    ret
+  %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c16(i64 %a, i64 %b, i32 4)
+  %pext = call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") %while, i32 0)
+  %bit = extractelement <vscale x 8 x i1> %pext, i64 1
+  ret i1 %bit
+}

>From a5d443732230456633c79d5d42a579d1cb3d5625 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 29 Jun 2026 09:45:37 +0000
Subject: [PATCH 2/3] [SDAG][AArch64] Fold extract from `pext` to use status
 flags

This folds extracting the first bit from the first segment of a
predicate-as-counter to use the "first active" status. E.g.:

```
%pn:aarch64svcount, %flags:FlagsVT = WHILELO_PRED_COUNTER(a, b, VLx4)
%first_pred:nxv4i1 = pext(%pn, 0)
%more:i1 = extractelement(%first_pred, 0)
```
->
```
%pn:aarch64svcount, %flags:FlagsVT = WHILELO_PRED_COUNTER(a, b, VLx4)
%more = CSET(%flags, FIRST_ACTIVE)
```

Assisted-by: Codex (adding test variations)
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 61 +++++++++++++++++++
 .../CodeGen/AArch64/sve2p1-while-pn-folds.ll  | 50 ++++-----------
 2 files changed, 72 insertions(+), 39 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..278dcf99d844e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -21970,6 +21970,65 @@ performExtractLastActiveCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                      Vec);
 }
 
+static bool hasSVE2p1OrStreamingSME2(const AArch64Subtarget *Subtarget) {
+  return (Subtarget->isSVEorStreamingSVEAvailable() &&
+          Subtarget->hasSVE2p1()) ||
+         (Subtarget->isStreaming() && Subtarget->hasSME2());
+}
+
+static auto m_PredicateAsCounterWhile() {
+  using namespace llvm::SDPatternMatch;
+  return m_AnyOf(m_SpecificOpc(AArch64ISD::WHILEGE_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILEGT_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILELT_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILELE_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILEHS_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILEHI_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILELO_PRED_COUNTER),
+                 m_SpecificOpc(AArch64ISD::WHILELS_PRED_COUNTER));
+}
+
+/// Folds extracting the first lane from the first segment of a
+/// predicate-as-counter while to a conditional set (CSET) based on the
+/// "FIRST_ACTIVE" status flag from the while.
+///
+///   %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+///   %first.segment = pext(%while, 0) ; predicate extract of segment 0
+///   %first.active = extract_elt(%first_segment, 0) ; extract first lane
+///
+///   ->
+///
+///   %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+///   %first.active = cset(%while, FIRST_ACTIVE)
+static SDValue
+perfomPextFirstTrueVectorCombine(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 const AArch64Subtarget *Subtarget) {
+  using namespace llvm::SDPatternMatch;
+  assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT);
+  if (DCI.isBeforeLegalize() || !hasSVE2p1OrStreamingSME2(Subtarget))
+    return SDValue();
+
+  SDValue N0 = N->getOperand(0);
+  EVT VT = N0.getValueType();
+
+  if (!VT.isScalableVectorOf(MVT::i1) || !isNullConstant(N->getOperand(1)))
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+
+  // Fold: extract_elt(pext(WHILE_*_PRED_COUNTER, 0), 0)
+  // to cset(WHILE_*_PRED_COUNTER, FIRST_ACTIVE).
+  if (sd_match(N0, m_IntrinsicWOChain<Intrinsic::aarch64_sve_pext>(
+                       m_PredicateAsCounterWhile(), m_Zero()))) {
+    SDValue WhilePredCounter = N0->getOperand(1);
+    SDValue Flags = SDValue(WhilePredCounter.getNode(), 1);
+    return getSETCC(AArch64CC::CondCode::FIRST_ACTIVE, Flags, SDLoc(N), DAG);
+  }
+
+  return SDValue();
+}
+
 static SDValue
 performExtractVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                const AArch64Subtarget *Subtarget) {
@@ -21980,6 +22039,8 @@ performExtractVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
     return Res;
   if (SDValue Res = performExtractLastActiveCombine(N, DCI, Subtarget))
     return Res;
+  if (SDValue Res = perfomPextFirstTrueVectorCombine(N, DCI, Subtarget))
+    return Res;
 
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll b/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
index 580f1c413cd6f..e4c0307fd0dcb 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
@@ -19,10 +19,7 @@ define i1 @whilege_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilege_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.b, pn8[0]
-; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilege.c8(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") %while, i32 0)
@@ -34,10 +31,7 @@ define i1 @whilegt_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilegt_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilegt pn8.h, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.h, pn8[0]
-; CHECK-NEXT:    mov z0.h, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilegt.c16(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") %while, i32 0)
@@ -49,10 +43,7 @@ define i1 @whilelt_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilelt_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilelt pn8.s, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.s, pn8[0]
-; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelt.c32(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") %while, i32 0)
@@ -64,10 +55,7 @@ define i1 @whilele_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilele_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilele pn8.d, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.d, pn8[0]
-; CHECK-NEXT:    mov z0.d, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilele.c64(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") %while, i32 0)
@@ -79,10 +67,7 @@ define i1 @whilehs_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilehs_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilehs pn8.b, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.b, pn8[0]
-; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilehs.c8(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 16 x i1> @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") %while, i32 0)
@@ -94,10 +79,7 @@ define i1 @whilehi_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilehi_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilehi pn8.h, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.h, pn8[0]
-; CHECK-NEXT:    mov z0.h, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilehi.c16(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 8 x i1> @llvm.aarch64.sve.pext.nxv8i1(target("aarch64.svcount") %while, i32 0)
@@ -109,10 +91,7 @@ define i1 @whilelo_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilelo_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.s, pn8[0]
-; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilelo.c32(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 4 x i1> @llvm.aarch64.sve.pext.nxv4i1(target("aarch64.svcount") %while, i32 0)
@@ -124,10 +103,7 @@ define i1 @whilels_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilels_first_active:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    whilels pn8.d, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.d, pn8[0]
-; CHECK-NEXT:    mov z0.d, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    and w0, w8, #0x1
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ret
   %while = call target("aarch64.svcount") @llvm.aarch64.sve.whilels.c64(i64 %a, i64 %b, i32 4)
   %pext = call <vscale x 2 x i1> @llvm.aarch64.sve.pext.nxv2i1(target("aarch64.svcount") %while, i32 0)
@@ -139,10 +115,8 @@ define void @whilege_first_active_branch(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilege_first_active_branch:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    whilege pn8.b, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.b, pn8[0]
-; CHECK-NEXT:    mov z0.b, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    tbz w8, #0, .LBB8_2
+; CHECK-NEXT:    cset w8, mi
+; CHECK-NEXT:    cbz w8, .LBB8_2
 ; CHECK-NEXT:  // %bb.1: // %then
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
@@ -166,9 +140,7 @@ define void @whilelo_first_active_branch(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilelo_first_active_branch:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    whilelo pn8.s, x0, x1, vlx4
-; CHECK-NEXT:    pext p0.s, pn8[0]
-; CHECK-NEXT:    mov z0.s, p0/z, #1 // =0x1
-; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    cset w8, mi
 ; CHECK-NEXT:    tbnz w8, #0, .LBB9_2
 ; CHECK-NEXT:  // %bb.1: // %then
 ; CHECK-NEXT:    //APP

>From 76e53e88b196693ae8653d4811ae5f1b35cbeb76 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 2 Jul 2026 17:09:44 +0000
Subject: [PATCH 3/3] Fixups

---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 31 +++++++++----------
 .../CodeGen/AArch64/sve2p1-while-pn-folds.ll  | 11 ++++---
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 278dcf99d844e..8fdd61fdb1ecd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -21970,7 +21970,7 @@ performExtractLastActiveCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                      Vec);
 }
 
-static bool hasSVE2p1OrStreamingSME2(const AArch64Subtarget *Subtarget) {
+static bool hasSVEMultiVectorOps(const AArch64Subtarget *Subtarget) {
   return (Subtarget->isSVEorStreamingSVEAvailable() &&
           Subtarget->hasSVE2p1()) ||
          (Subtarget->isStreaming() && Subtarget->hasSME2());
@@ -21992,21 +21992,21 @@ static auto m_PredicateAsCounterWhile() {
 /// predicate-as-counter while to a conditional set (CSET) based on the
 /// "FIRST_ACTIVE" status flag from the while.
 ///
-///   %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+///   %while, %flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
 ///   %first.segment = pext(%while, 0) ; predicate extract of segment 0
-///   %first.active = extract_elt(%first_segment, 0) ; extract first lane
+///   %first.active = extract_elt(%first.segment, 0) ; extract first lane
 ///
 ///   ->
 ///
-///   %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
-///   %first.active = cset(%while, FIRST_ACTIVE)
+///   %while, %flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+///   %first.active = cset(%flags, FIRST_ACTIVE)
 static SDValue
 perfomPextFirstTrueVectorCombine(SDNode *N,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const AArch64Subtarget *Subtarget) {
   using namespace llvm::SDPatternMatch;
   assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT);
-  if (DCI.isBeforeLegalize() || !hasSVE2p1OrStreamingSME2(Subtarget))
+  if (DCI.isBeforeLegalize() || !hasSVEMultiVectorOps(Subtarget))
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
@@ -22015,18 +22015,15 @@ perfomPextFirstTrueVectorCombine(SDNode *N,
   if (!VT.isScalableVectorOf(MVT::i1) || !isNullConstant(N->getOperand(1)))
     return SDValue();
 
-  SelectionDAG &DAG = DCI.DAG;
-
-  // Fold: extract_elt(pext(WHILE_*_PRED_COUNTER, 0), 0)
-  // to cset(WHILE_*_PRED_COUNTER, FIRST_ACTIVE).
-  if (sd_match(N0, m_IntrinsicWOChain<Intrinsic::aarch64_sve_pext>(
-                       m_PredicateAsCounterWhile(), m_Zero()))) {
-    SDValue WhilePredCounter = N0->getOperand(1);
-    SDValue Flags = SDValue(WhilePredCounter.getNode(), 1);
-    return getSETCC(AArch64CC::CondCode::FIRST_ACTIVE, Flags, SDLoc(N), DAG);
-  }
+  if (!sd_match(N0, m_IntrinsicWOChain<Intrinsic::aarch64_sve_pext>(
+                        m_PredicateAsCounterWhile(), m_Zero())))
+    return SDValue();
 
-  return SDValue();
+  // Fold: extract_elt(pext(WHILE_*_PRED_COUNTER:0, 0), 0)
+  // to cset(WHILE_*_PRED_COUNTER:1, FIRST_ACTIVE).
+  SDValue WhilePredCounter = N0->getOperand(1);
+  SDValue Flags = SDValue(WhilePredCounter.getNode(), 1);
+  return getSETCC(AArch64CC::CondCode::FIRST_ACTIVE, Flags, SDLoc(N), DCI.DAG);
 }
 
 static SDValue
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll b/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
index e4c0307fd0dcb..d58097f77f6d3 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-while-pn-folds.ll
@@ -4,16 +4,19 @@
 
 ; Tests extracting the first lane from the first segment of a
 ; predicate-as-counter while folds to a conditional set (CSET) based on the
-; "FIRST_ACTIVE" status flag from the while.
+; "FIRST_ACTIVE" status flags from the while.
 ;
-; %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+; %while, %flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
 ; %first.segment = pext(%while, 0) ; predicate extract of segment 0
 ; %first.active = extract_elt(%first_segment, 0) ; extract first lane
 ;
 ; ->
 ;
-; %while = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
-; %first.active = cset(%while, FIRST_ACTIVE)
+; %while, %flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while
+; %first.active = cset(%flags, FIRST_ACTIVE)
+
+; Note: FIRST_ACTIVE corresponds to the MI condition code (N == 1), and
+; not(FIRST_ACTIVE) corresponds to the PL condition code (N == 0).
 
 define i1 @whilege_first_active(i64 %a, i64 %b) {
 ; CHECK-LABEL: whilege_first_active:



More information about the llvm-commits mailing list