[llvm] [AArch64] If-convert diamonds merging a 128-bit vector (PR #220616)
Guy David via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 04:54:43 PDT 2026
https://github.com/guy-david updated https://github.com/llvm/llvm-project/pull/220616
>From 52c25cd338cece26afaa0cf1e0ef06aa0585f681 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Tue, 1 Sep 2026 14:39:21 +0300
Subject: [PATCH 1/3] [AArch64] Add tests (NFC)
---
.../CodeGen/AArch64/early-ifcvt-fpr128.ll | 154 +++++++++++++++
.../CodeGen/AArch64/early-ifcvt-fpr128.mir | 181 ++++++++++++++++++
2 files changed, 335 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll
create mode 100644 llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll
new file mode 100644
index 0000000000000..b9ce6ff00eb37
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -mtriple=aarch64 < %s | FileCheck %s
+
+define <4 x i32> @diamond_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i1 %c) {
+; CHECK-LABEL: diamond_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: tbz w0, #0, .LBB0_2
+; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: neg v0.4s, v0.4s
+; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB0_2: // %else
+; CHECK-NEXT: sub v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: ret
+entry:
+ br i1 %c, label %then, label %else
+then:
+ %t1 = add <4 x i32> %a, %b
+ %t2 = mul <4 x i32> %t1, %k
+ %t3 = sub <4 x i32> %t2, %a
+ br label %join
+else:
+ %e1 = sub <4 x i32> %a, %b
+ %e2 = mul <4 x i32> %e1, %k
+ %e3 = add <4 x i32> %e2, %a
+ br label %join
+join:
+ %p = phi <4 x i32> [ %t3, %then ], [ %e3, %else ]
+ ret <4 x i32> %p
+}
+
+define <4 x i16> @diamond_v4i16_fpr64(<4 x i16> %a, <4 x i16> %b, <4 x i16> %k, i1 %c) {
+; CHECK-LABEL: diamond_v4i16_fpr64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sub v3.4h, v0.4h, v1.4h
+; CHECK-NEXT: add v1.4h, v0.4h, v1.4h
+; CHECK-NEXT: tst w0, #0x1
+; CHECK-NEXT: neg v4.4h, v0.4h
+; CHECK-NEXT: mla v0.4h, v3.4h, v2.4h
+; CHECK-NEXT: mla v4.4h, v1.4h, v2.4h
+; CHECK-NEXT: fcsel d0, d0, d4, eq
+; CHECK-NEXT: ret
+entry:
+ br i1 %c, label %then, label %else
+then:
+ %t1 = add <4 x i16> %a, %b
+ %t2 = mul <4 x i16> %t1, %k
+ %t3 = sub <4 x i16> %t2, %a
+ br label %join
+else:
+ %e1 = sub <4 x i16> %a, %b
+ %e2 = mul <4 x i16> %e1, %k
+ %e3 = add <4 x i16> %e2, %a
+ br label %join
+join:
+ %p = phi <4 x i16> [ %t3, %then ], [ %e3, %else ]
+ ret <4 x i16> %p
+}
+
+define <4 x i32> @diamond_slt_cond_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i32 %x, i32 %y) {
+; CHECK-LABEL: diamond_slt_cond_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cmp w0, w1
+; CHECK-NEXT: b.ge .LBB2_2
+; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: neg v0.4s, v0.4s
+; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB2_2: // %else
+; CHECK-NEXT: sub v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: ret
+entry:
+ %c = icmp slt i32 %x, %y
+ br i1 %c, label %then, label %else
+then:
+ %t1 = add <4 x i32> %a, %b
+ %t2 = mul <4 x i32> %t1, %k
+ %t3 = sub <4 x i32> %t2, %a
+ br label %join
+else:
+ %e1 = sub <4 x i32> %a, %b
+ %e2 = mul <4 x i32> %e1, %k
+ %e3 = add <4 x i32> %e2, %a
+ br label %join
+join:
+ %p = phi <4 x i32> [ %t3, %then ], [ %e3, %else ]
+ ret <4 x i32> %p
+}
+
+define <4 x i32> @diamond_streaming_stays_branchy(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i1 %c) "aarch64_pstate_sm_enabled" "target-features"="+sme" {
+; CHECK-LABEL: diamond_streaming_stays_branchy:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: tbz w0, #0, .LBB3_2
+; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: add z1.s, z0.s, z1.s
+; CHECK-NEXT: mul z1.s, z1.s, z2.s
+; CHECK-NEXT: sub z0.s, z1.s, z0.s
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB3_2: // %else
+; CHECK-NEXT: sub z1.s, z0.s, z1.s
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: mla z0.s, p0/m, z1.s, z2.s
+; CHECK-NEXT: ret
+entry:
+ br i1 %c, label %then, label %else
+then:
+ %t1 = add <4 x i32> %a, %b
+ %t2 = mul <4 x i32> %t1, %k
+ %t3 = sub <4 x i32> %t2, %a
+ br label %join
+else:
+ %e1 = sub <4 x i32> %a, %b
+ %e2 = mul <4 x i32> %e1, %k
+ %e3 = add <4 x i32> %e2, %a
+ br label %join
+join:
+ %p = phi <4 x i32> [ %t3, %then ], [ %e3, %else ]
+ ret <4 x i32> %p
+}
+
+define <4 x i32> @diamond_streaming_fa64_converts(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i1 %c) "aarch64_pstate_sm_enabled" "target-features"="+sme,+sme-fa64" {
+; CHECK-LABEL: diamond_streaming_fa64_converts:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: tbz w0, #0, .LBB4_2
+; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: add v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: neg v0.4s, v0.4s
+; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB4_2: // %else
+; CHECK-NEXT: sub v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: ret
+entry:
+ br i1 %c, label %then, label %else
+then:
+ %t1 = add <4 x i32> %a, %b
+ %t2 = mul <4 x i32> %t1, %k
+ %t3 = sub <4 x i32> %t2, %a
+ br label %join
+else:
+ %e1 = sub <4 x i32> %a, %b
+ %e2 = mul <4 x i32> %e1, %k
+ %e3 = add <4 x i32> %e2, %a
+ br label %join
+join:
+ %p = phi <4 x i32> [ %t3, %then ], [ %e3, %else ]
+ ret <4 x i32> %p
+}
+
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
new file mode 100644
index 0000000000000..af16822b7e303
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
@@ -0,0 +1,181 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -run-pass=early-ifcvt -stress-early-ifcvt %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -passes=early-ifcvt -stress-early-ifcvt %s -o - | FileCheck %s
+
+--- |
+ target triple = "aarch64"
+
+ define <4 x i32> @diamond(<4 x i32> %a, <4 x i32> %b, i1 %c) {
+ unreachable
+ }
+
+ define <4 x i32> @triangle(<4 x i32> %a, <4 x i32> %b, i1 %c) {
+ unreachable
+ }
+
+ define <4 x i32> @diamond_minsize(<4 x i32> %a, <4 x i32> %b, i1 %c) minsize {
+ unreachable
+ }
+...
+---
+name: diamond
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: diamond
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $q0, $q1, $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
+ ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBv4i32_:%[0-9]+]]:fpr128 = SUBv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[SUBv4i32_]], %bb.2
+ ; CHECK-NEXT: $q0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $q0, $q1, $w0
+
+ %0:fpr128 = COPY $q0
+ %1:fpr128 = COPY $q1
+ %2:gpr32 = COPY $w0
+ TBZW %2, 0, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.3
+
+ %3:fpr128 = ADDv4i32 %0, %1
+ B %bb.3
+
+ bb.2:
+ successors: %bb.3
+
+ %4:fpr128 = SUBv4i32 %0, %1
+
+ bb.3:
+ %5:fpr128 = PHI %3, %bb.1, %4, %bb.2
+ $q0 = COPY %5
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: triangle
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: triangle
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $q0, $q1, $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
+ ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[COPY]], %bb.0
+ ; CHECK-NEXT: $q0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $q0, $q1, $w0
+
+ %0:fpr128 = COPY $q0
+ %1:fpr128 = COPY $q1
+ %2:gpr32 = COPY $w0
+ TBZW %2, 0, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.2
+
+ %3:fpr128 = ADDv4i32 %0, %1
+
+ bb.2:
+ %4:fpr128 = PHI %3, %bb.1, %0, %bb.0
+ $q0 = COPY %4
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: diamond_minsize
+alignment: 4
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: diamond_minsize
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $q0, $q1, $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
+ ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBv4i32_:%[0-9]+]]:fpr128 = SUBv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[SUBv4i32_]], %bb.2
+ ; CHECK-NEXT: $q0 = COPY [[PHI]]
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $q0, $q1, $w0
+
+ %0:fpr128 = COPY $q0
+ %1:fpr128 = COPY $q1
+ %2:gpr32 = COPY $w0
+ TBZW %2, 0, %bb.2
+ B %bb.1
+
+ bb.1:
+ successors: %bb.3
+
+ %3:fpr128 = ADDv4i32 %0, %1
+ B %bb.3
+
+ bb.2:
+ successors: %bb.3
+
+ %4:fpr128 = SUBv4i32 %0, %1
+
+ bb.3:
+ %5:fpr128 = PHI %3, %bb.1, %4, %bb.2
+ $q0 = COPY %5
+ RET_ReallyLR implicit $q0
+
+...
>From b648f601301fed34bc7172fd509422d1b9a8d8d3 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Tue, 1 Sep 2026 14:39:24 +0300
Subject: [PATCH 2/3] [AArch64] If-convert diamonds merging a 128-bit vector
canInsertSelect handles GPRs via csel and 32/64-bit FPRs via fcsel, but
returns false for everything else. For example, EarlyIfConversion
flattens a diamond merging a v4i16 but leaves the equivalent v4i32
branchy.
We can create a conditional select similarly to how vector selects are
lowered in the backend by spreading the condition into an all-1's /
all-0's mask and use a bit select.
Assisted-by: Opus 5.0
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 28 +++++++++
.../CodeGen/AArch64/early-ifcvt-fpr128.ll | 49 +++++++--------
.../CodeGen/AArch64/early-ifcvt-fpr128.mir | 62 +++++--------------
3 files changed, 69 insertions(+), 70 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index fe0c59622e17b..b30f5861d05e3 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1067,6 +1067,15 @@ bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
return true;
}
+ // No single conditional move for a 128-bit vector, but we can emit a sequence
+ // of csetm (~1), dup (~5, cross domain), bsl (~2).
+ if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
+ Subtarget.isNeonAvailable()) {
+ CondCycles = 8 + ExtraCondLat;
+ TrueCycles = FalseCycles = 2;
+ return true;
+ }
+
// Can't do vectors.
return false;
}
@@ -1279,6 +1288,25 @@ void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
} break;
}
+ // A 128-bit vector has no conditional move so blend the operands with a mask
+ // built from the flags.
+ if (MRI.constrainRegClass(DstReg, &AArch64::FPR128RegClass)) {
+ MRI.constrainRegClass(TrueReg, &AArch64::FPR128RegClass);
+ MRI.constrainRegClass(FalseReg, &AArch64::FPR128RegClass);
+ Register CondSet = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
+ BuildMI(MBB, I, DL, get(AArch64::CSINVXr), CondSet)
+ .addReg(AArch64::XZR)
+ .addReg(AArch64::XZR)
+ .addImm(AArch64CC::getInvertedCondCode(CC));
+ Register Mask = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
+ BuildMI(MBB, I, DL, get(AArch64::DUPv2i64gpr), Mask).addReg(CondSet);
+ BuildMI(MBB, I, DL, get(AArch64::BSPv16i8), DstReg)
+ .addReg(Mask)
+ .addReg(TrueReg)
+ .addReg(FalseReg);
+ return;
+ }
+
unsigned Opc = 0;
const TargetRegisterClass *RC = nullptr;
bool TryFold = false;
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll
index b9ce6ff00eb37..95ad44260df18 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.ll
@@ -4,15 +4,15 @@
define <4 x i32> @diamond_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i1 %c) {
; CHECK-LABEL: diamond_v4i32:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: tbz w0, #0, .LBB0_2
-; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: sub v3.4s, v0.4s, v1.4s
; CHECK-NEXT: add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT: neg v0.4s, v0.4s
-; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB0_2: // %else
-; CHECK-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: tst w0, #0x1
+; CHECK-NEXT: neg v4.4s, v0.4s
+; CHECK-NEXT: csetm x8, eq
+; CHECK-NEXT: mla v0.4s, v3.4s, v2.4s
+; CHECK-NEXT: mla v4.4s, v1.4s, v2.4s
+; CHECK-NEXT: dup v1.2d, x8
+; CHECK-NEXT: bif v0.16b, v4.16b, v1.16b
; CHECK-NEXT: ret
entry:
br i1 %c, label %then, label %else
@@ -62,16 +62,15 @@ join:
define <4 x i32> @diamond_slt_cond_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i32 %x, i32 %y) {
; CHECK-LABEL: diamond_slt_cond_v4i32:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: cmp w0, w1
-; CHECK-NEXT: b.ge .LBB2_2
-; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: sub v3.4s, v0.4s, v1.4s
; CHECK-NEXT: add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT: neg v0.4s, v0.4s
-; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB2_2: // %else
-; CHECK-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: cmp w0, w1
+; CHECK-NEXT: neg v4.4s, v0.4s
+; CHECK-NEXT: csetm x8, ge
+; CHECK-NEXT: mla v0.4s, v3.4s, v2.4s
+; CHECK-NEXT: mla v4.4s, v1.4s, v2.4s
+; CHECK-NEXT: dup v1.2d, x8
+; CHECK-NEXT: bif v0.16b, v4.16b, v1.16b
; CHECK-NEXT: ret
entry:
%c = icmp slt i32 %x, %y
@@ -125,15 +124,15 @@ join:
define <4 x i32> @diamond_streaming_fa64_converts(<4 x i32> %a, <4 x i32> %b, <4 x i32> %k, i1 %c) "aarch64_pstate_sm_enabled" "target-features"="+sme,+sme-fa64" {
; CHECK-LABEL: diamond_streaming_fa64_converts:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: tbz w0, #0, .LBB4_2
-; CHECK-NEXT: // %bb.1: // %then
+; CHECK-NEXT: sub v3.4s, v0.4s, v1.4s
; CHECK-NEXT: add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT: neg v0.4s, v0.4s
-; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
-; CHECK-NEXT: ret
-; CHECK-NEXT: .LBB4_2: // %else
-; CHECK-NEXT: sub v1.4s, v0.4s, v1.4s
-; CHECK-NEXT: mla v0.4s, v1.4s, v2.4s
+; CHECK-NEXT: tst w0, #0x1
+; CHECK-NEXT: neg v4.4s, v0.4s
+; CHECK-NEXT: csetm x8, eq
+; CHECK-NEXT: mla v0.4s, v3.4s, v2.4s
+; CHECK-NEXT: mla v4.4s, v1.4s, v2.4s
+; CHECK-NEXT: dup v1.2d, x8
+; CHECK-NEXT: bif v0.16b, v4.16b, v1.16b
; CHECK-NEXT: ret
entry:
br i1 %c, label %then, label %else
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
index af16822b7e303..e9fcb0787438a 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
@@ -24,29 +24,18 @@ tracksRegLiveness: true
body: |
; CHECK-LABEL: name: diamond
; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $q0, $q1, $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
- ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.3(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x80000000)
- ; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[SUBv4i32_:%[0-9]+]]:fpr128 = SUBv4i32 [[COPY]], [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[SUBv4i32_]], %bb.2
- ; CHECK-NEXT: $q0 = COPY [[PHI]]
+ ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: $wzr = ANDSWri [[COPY2]], 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[CSINVXr:%[0-9]+]]:gpr64 = CSINVXr $xzr, $xzr, 1, implicit $nzcv
+ ; CHECK-NEXT: [[DUPv2i64gpr:%[0-9]+]]:fpr128 = DUPv2i64gpr [[CSINVXr]]
+ ; CHECK-NEXT: [[BSPv16i8_:%[0-9]+]]:fpr128 = BSPv16i8 [[DUPv2i64gpr]], [[SUBv4i32_]], [[ADDv4i32_]]
+ ; CHECK-NEXT: $q0 = COPY [[BSPv16i8_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
bb.0:
successors: %bb.1, %bb.2
@@ -82,23 +71,17 @@ tracksRegLiveness: true
body: |
; CHECK-LABEL: name: triangle
; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $q0, $q1, $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
- ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[COPY]], %bb.0
- ; CHECK-NEXT: $q0 = COPY [[PHI]]
+ ; CHECK-NEXT: $wzr = ANDSWri [[COPY2]], 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[CSINVXr:%[0-9]+]]:gpr64 = CSINVXr $xzr, $xzr, 1, implicit $nzcv
+ ; CHECK-NEXT: [[DUPv2i64gpr:%[0-9]+]]:fpr128 = DUPv2i64gpr [[CSINVXr]]
+ ; CHECK-NEXT: [[BSPv16i8_:%[0-9]+]]:fpr128 = BSPv16i8 [[DUPv2i64gpr]], [[COPY]], [[ADDv4i32_]]
+ ; CHECK-NEXT: $q0 = COPY [[BSPv16i8_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
bb.0:
successors: %bb.1, %bb.2
@@ -128,29 +111,18 @@ tracksRegLiveness: true
body: |
; CHECK-LABEL: name: diamond_minsize
; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $q0, $q1, $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
- ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
- ; CHECK-NEXT: B %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.3(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
- ; CHECK-NEXT: B %bb.3
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: successors: %bb.3(0x80000000)
- ; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[SUBv4i32_:%[0-9]+]]:fpr128 = SUBv4i32 [[COPY]], [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.3:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[SUBv4i32_]], %bb.2
- ; CHECK-NEXT: $q0 = COPY [[PHI]]
+ ; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: $wzr = ANDSWri [[COPY2]], 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[CSINVXr:%[0-9]+]]:gpr64 = CSINVXr $xzr, $xzr, 1, implicit $nzcv
+ ; CHECK-NEXT: [[DUPv2i64gpr:%[0-9]+]]:fpr128 = DUPv2i64gpr [[CSINVXr]]
+ ; CHECK-NEXT: [[BSPv16i8_:%[0-9]+]]:fpr128 = BSPv16i8 [[DUPv2i64gpr]], [[SUBv4i32_]], [[ADDv4i32_]]
+ ; CHECK-NEXT: $q0 = COPY [[BSPv16i8_]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
bb.0:
successors: %bb.1, %bb.2
>From 29fcb0afc73ca81be8152e2c684a5c3a4f9a2668 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Thu, 10 Sep 2026 12:42:23 +0300
Subject: [PATCH 3/3] Address comments 1
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 5 ++--
.../CodeGen/AArch64/early-ifcvt-fpr128.mir | 23 ++++++++++++++-----
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index b30f5861d05e3..56a6d0f53a3b9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1070,13 +1070,13 @@ bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
// No single conditional move for a 128-bit vector, but we can emit a sequence
// of csetm (~1), dup (~5, cross domain), bsl (~2).
if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
- Subtarget.isNeonAvailable()) {
+ Subtarget.isNeonAvailable() &&
+ !MBB.getParent()->getFunction().hasMinSize()) {
CondCycles = 8 + ExtraCondLat;
TrueCycles = FalseCycles = 2;
return true;
}
- // Can't do vectors.
return false;
}
@@ -1291,6 +1291,7 @@ void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
// A 128-bit vector has no conditional move so blend the operands with a mask
// built from the flags.
if (MRI.constrainRegClass(DstReg, &AArch64::FPR128RegClass)) {
+ assert(Subtarget.isNeonAvailable() && "Expected NEON for a vector select");
MRI.constrainRegClass(TrueReg, &AArch64::FPR128RegClass);
MRI.constrainRegClass(FalseReg, &AArch64::FPR128RegClass);
Register CondSet = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
index e9fcb0787438a..8a276ed8ca76f 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-fpr128.mir
@@ -111,18 +111,29 @@ tracksRegLiveness: true
body: |
; CHECK-LABEL: name: diamond_minsize
; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: liveins: $q0, $q1, $w0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
- ; CHECK-NEXT: [[SUBv4i32_:%[0-9]+]]:fpr128 = SUBv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: TBZW [[COPY2]], 0, %bb.2
+ ; CHECK-NEXT: B %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[ADDv4i32_:%[0-9]+]]:fpr128 = ADDv4i32 [[COPY]], [[COPY1]]
- ; CHECK-NEXT: $wzr = ANDSWri [[COPY2]], 0, implicit-def $nzcv
- ; CHECK-NEXT: [[CSINVXr:%[0-9]+]]:gpr64 = CSINVXr $xzr, $xzr, 1, implicit $nzcv
- ; CHECK-NEXT: [[DUPv2i64gpr:%[0-9]+]]:fpr128 = DUPv2i64gpr [[CSINVXr]]
- ; CHECK-NEXT: [[BSPv16i8_:%[0-9]+]]:fpr128 = BSPv16i8 [[DUPv2i64gpr]], [[SUBv4i32_]], [[ADDv4i32_]]
- ; CHECK-NEXT: $q0 = COPY [[BSPv16i8_]]
+ ; CHECK-NEXT: B %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[SUBv4i32_:%[0-9]+]]:fpr128 = SUBv4i32 [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:fpr128 = PHI [[ADDv4i32_]], %bb.1, [[SUBv4i32_]], %bb.2
+ ; CHECK-NEXT: $q0 = COPY [[PHI]]
; CHECK-NEXT: RET_ReallyLR implicit $q0
bb.0:
successors: %bb.1, %bb.2
More information about the llvm-commits
mailing list