[llvm] [AArch64] Use SVE for load-extend-broadcast (PR #221689)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 07:07:16 PDT 2026
https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/221689
>From b9ae9f11f17222b2bbe1ff29b585e580f9c38d67 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Fri, 4 Sep 2026 11:39:59 +0100
Subject: [PATCH 1/3] [AArch64] Use SVE for load-extend-broadcast on V3AE
On Neoverse-V3AE, it is faster to use SVE ld1r instructions to
load, extend and broadcast an integer value to a vector in all
cases other than 64bit vectors where no type promotion occurs.
To acheive this, introduce a tuning option, `PreferSVEVectors`
for use when combining DUP instructions to create a Splat Vector to
the scalable alternative, and then use the already implemented V3AE
scheduling model pattern for splat vectors to end up generate the
ld1r instruction.
If used in a loop, the predicate will be hoisted outside of the loop.
---
llvm/lib/Target/AArch64/AArch64Features.td | 4 +
.../Target/AArch64/AArch64ISelLowering.cpp | 17 +-
llvm/lib/Target/AArch64/AArch64Processors.td | 3 +-
.../CodeGen/AArch64/dup-ext-load-combine.ll | 332 ++++++++++++++----
4 files changed, 287 insertions(+), 69 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 6eea064b257f6..cd03583c7f4e5 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -1022,6 +1022,10 @@ def FeatureLimited64bitVectorMulBandwidth : SubtargetFeature<
"Has limited 64bit vector multiply bandwidth compared to scalar multiply",
[], InlineIgnore>;
+def FeaturePreferSVEVectors : SubtargetFeature <
+ "prefer vectors to be SVE", "PreferSVEVectors", "true",
+ "Prefer Vectors to use SVE", [], InlineIgnore>;
+
//===----------------------------------------------------------------------===//
// Architectures.
//
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 22198cd122fc7..568d7b581370d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -30117,7 +30117,8 @@ static SDValue performSelectCombine(SDNode *N,
}
static SDValue performDUPCombine(SDNode *N,
- TargetLowering::DAGCombinerInfo &DCI) {
+ TargetLowering::DAGCombinerInfo &DCI,
+ const AArch64Subtarget *SubTarget) {
EVT VT = N->getValueType(0);
SDLoc DL(N);
// If "v2i32 DUP(x)" and "v4i32 DUP(x)" both exist, use an extract from the
@@ -30141,6 +30142,18 @@ static SDValue performDUPCombine(SDNode *N,
// v4i32 = SCALAR_TO_VECTOR (i32 (zextloadi8 addr)) ; Matches to ldr b0
// v4i32 = DUPLANE32 (v4i32), 0
if (auto *LD = dyn_cast<LoadSDNode>(Op)) {
+ if (SubTarget->preferSVEVectors() &&
+ SubTarget->isSVEorStreamingSVEAvailable() && Op->hasOneUse() &&
+ VT.getScalarType().isInteger()) {
+ EVT ScalaleVT = getContainerForFixedLengthVector(DCI.DAG, VT);
+ SDValue SplatNode =
+ DCI.DAG.getNode(ISD::SPLAT_VECTOR, DL, ScalaleVT, Op);
+ // Using SVE on 64bit Vectors with the same scalar type is not
+ // profitable
+ if (!(VT.is64BitVector() && SplatNode.getValueType().getScalarType() ==
+ LD->getMemoryVT().getScalarType()))
+ return convertFromScalableVector(DCI.DAG, VT, SplatNode);
+ }
ISD::LoadExtType ExtType = LD->getExtensionType();
EVT MemVT = LD->getMemoryVT();
EVT ElemVT = VT.getVectorElementType();
@@ -31626,7 +31639,7 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
case AArch64ISD::DUPLANE16:
case AArch64ISD::DUPLANE32:
case AArch64ISD::DUPLANE64:
- return performDUPCombine(N, DCI);
+ return performDUPCombine(N, DCI, Subtarget);
case AArch64ISD::DUPLANE128:
return performDupLane128Combine(N, DAG);
case AArch64ISD::NVCAST:
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index f996522ba4d09..5b1411501a63e 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -618,7 +618,8 @@ def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "Neover
FeatureUseFixedOverScalableIfEqualCost,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive,
- FeatureLimited64bitVectorMulBandwidth]>;
+ FeatureLimited64bitVectorMulBandwidth,
+ FeaturePreferSVEVectors]>;
def TuneSaphira : SubtargetFeature<"saphira", "ARMProcFamily", "Saphira",
"Qualcomm Saphira processors", [
diff --git a/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll b/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll
index cf529343c6728..cb41f89574cea 100644
--- a/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll
@@ -1,15 +1,24 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v3ae -mattr=-sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v3ae < %s | FileCheck %s --check-prefixes=CHECK,CHECK-V3AE
; Test optimization of DUP with extended narrow loads
; This should avoid GPR->SIMD transfers by loading directly into vector registers
define <4 x i16> @test_dup_zextload_i8_v4i16(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i8_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0]
-; CHECK-NEXT: dup v0.4h, v0.h[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v4i16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0]
+; CHECK-BASE-NEXT: dup v0.4h, v0.h[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i16:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.h
+; CHECK-V3AE-NEXT: ld1rb { z0.h }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i16
%vec = insertelement <4 x i16> poison, i16 %ext, i32 0
@@ -18,11 +27,18 @@ define <4 x i16> @test_dup_zextload_i8_v4i16(ptr %p) {
}
define <8 x i16> @test_dup_zextload_i8_v8i16(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i8_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0]
-; CHECK-NEXT: dup v0.8h, v0.h[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v8i16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0]
+; CHECK-BASE-NEXT: dup v0.8h, v0.h[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v8i16:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.h
+; CHECK-V3AE-NEXT: ld1rb { z0.h }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i16
%vec = insertelement <8 x i16> poison, i16 %ext, i32 0
@@ -31,11 +47,18 @@ define <8 x i16> @test_dup_zextload_i8_v8i16(ptr %p) {
}
define <2 x i32> @test_dup_zextload_i8_v2i32(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i8_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0]
-; CHECK-NEXT: dup v0.2s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v2i32:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0]
+; CHECK-BASE-NEXT: dup v0.2s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v2i32:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i32
%vec = insertelement <2 x i32> poison, i32 %ext, i32 0
@@ -44,11 +67,18 @@ define <2 x i32> @test_dup_zextload_i8_v2i32(ptr %p) {
}
define <4 x i32> @test_dup_zextload_i8_v4i32(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i8_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0]
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v4i32:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0]
+; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i32:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i32
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
@@ -57,11 +87,18 @@ define <4 x i32> @test_dup_zextload_i8_v4i32(ptr %p) {
}
define <4 x i32> @test_dup_zextload_i8_v4i32_offset(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i8_v4i32_offset:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0, #4]
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v4i32_offset:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0, #4]
+; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i32_offset:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x0, #4]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%addr = getelementptr inbounds i8, ptr %p, i64 4
%load = load i8, ptr %addr, align 1
%ext = zext i8 %load to i32
@@ -71,11 +108,19 @@ define <4 x i32> @test_dup_zextload_i8_v4i32_offset(ptr %p) {
}
define <4 x i32> @test_dup_zextload_i8_v4i32_reg_offset(ptr %p, i64 %offset) {
-; CHECK-LABEL: test_dup_zextload_i8_v4i32_reg_offset:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0, x1]
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v4i32_reg_offset:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0, x1]
+; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i32_reg_offset:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: add x8, x0, x1
+; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x8]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%addr = getelementptr inbounds i8, ptr %p, i64 %offset
%load = load i8, ptr %addr, align 1
%ext = zext i8 %load to i32
@@ -85,11 +130,18 @@ define <4 x i32> @test_dup_zextload_i8_v4i32_reg_offset(ptr %p, i64 %offset) {
}
define <2 x i64> @test_dup_zextload_i8_v2i64(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i8_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr b0, [x0]
-; CHECK-NEXT: dup v0.2d, v0.d[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i8_v2i64:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr b0, [x0]
+; CHECK-BASE-NEXT: dup v0.2d, v0.d[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i8_v2i64:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.d
+; CHECK-V3AE-NEXT: ld1rb { z0.d }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i64
%vec = insertelement <2 x i64> poison, i64 %ext, i32 0
@@ -98,11 +150,18 @@ define <2 x i64> @test_dup_zextload_i8_v2i64(ptr %p) {
}
define <2 x i32> @test_dup_zextload_i16_v2i32(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i16_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr h0, [x0]
-; CHECK-NEXT: dup v0.2s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i16_v2i32:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr h0, [x0]
+; CHECK-BASE-NEXT: dup v0.2s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i16_v2i32:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i16, ptr %p, align 1
%ext = zext i16 %load to i32
%vec = insertelement <2 x i32> poison, i32 %ext, i32 0
@@ -111,11 +170,18 @@ define <2 x i32> @test_dup_zextload_i16_v2i32(ptr %p) {
}
define <4 x i32> @test_dup_zextload_i16_v4i32(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i16_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr h0, [x0]
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i16_v4i32:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr h0, [x0]
+; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i16_v4i32:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i16, ptr %p, align 1
%ext = zext i16 %load to i32
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
@@ -124,11 +190,18 @@ define <4 x i32> @test_dup_zextload_i16_v4i32(ptr %p) {
}
define <4 x i32> @test_dup_zextload_i16_v4i32_offset(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i16_v4i32_offset:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr h0, [x0, #8]
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i16_v4i32_offset:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr h0, [x0, #8]
+; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i16_v4i32_offset:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x0, #8]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%addr = getelementptr inbounds i16, ptr %p, i64 4
%load = load i16, ptr %addr, align 1
%ext = zext i16 %load to i32
@@ -138,11 +211,19 @@ define <4 x i32> @test_dup_zextload_i16_v4i32_offset(ptr %p) {
}
define <4 x i32> @test_dup_zextload_i16_v4i32_reg_offset(ptr %p, i64 %offset) {
-; CHECK-LABEL: test_dup_zextload_i16_v4i32_reg_offset:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]
-; CHECK-NEXT: dup v0.4s, v0.s[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i16_v4i32_reg_offset:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr h0, [x0, x1, lsl #1]
+; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i16_v4i32_reg_offset:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: add x8, x0, x1, lsl #1
+; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x8]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%addr = getelementptr inbounds i16, ptr %p, i64 %offset
%load = load i16, ptr %addr, align 1
%ext = zext i16 %load to i32
@@ -152,11 +233,18 @@ define <4 x i32> @test_dup_zextload_i16_v4i32_reg_offset(ptr %p, i64 %offset) {
}
define <2 x i64> @test_dup_zextload_i16_v2i64(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i16_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr h0, [x0]
-; CHECK-NEXT: dup v0.2d, v0.d[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i16_v2i64:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr h0, [x0]
+; CHECK-BASE-NEXT: dup v0.2d, v0.d[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i16_v2i64:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.d
+; CHECK-V3AE-NEXT: ld1rh { z0.d }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i16, ptr %p, align 1
%ext = zext i16 %load to i64
%vec = insertelement <2 x i64> poison, i64 %ext, i32 0
@@ -165,14 +253,126 @@ define <2 x i64> @test_dup_zextload_i16_v2i64(ptr %p) {
}
define <2 x i64> @test_dup_zextload_i32_v2i64(ptr %p) {
-; CHECK-LABEL: test_dup_zextload_i32_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldr s0, [x0]
-; CHECK-NEXT: dup v0.2d, v0.d[0]
-; CHECK-NEXT: ret
+; CHECK-BASE-LABEL: test_dup_zextload_i32_v2i64:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr s0, [x0]
+; CHECK-BASE-NEXT: dup v0.2d, v0.d[0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_zextload_i32_v2i64:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.d
+; CHECK-V3AE-NEXT: ld1rw { z0.d }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
%load = load i32, ptr %p, align 1
%ext = zext i32 %load to i64
%vec = insertelement <2 x i64> poison, i64 %ext, i32 0
%dup = shufflevector <2 x i64> %vec, <2 x i64> poison, <2 x i32> zeroinitializer
ret <2 x i64> %dup
}
+
+define <16 x i8> @test_dup_load_i8_v16i8(ptr %p) {
+; CHECK-BASE-LABEL: test_dup_load_i8_v16i8:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ld1r { v0.16b }, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_load_i8_v16i8:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.b
+; CHECK-V3AE-NEXT: ld1rb { z0.b }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
+ %load = load i8, ptr %p, align 1
+ %vec = insertelement <16 x i8> poison, i8 %load, i64 0
+ %dup = shufflevector <16 x i8> %vec, <16 x i8> poison, <16 x i32> zeroinitializer
+ ret <16 x i8> %dup
+}
+
+define <8 x i8> @test_dup_load_i8_v8i8(ptr %p) {
+; CHECK-LABEL: test_dup_load_i8_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.8b }, [x0]
+; CHECK-NEXT: ret
+ %load = load i8, ptr %p, align 1
+ %vec = insertelement <8 x i8> poison, i8 %load, i64 0
+ %dup = shufflevector <8 x i8> %vec, <8 x i8> poison, <8 x i32> zeroinitializer
+ ret <8 x i8> %dup
+}
+
+define <8 x i16> @test_dup_load_i16_v8i16(ptr %p) {
+; CHECK-BASE-LABEL: test_dup_load_i16_v8i16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ld1r { v0.8h }, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_load_i16_v8i16:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.h
+; CHECK-V3AE-NEXT: ld1rh { z0.h }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
+ %load = load i16, ptr %p, align 1
+ %vec = insertelement <8 x i16> poison, i16 %load, i64 0
+ %dup = shufflevector <8 x i16> %vec, <8 x i16> poison, <8 x i32> zeroinitializer
+ ret <8 x i16> %dup
+}
+
+define <4 x i16> @test_dup_load_i16_v4i16(ptr %p) {
+; CHECK-LABEL: test_dup_load_i16_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.4h }, [x0]
+; CHECK-NEXT: ret
+ %load = load i16, ptr %p, align 1
+ %vec = insertelement <4 x i16> poison, i16 %load, i64 0
+ %dup = shufflevector <4 x i16> %vec, <4 x i16> poison, <4 x i32> zeroinitializer
+ ret <4 x i16> %dup
+}
+
+define <4 x i32> @test_dup_load_i32_v4i32(ptr %p) {
+; CHECK-BASE-LABEL: test_dup_load_i32_v4i32:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ld1r { v0.4s }, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_load_i32_v4i32:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.s
+; CHECK-V3AE-NEXT: ld1rw { z0.s }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
+ %load = load i32, ptr %p, align 1
+ %vec = insertelement <4 x i32> poison, i32 %load, i64 0
+ %dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
+ ret <4 x i32> %dup
+}
+
+define <2 x i32> @test_dup_load_i32_v2i32(ptr %p) {
+; CHECK-LABEL: test_dup_load_i32_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.2s }, [x0]
+; CHECK-NEXT: ret
+ %load = load i32, ptr %p, align 1
+ %vec = insertelement <2 x i32> poison, i32 %load, i64 0
+ %dup = shufflevector <2 x i32> %vec, <2 x i32> poison, <2 x i32> zeroinitializer
+ ret <2 x i32> %dup
+}
+
+define <2 x i64> @test_dup_load_i64_v2i64(ptr %p) {
+; CHECK-BASE-LABEL: test_dup_load_i64_v2i64:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ld1r { v0.2d }, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-V3AE-LABEL: test_dup_load_i64_v2i64:
+; CHECK-V3AE: // %bb.0:
+; CHECK-V3AE-NEXT: ptrue p0.d
+; CHECK-V3AE-NEXT: ld1rd { z0.d }, p0/z, [x0]
+; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-V3AE-NEXT: ret
+ %load = load i64, ptr %p, align 1
+ %vec = insertelement <2 x i64> poison, i64 %load, i64 0
+ %dup = shufflevector <2 x i64> %vec, <2 x i64> poison, <2 x i32> zeroinitializer
+ ret <2 x i64> %dup
+}
>From 3ee538e36d771a62232f24cc66ee12d2b3198528 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 7 Sep 2026 14:21:29 +0100
Subject: [PATCH 2/3] Respond to comments
- Update RUN lines to just add 1 extra
- Expand new feature to Neoverse V3
- Fixups based off comments
- exclude all vectors where scalar types match
- typos
- moved if condition so node is not created until we know we want
to perform the transformation.
---
llvm/lib/Target/AArch64/AArch64Features.td | 2 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 20 +-
llvm/lib/Target/AArch64/AArch64Processors.td | 3 +-
.../CodeGen/AArch64/dup-ext-load-combine.ll | 223 ++++++++----------
4 files changed, 110 insertions(+), 138 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index cd03583c7f4e5..b01a4c9a8a777 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -1023,7 +1023,7 @@ def FeatureLimited64bitVectorMulBandwidth : SubtargetFeature<
[], InlineIgnore>;
def FeaturePreferSVEVectors : SubtargetFeature <
- "prefer vectors to be SVE", "PreferSVEVectors", "true",
+ "prefer-sve-vectors", "PreferSVEVectors", "true",
"Prefer Vectors to use SVE", [], InlineIgnore>;
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 568d7b581370d..fb5dbdaecbee3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -30118,7 +30118,7 @@ static SDValue performSelectCombine(SDNode *N,
static SDValue performDUPCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
- const AArch64Subtarget *SubTarget) {
+ const AArch64Subtarget *Subtarget) {
EVT VT = N->getValueType(0);
SDLoc DL(N);
// If "v2i32 DUP(x)" and "v4i32 DUP(x)" both exist, use an extract from the
@@ -30142,17 +30142,17 @@ static SDValue performDUPCombine(SDNode *N,
// v4i32 = SCALAR_TO_VECTOR (i32 (zextloadi8 addr)) ; Matches to ldr b0
// v4i32 = DUPLANE32 (v4i32), 0
if (auto *LD = dyn_cast<LoadSDNode>(Op)) {
- if (SubTarget->preferSVEVectors() &&
- SubTarget->isSVEorStreamingSVEAvailable() && Op->hasOneUse() &&
+ if (Subtarget->preferSVEVectors() &&
+ Subtarget->isSVEorStreamingSVEAvailable() && Op->hasOneUse() &&
VT.getScalarType().isInteger()) {
- EVT ScalaleVT = getContainerForFixedLengthVector(DCI.DAG, VT);
- SDValue SplatNode =
- DCI.DAG.getNode(ISD::SPLAT_VECTOR, DL, ScalaleVT, Op);
- // Using SVE on 64bit Vectors with the same scalar type is not
- // profitable
- if (!(VT.is64BitVector() && SplatNode.getValueType().getScalarType() ==
- LD->getMemoryVT().getScalarType()))
+ EVT ScalableVT = getContainerForFixedLengthVector(DCI.DAG, VT);
+ // Using SVE Vectors with the same scalar type is not profitable
+ if (!(ScalableVT.getScalarType() ==
+ LD->getMemoryVT().getScalarType())) {
+ SDValue SplatNode =
+ DCI.DAG.getNode(ISD::SPLAT_VECTOR, DL, ScalableVT, Op);
return convertFromScalableVector(DCI.DAG, VT, SplatNode);
+ }
}
ISD::LoadExtType ExtType = LD->getExtensionType();
EVT MemVT = LD->getMemoryVT();
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 5b1411501a63e..17b49db08b205 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -604,7 +604,8 @@ def TuneNeoverseV3 : SubtargetFeature<"neoversev3", "ARMProcFamily", "NeoverseV3
FeatureUseFixedOverScalableIfEqualCost,
FeatureAvoidLDAPUR,
FeaturePredictableSelectIsExpensive,
- FeatureMaxInterleaveFactor4]>;
+ FeatureMaxInterleaveFactor4,
+ FeaturePreferSVEVectors]>;
def TuneNeoverseV3AE : SubtargetFeature<"neoversev3AE", "ARMProcFamily", "NeoverseV3AE",
"Neoverse V3AE ARM processors", [
diff --git a/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll b/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll
index cb41f89574cea..20356317fbdcf 100644
--- a/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll
+++ b/llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll
@@ -1,7 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v3ae -mattr=-sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=neoverse-v3ae < %s | FileCheck %s --check-prefixes=CHECK,CHECK-V3AE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve -mattr=+prefer-sve-vectors < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
; Test optimization of DUP with extended narrow loads
; This should avoid GPR->SIMD transfers by loading directly into vector registers
@@ -13,12 +12,12 @@ define <4 x i16> @test_dup_zextload_i8_v4i16(ptr %p) {
; CHECK-BASE-NEXT: dup v0.4h, v0.h[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i16:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.h
-; CHECK-V3AE-NEXT: ld1rb { z0.h }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v4i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.h
+; CHECK-SVE-NEXT: ld1rb { z0.h }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i16
%vec = insertelement <4 x i16> poison, i16 %ext, i32 0
@@ -33,12 +32,12 @@ define <8 x i16> @test_dup_zextload_i8_v8i16(ptr %p) {
; CHECK-BASE-NEXT: dup v0.8h, v0.h[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v8i16:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.h
-; CHECK-V3AE-NEXT: ld1rb { z0.h }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v8i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.h
+; CHECK-SVE-NEXT: ld1rb { z0.h }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i16
%vec = insertelement <8 x i16> poison, i16 %ext, i32 0
@@ -53,12 +52,12 @@ define <2 x i32> @test_dup_zextload_i8_v2i32(ptr %p) {
; CHECK-BASE-NEXT: dup v0.2s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v2i32:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v2i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: ld1rb { z0.s }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i32
%vec = insertelement <2 x i32> poison, i32 %ext, i32 0
@@ -73,12 +72,12 @@ define <4 x i32> @test_dup_zextload_i8_v4i32(ptr %p) {
; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i32:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v4i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: ld1rb { z0.s }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i32
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
@@ -93,12 +92,12 @@ define <4 x i32> @test_dup_zextload_i8_v4i32_offset(ptr %p) {
; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i32_offset:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x0, #4]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v4i32_offset:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: ld1rb { z0.s }, p0/z, [x0, #4]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%addr = getelementptr inbounds i8, ptr %p, i64 4
%load = load i8, ptr %addr, align 1
%ext = zext i8 %load to i32
@@ -114,13 +113,13 @@ define <4 x i32> @test_dup_zextload_i8_v4i32_reg_offset(ptr %p, i64 %offset) {
; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v4i32_reg_offset:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: add x8, x0, x1
-; CHECK-V3AE-NEXT: ld1rb { z0.s }, p0/z, [x8]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v4i32_reg_offset:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: add x8, x0, x1
+; CHECK-SVE-NEXT: ld1rb { z0.s }, p0/z, [x8]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%addr = getelementptr inbounds i8, ptr %p, i64 %offset
%load = load i8, ptr %addr, align 1
%ext = zext i8 %load to i32
@@ -136,12 +135,12 @@ define <2 x i64> @test_dup_zextload_i8_v2i64(ptr %p) {
; CHECK-BASE-NEXT: dup v0.2d, v0.d[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i8_v2i64:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.d
-; CHECK-V3AE-NEXT: ld1rb { z0.d }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i8_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: ld1rb { z0.d }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i8, ptr %p, align 1
%ext = zext i8 %load to i64
%vec = insertelement <2 x i64> poison, i64 %ext, i32 0
@@ -156,12 +155,12 @@ define <2 x i32> @test_dup_zextload_i16_v2i32(ptr %p) {
; CHECK-BASE-NEXT: dup v0.2s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i16_v2i32:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i16_v2i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: ld1rh { z0.s }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i16, ptr %p, align 1
%ext = zext i16 %load to i32
%vec = insertelement <2 x i32> poison, i32 %ext, i32 0
@@ -176,12 +175,12 @@ define <4 x i32> @test_dup_zextload_i16_v4i32(ptr %p) {
; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i16_v4i32:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i16_v4i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: ld1rh { z0.s }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i16, ptr %p, align 1
%ext = zext i16 %load to i32
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
@@ -196,12 +195,12 @@ define <4 x i32> @test_dup_zextload_i16_v4i32_offset(ptr %p) {
; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i16_v4i32_offset:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x0, #8]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i16_v4i32_offset:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: ld1rh { z0.s }, p0/z, [x0, #8]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%addr = getelementptr inbounds i16, ptr %p, i64 4
%load = load i16, ptr %addr, align 1
%ext = zext i16 %load to i32
@@ -217,13 +216,13 @@ define <4 x i32> @test_dup_zextload_i16_v4i32_reg_offset(ptr %p, i64 %offset) {
; CHECK-BASE-NEXT: dup v0.4s, v0.s[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i16_v4i32_reg_offset:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: add x8, x0, x1, lsl #1
-; CHECK-V3AE-NEXT: ld1rh { z0.s }, p0/z, [x8]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i16_v4i32_reg_offset:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: add x8, x0, x1, lsl #1
+; CHECK-SVE-NEXT: ld1rh { z0.s }, p0/z, [x8]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%addr = getelementptr inbounds i16, ptr %p, i64 %offset
%load = load i16, ptr %addr, align 1
%ext = zext i16 %load to i32
@@ -239,12 +238,12 @@ define <2 x i64> @test_dup_zextload_i16_v2i64(ptr %p) {
; CHECK-BASE-NEXT: dup v0.2d, v0.d[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i16_v2i64:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.d
-; CHECK-V3AE-NEXT: ld1rh { z0.d }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i16_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: ld1rh { z0.d }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i16, ptr %p, align 1
%ext = zext i16 %load to i64
%vec = insertelement <2 x i64> poison, i64 %ext, i32 0
@@ -259,12 +258,12 @@ define <2 x i64> @test_dup_zextload_i32_v2i64(ptr %p) {
; CHECK-BASE-NEXT: dup v0.2d, v0.d[0]
; CHECK-BASE-NEXT: ret
;
-; CHECK-V3AE-LABEL: test_dup_zextload_i32_v2i64:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.d
-; CHECK-V3AE-NEXT: ld1rw { z0.d }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-SVE-LABEL: test_dup_zextload_i32_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: ld1rw { z0.d }, p0/z, [x0]
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
%load = load i32, ptr %p, align 1
%ext = zext i32 %load to i64
%vec = insertelement <2 x i64> poison, i64 %ext, i32 0
@@ -273,17 +272,10 @@ define <2 x i64> @test_dup_zextload_i32_v2i64(ptr %p) {
}
define <16 x i8> @test_dup_load_i8_v16i8(ptr %p) {
-; CHECK-BASE-LABEL: test_dup_load_i8_v16i8:
-; CHECK-BASE: // %bb.0:
-; CHECK-BASE-NEXT: ld1r { v0.16b }, [x0]
-; CHECK-BASE-NEXT: ret
-;
-; CHECK-V3AE-LABEL: test_dup_load_i8_v16i8:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.b
-; CHECK-V3AE-NEXT: ld1rb { z0.b }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-LABEL: test_dup_load_i8_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.16b }, [x0]
+; CHECK-NEXT: ret
%load = load i8, ptr %p, align 1
%vec = insertelement <16 x i8> poison, i8 %load, i64 0
%dup = shufflevector <16 x i8> %vec, <16 x i8> poison, <16 x i32> zeroinitializer
@@ -302,17 +294,10 @@ define <8 x i8> @test_dup_load_i8_v8i8(ptr %p) {
}
define <8 x i16> @test_dup_load_i16_v8i16(ptr %p) {
-; CHECK-BASE-LABEL: test_dup_load_i16_v8i16:
-; CHECK-BASE: // %bb.0:
-; CHECK-BASE-NEXT: ld1r { v0.8h }, [x0]
-; CHECK-BASE-NEXT: ret
-;
-; CHECK-V3AE-LABEL: test_dup_load_i16_v8i16:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.h
-; CHECK-V3AE-NEXT: ld1rh { z0.h }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-LABEL: test_dup_load_i16_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.8h }, [x0]
+; CHECK-NEXT: ret
%load = load i16, ptr %p, align 1
%vec = insertelement <8 x i16> poison, i16 %load, i64 0
%dup = shufflevector <8 x i16> %vec, <8 x i16> poison, <8 x i32> zeroinitializer
@@ -331,17 +316,10 @@ define <4 x i16> @test_dup_load_i16_v4i16(ptr %p) {
}
define <4 x i32> @test_dup_load_i32_v4i32(ptr %p) {
-; CHECK-BASE-LABEL: test_dup_load_i32_v4i32:
-; CHECK-BASE: // %bb.0:
-; CHECK-BASE-NEXT: ld1r { v0.4s }, [x0]
-; CHECK-BASE-NEXT: ret
-;
-; CHECK-V3AE-LABEL: test_dup_load_i32_v4i32:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.s
-; CHECK-V3AE-NEXT: ld1rw { z0.s }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-LABEL: test_dup_load_i32_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.4s }, [x0]
+; CHECK-NEXT: ret
%load = load i32, ptr %p, align 1
%vec = insertelement <4 x i32> poison, i32 %load, i64 0
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
@@ -360,17 +338,10 @@ define <2 x i32> @test_dup_load_i32_v2i32(ptr %p) {
}
define <2 x i64> @test_dup_load_i64_v2i64(ptr %p) {
-; CHECK-BASE-LABEL: test_dup_load_i64_v2i64:
-; CHECK-BASE: // %bb.0:
-; CHECK-BASE-NEXT: ld1r { v0.2d }, [x0]
-; CHECK-BASE-NEXT: ret
-;
-; CHECK-V3AE-LABEL: test_dup_load_i64_v2i64:
-; CHECK-V3AE: // %bb.0:
-; CHECK-V3AE-NEXT: ptrue p0.d
-; CHECK-V3AE-NEXT: ld1rd { z0.d }, p0/z, [x0]
-; CHECK-V3AE-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-V3AE-NEXT: ret
+; CHECK-LABEL: test_dup_load_i64_v2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld1r { v0.2d }, [x0]
+; CHECK-NEXT: ret
%load = load i64, ptr %p, align 1
%vec = insertelement <2 x i64> poison, i64 %load, i64 0
%dup = shufflevector <2 x i64> %vec, <2 x i64> poison, <2 x i32> zeroinitializer
>From b5279f4528f39e6ceb5e5af1f4b1f35efdb4c8e2 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 7 Sep 2026 15:03:37 +0100
Subject: [PATCH 3/3] Remove inner if block
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fb5dbdaecbee3..92b7d506a9130 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -30144,15 +30144,12 @@ static SDValue performDUPCombine(SDNode *N,
if (auto *LD = dyn_cast<LoadSDNode>(Op)) {
if (Subtarget->preferSVEVectors() &&
Subtarget->isSVEorStreamingSVEAvailable() && Op->hasOneUse() &&
- VT.getScalarType().isInteger()) {
+ VT.getScalarType().isInteger() &&
+ VT.getScalarType() != LD->getMemoryVT().getScalarType()) {
EVT ScalableVT = getContainerForFixedLengthVector(DCI.DAG, VT);
- // Using SVE Vectors with the same scalar type is not profitable
- if (!(ScalableVT.getScalarType() ==
- LD->getMemoryVT().getScalarType())) {
- SDValue SplatNode =
- DCI.DAG.getNode(ISD::SPLAT_VECTOR, DL, ScalableVT, Op);
- return convertFromScalableVector(DCI.DAG, VT, SplatNode);
- }
+ SDValue SplatNode =
+ DCI.DAG.getNode(ISD::SPLAT_VECTOR, DL, ScalableVT, Op);
+ return convertFromScalableVector(DCI.DAG, VT, SplatNode);
}
ISD::LoadExtType ExtType = LD->getExtensionType();
EVT MemVT = LD->getMemoryVT();
More information about the llvm-commits
mailing list