[llvm] [AArch64][SDAG] Fix invalid index into STEP_VECTOR operand (PR #193236)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 22 11:48:04 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/193236
>From 161b208894aff9c2d8a9737b970ff4cec8489739 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 21 Apr 2026 14:21:17 +0000
Subject: [PATCH 1/3] [AArch64][SDAG] Fix invalid index into STEP_VECTOR
operand
This commit fixes an invalid index into STEP_VECTOR constant operand
trying to find a more optimal addressing type.
---
.../CodeGen/AArch64/aarch64-masked-scatter.ll | 39 +++++++++++++++++++
1 file changed, 39 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll
diff --git a/llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll b/llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll
new file mode 100644
index 0000000000000..1ecf92ae3492e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mcpu=grace < %s -o -| FileCheck %s
+
+; Function Attrs: vscale_range(1,16)
+define <vscale x 8 x ptr> @masked_scatter_shft_left(ptr %f, <vscale x 8 x i1> %0, <vscale x 8 x i1> %1) #0 {
+; CHECK-LABEL: masked_scatter_shft_left:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov w9, #64 // =0x40
+; CHECK-NEXT: punpklo p1.h, p0.b
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: uunpklo z2.s, z1.h
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: punpkhi p0.h, p0.b
+; CHECK-NEXT: index z0.s, #0, w9
+; CHECK-NEXT: rdvl x9, #16
+; CHECK-NEXT: uunpkhi z1.s, z1.h
+; CHECK-NEXT: st1b { z2.s }, p1, [x8, z0.s, sxtw]
+; CHECK-NEXT: mov z2.s, w9
+; CHECK-NEXT: add z0.s, z0.s, z2.s
+; CHECK-NEXT: st1b { z1.s }, p0, [x8, z0.s, sxtw]
+; CHECK-NEXT: index z0.d, x0, #8
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: incd z1.d, all, mul #8
+; CHECK-NEXT: incd z2.d, all, mul #16
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: incd z3.d, all, mul #16
+; CHECK-NEXT: ret
+entry:
+ %2 = tail call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+ %3 = shl <vscale x 8 x i64> %2, splat (i64 2)
+ %4 = and <vscale x 8 x i64> %3, splat (i64 4294967292)
+ %5 = getelementptr [16 x i8], ptr null, <vscale x 8 x i64> %4
+ tail call void @llvm.masked.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> zeroinitializer, <vscale x 8 x ptr> %5, <vscale x 8 x i1> %0)
+ %6 = getelementptr [2 x i8], ptr %f, <vscale x 8 x i64> %4
+ ret <vscale x 8 x ptr> %6
+}
+
+attributes #0 = { vscale_range(1,16) "target-cpu"="grace" }
>From b0fdef08e7b738d158ddafe8005e8ac89d974a3b Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 21 Apr 2026 14:25:47 +0000
Subject: [PATCH 2/3] fixup! [AArch64][SDAG] Fix invalid index into STEP_VECTOR
operand
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 189347f9e07e8..e2ce064892caf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26322,7 +26322,7 @@ static bool findMoreOptimalIndexType(const MaskedGatherScatterSDNode *N,
SDValue RHS = Index.getOperand(1);
if (auto *Shift =
dyn_cast_or_null<ConstantSDNode>(DAG.getSplatValue(RHS))) {
- int64_t Step = (int64_t)Index.getOperand(0).getConstantOperandVal(1);
+ int64_t Step = (int64_t)Index.getOperand(0).getConstantOperandVal(0);
Stride = Step << Shift->getZExtValue();
}
}
>From ff8369b7b091e77ca84e050f85fbfb2e5e4ee205 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Wed, 22 Apr 2026 10:33:02 +0000
Subject: [PATCH 3/3] fixup! fixup! [AArch64][SDAG] Fix invalid index into
STEP_VECTOR operand
---
.../Target/AArch64/AArch64ISelLowering.cpp | 11 ------
.../CodeGen/AArch64/aarch64-masked-scatter.ll | 39 -------------------
.../AArch64/sve-gather-scatter-addr-opts.ll | 22 +++++++++++
3 files changed, 22 insertions(+), 50 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e2ce064892caf..66beb40dac76a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -26315,17 +26315,6 @@ static bool findMoreOptimalIndexType(const MaskedGatherScatterSDNode *N,
if (Index.getOpcode() == ISD::STEP_VECTOR) {
Stride = cast<ConstantSDNode>(Index.getOperand(0))->getSExtValue();
}
- // Match:
- // Index = step(const) << shift(const)
- else if (Index.getOpcode() == ISD::SHL &&
- Index.getOperand(0).getOpcode() == ISD::STEP_VECTOR) {
- SDValue RHS = Index.getOperand(1);
- if (auto *Shift =
- dyn_cast_or_null<ConstantSDNode>(DAG.getSplatValue(RHS))) {
- int64_t Step = (int64_t)Index.getOperand(0).getConstantOperandVal(0);
- Stride = Step << Shift->getZExtValue();
- }
- }
// Return early because no supported pattern is found.
if (Stride == 0)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll b/llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll
deleted file mode 100644
index 1ecf92ae3492e..0000000000000
--- a/llvm/test/CodeGen/AArch64/aarch64-masked-scatter.ll
+++ /dev/null
@@ -1,39 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mcpu=grace < %s -o -| FileCheck %s
-
-; Function Attrs: vscale_range(1,16)
-define <vscale x 8 x ptr> @masked_scatter_shft_left(ptr %f, <vscale x 8 x i1> %0, <vscale x 8 x i1> %1) #0 {
-; CHECK-LABEL: masked_scatter_shft_left:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w9, #64 // =0x40
-; CHECK-NEXT: punpklo p1.h, p0.b
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: uunpklo z2.s, z1.h
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: index z0.s, #0, w9
-; CHECK-NEXT: rdvl x9, #16
-; CHECK-NEXT: uunpkhi z1.s, z1.h
-; CHECK-NEXT: st1b { z2.s }, p1, [x8, z0.s, sxtw]
-; CHECK-NEXT: mov z2.s, w9
-; CHECK-NEXT: add z0.s, z0.s, z2.s
-; CHECK-NEXT: st1b { z1.s }, p0, [x8, z0.s, sxtw]
-; CHECK-NEXT: index z0.d, x0, #8
-; CHECK-NEXT: mov z1.d, z0.d
-; CHECK-NEXT: mov z2.d, z0.d
-; CHECK-NEXT: incd z1.d, all, mul #8
-; CHECK-NEXT: incd z2.d, all, mul #16
-; CHECK-NEXT: mov z3.d, z1.d
-; CHECK-NEXT: incd z3.d, all, mul #16
-; CHECK-NEXT: ret
-entry:
- %2 = tail call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
- %3 = shl <vscale x 8 x i64> %2, splat (i64 2)
- %4 = and <vscale x 8 x i64> %3, splat (i64 4294967292)
- %5 = getelementptr [16 x i8], ptr null, <vscale x 8 x i64> %4
- tail call void @llvm.masked.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> zeroinitializer, <vscale x 8 x ptr> %5, <vscale x 8 x i1> %0)
- %6 = getelementptr [2 x i8], ptr %f, <vscale x 8 x i64> %4
- ret <vscale x 8 x ptr> %6
-}
-
-attributes #0 = { vscale_range(1,16) "target-cpu"="grace" }
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
index 9f5e0eb9878c2..f7f792e2613bc 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
@@ -457,6 +457,28 @@ define void @masked_scatter_nxv4i32_u32s8_offsets(ptr %base, <vscale x 4 x i8> %
ret void
}
+define <vscale x 4 x ptr> @masked_scatter_shift_left(ptr %f, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) #0 {
+; CHECK-LABEL: masked_scatter_shift_left:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z0.d, x0, #8
+; CHECK-NEXT: mov w8, #64 // =0x40
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: index z2.s, #0, w8
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: st1b { z3.s }, p0, [x8, z2.s, sxtw]
+; CHECK-NEXT: incd z1.d, all, mul #8
+; CHECK-NEXT: ret
+entry:
+ %step = tail call <vscale x 4 x i64> @llvm.stepvector.nxv8i64()
+ %step_shifted = shl <vscale x 4 x i64> %step, splat (i64 2)
+ %step_aligned = and <vscale x 4 x i64> %step_shifted, splat (i64 4294967292)
+ %pointer = getelementptr [16 x i8], ptr null, <vscale x 4 x i64> %step_aligned
+ tail call void @llvm.masked.scatter.nxv8i8.nxv8p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> %pointer, <vscale x 4 x i1> %0)
+ %result = getelementptr [2 x i8], ptr %f, <vscale x 4 x i64> %step_aligned
+ ret <vscale x 4 x ptr> %result
+}
+
attributes #0 = { "target-features"="+sve" vscale_range(1, 16) }
declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
More information about the llvm-commits
mailing list