[llvm] [LSR] Improve the recognition of vscale immediates (PR #196998)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Mon May 11 09:55:04 PDT 2026
https://github.com/MacDue created https://github.com/llvm/llvm-project/pull/196998
Previously, the extraction of vscale immediates was handled the same as the non-scalable case. That is we'd look for a `vscale * constant` scMulExpr on the LHS of the SCEV expression.
However, within a SCEV expression operands are ordered by "complexity", which is defined (mostly) by the order of the SCEVTypes enum: https://github.com/llvm/llvm-project/blob/bcb84e234980ca42ee47e21e4c97180c788c946a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h#L38-L59
So, (non-scalable) constants would always occur on the LHS as the simplest expression type, but a scMulExpr could occur later in the operand list for a scAddExpr (e.g., after a sext/zext expression). This meant in some cases we'd fail to find vscale immediates.
This patch updates `ExtractImmediate` to search later operands to find vscale offsets, and prefer vscale offsets based on the access type and current BaseOffset. So for scalable accesses, we prefer finding vscale immediates over fixed immediates when both are present.
>From aafb03453db5ba9c2251fe0502e867ade67e4262 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 11 May 2026 16:25:58 +0000
Subject: [PATCH 1/2] Precommit test
---
.../AArch64/vscale-fixups.ll | 147 +++++++++++++++++-
1 file changed, 143 insertions(+), 4 deletions(-)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
index dd347a7a6519d..2347e5ff58b31 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
@@ -375,6 +375,145 @@ for.exit:
ret void
}
+ at global_data = external local_unnamed_addr global [9 x [9 x [9 x i32]]]
+
+; In this test the accesses are offset by the vector length (i.e, #1, mul vl apart),
+; but include a non-trival SCEV expression for the base address.
+; The SCEV expressions for the accesses are:
+; %gep.part0 =
+; (324 + (4 * (sext i32 (3 + (-1 * %3)<nsw> + %0) to i64))<nsw> + @_QMbrute_forceEblock),+,(32 * vscale)
+; %gep.part1 =
+; (324 + (4 * (sext i32 (3 + (-1 * %3)<nsw> + %0) to i64))<nsw> + (16 * vscale) + @_QMbrute_forceEblock),+,(32 * vscale)
+; This test checks that LSR can extract the offset of `(16 * vscale)` from the
+; expression for %gep.part1 (and use mul vl addressing from a common base for
+; both loads and stores).
+define void @vscale_complex_base_address(ptr %ptr, i32 %0, i64 %n) local_unnamed_addr #0 {
+; BASE-LABEL: vscale_complex_base_address:
+; BASE: // %bb.0: // %entry
+; BASE-NEXT: mov w8, #21846 // =0x5556
+; BASE-NEXT: sub w9, w1, #1
+; BASE-NEXT: ptrue p0.s
+; BASE-NEXT: movk w8, #21845, lsl #16
+; BASE-NEXT: mov x10, xzr
+; BASE-NEXT: smull x8, w9, w8
+; BASE-NEXT: lsr x9, x8, #32
+; BASE-NEXT: add x8, x9, x8, lsr #63
+; BASE-NEXT: adrp x9, :got:global_data
+; BASE-NEXT: ldr x9, [x9, :got_lo12:global_data]
+; BASE-NEXT: add w8, w8, w8, lsl #1
+; BASE-NEXT: add w8, w8, #4
+; BASE-NEXT: add x8, x9, w8, sxtw #2
+; BASE-NEXT: add x8, x8, #324
+; BASE-NEXT: mov x9, x8
+; BASE-NEXT: incb x8
+; BASE-NEXT: .LBB6_1: // %vector.body
+; BASE-NEXT: // =>This Inner Loop Header: Depth=1
+; BASE-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
+; BASE-NEXT: ld1w { z1.s }, p0/z, [x8, x10, lsl #2]
+; BASE-NEXT: sub z0.s, z0.s, #10 // =0xa
+; BASE-NEXT: sub z1.s, z1.s, #10 // =0xa
+; BASE-NEXT: st1w { z0.s }, p0, [x9, x10, lsl #2]
+; BASE-NEXT: st1w { z1.s }, p0, [x8, x10, lsl #2]
+; BASE-NEXT: inch x10
+; BASE-NEXT: cmp x2, x10
+; BASE-NEXT: b.ne .LBB6_1
+; BASE-NEXT: // %bb.2: // %exit
+; BASE-NEXT: ret
+;
+; PREINDEX-LABEL: vscale_complex_base_address:
+; PREINDEX: // %bb.0: // %entry
+; PREINDEX-NEXT: mov w8, #21846 // =0x5556
+; PREINDEX-NEXT: sub w9, w1, #1
+; PREINDEX-NEXT: ptrue p0.s
+; PREINDEX-NEXT: movk w8, #21845, lsl #16
+; PREINDEX-NEXT: mov x10, xzr
+; PREINDEX-NEXT: smull x8, w9, w8
+; PREINDEX-NEXT: lsr x9, x8, #32
+; PREINDEX-NEXT: add x8, x9, x8, lsr #63
+; PREINDEX-NEXT: adrp x9, :got:global_data
+; PREINDEX-NEXT: ldr x9, [x9, :got_lo12:global_data]
+; PREINDEX-NEXT: add w8, w8, w8, lsl #1
+; PREINDEX-NEXT: add w8, w8, #4
+; PREINDEX-NEXT: add x8, x9, w8, sxtw #2
+; PREINDEX-NEXT: add x8, x8, #324
+; PREINDEX-NEXT: mov x9, x8
+; PREINDEX-NEXT: incb x8
+; PREINDEX-NEXT: .LBB6_1: // %vector.body
+; PREINDEX-NEXT: // =>This Inner Loop Header: Depth=1
+; PREINDEX-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
+; PREINDEX-NEXT: ld1w { z1.s }, p0/z, [x8, x10, lsl #2]
+; PREINDEX-NEXT: sub z0.s, z0.s, #10 // =0xa
+; PREINDEX-NEXT: sub z1.s, z1.s, #10 // =0xa
+; PREINDEX-NEXT: st1w { z0.s }, p0, [x9, x10, lsl #2]
+; PREINDEX-NEXT: st1w { z1.s }, p0, [x8, x10, lsl #2]
+; PREINDEX-NEXT: inch x10
+; PREINDEX-NEXT: cmp x2, x10
+; PREINDEX-NEXT: b.ne .LBB6_1
+; PREINDEX-NEXT: // %bb.2: // %exit
+; PREINDEX-NEXT: ret
+;
+; POSTINDEX-LABEL: vscale_complex_base_address:
+; POSTINDEX: // %bb.0: // %entry
+; POSTINDEX-NEXT: mov w10, #21846 // =0x5556
+; POSTINDEX-NEXT: sub w9, w1, #1
+; POSTINDEX-NEXT: ptrue p0.s
+; POSTINDEX-NEXT: movk w10, #21845, lsl #16
+; POSTINDEX-NEXT: mov x8, xzr
+; POSTINDEX-NEXT: smull x9, w9, w10
+; POSTINDEX-NEXT: lsr x10, x9, #32
+; POSTINDEX-NEXT: add x9, x10, x9, lsr #63
+; POSTINDEX-NEXT: adrp x10, :got:global_data
+; POSTINDEX-NEXT: ldr x10, [x10, :got_lo12:global_data]
+; POSTINDEX-NEXT: add w9, w9, w9, lsl #1
+; POSTINDEX-NEXT: add x10, x10, #324
+; POSTINDEX-NEXT: add w9, w9, #4
+; POSTINDEX-NEXT: sbfiz x11, x9, #2, #32
+; POSTINDEX-NEXT: add x9, x10, x11
+; POSTINDEX-NEXT: incb x11
+; POSTINDEX-NEXT: add x10, x10, x11
+; POSTINDEX-NEXT: .LBB6_1: // %vector.body
+; POSTINDEX-NEXT: // =>This Inner Loop Header: Depth=1
+; POSTINDEX-NEXT: ld1w { z0.s }, p0/z, [x9, x8, lsl #2]
+; POSTINDEX-NEXT: ld1w { z1.s }, p0/z, [x10, x8, lsl #2]
+; POSTINDEX-NEXT: sub z0.s, z0.s, #10 // =0xa
+; POSTINDEX-NEXT: sub z1.s, z1.s, #10 // =0xa
+; POSTINDEX-NEXT: st1w { z0.s }, p0, [x9, x8, lsl #2]
+; POSTINDEX-NEXT: st1w { z1.s }, p0, [x10, x8, lsl #2]
+; POSTINDEX-NEXT: inch x8
+; POSTINDEX-NEXT: cmp x2, x8
+; POSTINDEX-NEXT: b.ne .LBB6_1
+; POSTINDEX-NEXT: // %bb.2: // %exit
+; POSTINDEX-NEXT: ret
+entry:
+ %1 = add i32 %0, 3
+ %2 = add i32 %0, -1
+ %3 = srem i32 %2, 3
+ %4 = sub i32 %1, %3
+ %5 = sext i32 %4 to i64
+ %complex.base = getelementptr [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @global_data, i64 324), i64 %5
+ %vscale = tail call i64 @llvm.vscale.i64()
+ %VFxUF = shl nuw i64 %vscale, 3
+ %off = shl i64 %vscale, 4
+ br label %vector.body
+
+vector.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %vector.body ]
+ %gep.part0 = getelementptr [4 x i8], ptr %complex.base, i64 %iv
+ %gep.part1 = getelementptr i8, ptr %gep.part0, i64 %off
+ %data = load <vscale x 4 x i32>, ptr %gep.part0, align 4
+ %data2 = load <vscale x 4 x i32>, ptr %gep.part1, align 4
+ %add = add <vscale x 4 x i32> %data, splat (i32 -10)
+ %add2 = add <vscale x 4 x i32> %data2, splat (i32 -10)
+ store <vscale x 4 x i32> %add, ptr %gep.part0, align 4
+ store <vscale x 4 x i32> %add2, ptr %gep.part1, align 4
+ %iv.next = add nuw i64 %iv, %VFxUF
+ %exit.cond = icmp eq i64 %iv.next, %n
+ br i1 %exit.cond, label %exit, label %vector.body
+
+exit:
+ ret void
+}
+
;; Here are two writes that should be `16 * vscale * vscale` apart, so MUL VL
;; addressing cannot be used to offset the second write, as for example,
;; `#4, mul vl` would only be an offset of `16 * vscale` (dropping a vscale).
@@ -391,8 +530,8 @@ define void @vscale_squared_offset(ptr %alloc) #0 {
; COMMON-NEXT: umull x9, w9, w10
; COMMON-NEXT: cntw x10
; COMMON-NEXT: cmp x8, x10
-; COMMON-NEXT: b.ge .LBB6_2
-; COMMON-NEXT: .LBB6_1: // %for.body
+; COMMON-NEXT: b.ge .LBB7_2
+; COMMON-NEXT: .LBB7_1: // %for.body
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
; COMMON-NEXT: add x11, x0, x9
; COMMON-NEXT: st1w { z0.s }, p0, [x0]
@@ -400,8 +539,8 @@ define void @vscale_squared_offset(ptr %alloc) #0 {
; COMMON-NEXT: st1w { z1.s }, p0, [x11]
; COMMON-NEXT: add x8, x8, #1
; COMMON-NEXT: cmp x8, x10
-; COMMON-NEXT: b.lt .LBB6_1
-; COMMON-NEXT: .LBB6_2: // %for.exit
+; COMMON-NEXT: b.lt .LBB7_1
+; COMMON-NEXT: .LBB7_2: // %for.exit
; COMMON-NEXT: ret
entry:
%vscale = call i64 @llvm.vscale.i64()
>From 521c334e8c49fde0fc8a19bccb66363b1644a704 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 11 May 2026 16:27:08 +0000
Subject: [PATCH 2/2] [LSR] Improve the recognition of vscale immediates
Previously, the extraction of vscale immediates was handled the same as
the non-scalable case. That is we'd look for a `vscale * constant`
scMulExpr on the LHS of the SCEV expression.
However, within a SCEV expression operands are ordered by "complexity",
which is defined (mostly) on the order of the SCEVTypes enum:
https://github.com/llvm/llvm-project/blob/bcb84e234980ca42ee47e21e4c97180c788c946a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h#L38-L59
So, (non-scalable) constants would always occur on the LHS as the
simplest expression type, but a scMulExpr could occur later in the
operand list for a scAddExpr (e.g., after a sext/zext expression). This
meant in some cases we'd fail to find vscale immediates.
This patch updates `ExtractImmediate` to search later operands to find
vscale offsets, and prefer vscale offsets based on the access type and
current BaseOffset. So for scalable accesses, we prefer finding vscale
immediates over fixed immediates when both are present.
---
.../Transforms/Scalar/LoopStrengthReduce.cpp | 73 ++++++--
.../AArch64/vscale-fixups.ll | 176 ++++++------------
2 files changed, 110 insertions(+), 139 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 01322c26aa77e..488da2b8a7f73 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -925,35 +925,72 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
return nullptr;
}
-/// If S involves the addition of a constant integer value, return that integer
-/// value, and mutate S to point to a new SCEV with that value excluded.
-static Immediate ExtractImmediate(SCEVUse &S, ScalarEvolution &SE) {
+/// Extracts an immediate operand from \p Ops and replaces the operand with
+/// zero. If \p PreferScalable is true and \p Ops contains both a scalable and
+/// non-scalable offsets, the scalable offset will be extracted.
+static Immediate ExtractImmediateOperand(MutableArrayRef<SCEVUse> Ops,
+ ScalarEvolution &SE,
+ bool PreferScalable) {
const APInt *C;
- if (match(S, m_scev_APInt(C))) {
- if (C->getSignificantBits() <= 64) {
- S = SE.getConstant(S->getType(), 0);
- return Immediate::getFixed(C->getSExtValue());
+ SCEVUse *Op = nullptr;
+ Immediate Result = Immediate::getZero();
+
+ if (EnableVScaleImmediates) {
+ // A vscale immediate is a scMulExpr, which when sorted by complexity, can
+ // occur after casted operands (truncate/sext/zext).
+ for (SCEVUse &S : Ops) {
+ // We know anything past scMulExpr will not be a vscale immediate.
+ if (S->getSCEVType() > scMulExpr)
+ break;
+ if (match(S, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale()))) {
+ Op = &S;
+ Result = Immediate::getScalable(C->getZExtValue());
+ break;
+ }
}
- } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
+ }
+
+ if (Result.isZero() || !PreferScalable) {
+ // A constant SCEV operands are always sorted to the LHS.
+ SCEVUse &S = Ops.front();
+ if (match(S, m_scev_APInt(C)) && !C->isZero()) {
+ if (C->getSignificantBits() <= 64) {
+ Op = &S;
+ Result = Immediate::getFixed(C->getSExtValue());
+ }
+ }
+ }
+
+ if (Result.isNonZero()) {
+ SCEVUse &S = *Op;
+ S = SE.getConstant(S->getType(), 0);
+ }
+
+ return Result;
+}
+
+/// If S involves the addition of a constant integer value, return that integer
+/// value, and mutate S to point to a new SCEV with that value excluded.
+static Immediate ExtractImmediate(SCEVUse &S, ScalarEvolution &SE,
+ bool PreferScalable = false) {
+ if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
SmallVector<SCEVUse, 8> NewOps(Add->operands());
- Immediate Result = ExtractImmediate(NewOps.front(), SE);
+ Immediate Result = ExtractImmediateOperand(NewOps, SE, PreferScalable);
+ if (Result.isZero())
+ Result = ExtractImmediate(NewOps.front(), SE, PreferScalable);
if (Result.isNonZero())
S = SE.getAddExpr(NewOps);
return Result;
} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
SmallVector<SCEVUse, 8> NewOps(AR->operands());
- Immediate Result = ExtractImmediate(NewOps.front(), SE);
+ Immediate Result = ExtractImmediate(NewOps.front(), SE, PreferScalable);
if (Result.isNonZero())
S = SE.getAddRecExpr(NewOps, AR->getLoop(),
// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
SCEV::FlagAnyWrap);
return Result;
- } else if (EnableVScaleImmediates &&
- match(S, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale()))) {
- S = SE.getConstant(S->getType(), 0);
- return Immediate::getScalable(C->getSExtValue());
}
- return Immediate::getZero();
+ return ExtractImmediateOperand({S}, SE, PreferScalable);
}
/// If S involves the addition of a GlobalValue address, return that symbol, and
@@ -2817,7 +2854,8 @@ std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
MemAccessTy AccessTy) {
const SCEV *Copy = Expr;
SCEVUse ExprUse = Expr;
- Immediate Offset = ExtractImmediate(ExprUse, SE);
+ Immediate Offset = ExtractImmediate(
+ ExprUse, SE, AccessTy.MemTy && AccessTy.MemTy->isScalableTy());
Expr = ExprUse;
// Basic uses can't accept any offset, for example.
@@ -4187,7 +4225,7 @@ void LSRInstance::GenerateConstantOffsetsImpl(
for (Immediate Offset : Worklist)
GenerateOffset(G, Offset);
- Immediate Imm = ExtractImmediate(G, SE);
+ Immediate Imm = ExtractImmediate(G, SE, Base.BaseOffset.isScalable());
if (G->isZero() || Imm.isZero() ||
!Base.BaseOffset.isCompatibleImmediate(Imm))
return;
@@ -4518,6 +4556,7 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
SmallVector<const SCEV *, 8> Sequence;
for (const SCEV *Use : RegUses) {
SCEVUse Reg = Use; // Make a copy for ExtractImmediate to modify.
+ // TODO: Extract both scalable and fixed immediates (if present)?
Immediate Imm = ExtractImmediate(Reg, SE);
auto Pair = Map.try_emplace(Reg);
if (Pair.second)
diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
index 2347e5ff58b31..729f9da6c16b1 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/vscale-fixups.ll
@@ -155,15 +155,15 @@ for.exit:
define void @mixed_offsets_scalable_then_fixed(ptr %src, ptr %dst, i64 %count) #0 {
; BASE-LABEL: mixed_offsets_scalable_then_fixed:
; BASE: // %bb.0: // %entry
-; BASE-NEXT: incb x0, all, mul #4
-; BASE-NEXT: ptrue p0.s
-; BASE-NEXT: mov x8, #8 // =0x8
+; BASE-NEXT: rdvl x8, #4
+; BASE-NEXT: ptrue p0.b
+; BASE-NEXT: orr x8, x8, #0x20
; BASE-NEXT: .LBB3_1: // %for.body
; BASE-NEXT: // =>This Inner Loop Header: Depth=1
-; BASE-NEXT: ldr z0, [x0, #-4, mul vl]
-; BASE-NEXT: ldr z1, [x0]
+; BASE-NEXT: ldr z0, [x0]
+; BASE-NEXT: ldr z1, [x0, #4, mul vl]
; BASE-NEXT: decw x2
-; BASE-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; BASE-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
; BASE-NEXT: incb x0
; BASE-NEXT: add z0.s, z0.s, z1.s
; BASE-NEXT: add z0.s, z0.s, z2.s
@@ -175,15 +175,15 @@ define void @mixed_offsets_scalable_then_fixed(ptr %src, ptr %dst, i64 %count) #
;
; PREINDEX-LABEL: mixed_offsets_scalable_then_fixed:
; PREINDEX: // %bb.0: // %entry
-; PREINDEX-NEXT: incb x0, all, mul #4
-; PREINDEX-NEXT: ptrue p0.s
-; PREINDEX-NEXT: mov x8, #8 // =0x8
+; PREINDEX-NEXT: rdvl x8, #4
+; PREINDEX-NEXT: ptrue p0.b
+; PREINDEX-NEXT: orr x8, x8, #0x20
; PREINDEX-NEXT: .LBB3_1: // %for.body
; PREINDEX-NEXT: // =>This Inner Loop Header: Depth=1
-; PREINDEX-NEXT: ldr z0, [x0, #-4, mul vl]
-; PREINDEX-NEXT: ldr z1, [x0]
+; PREINDEX-NEXT: ldr z0, [x0]
+; PREINDEX-NEXT: ldr z1, [x0, #4, mul vl]
; PREINDEX-NEXT: decw x2
-; PREINDEX-NEXT: ld1w { z2.s }, p0/z, [x0, x8, lsl #2]
+; PREINDEX-NEXT: ld1b { z2.b }, p0/z, [x0, x8]
; PREINDEX-NEXT: incb x0
; PREINDEX-NEXT: add z0.s, z0.s, z1.s
; PREINDEX-NEXT: add z0.s, z0.s, z2.s
@@ -195,19 +195,20 @@ define void @mixed_offsets_scalable_then_fixed(ptr %src, ptr %dst, i64 %count) #
;
; POSTINDEX-LABEL: mixed_offsets_scalable_then_fixed:
; POSTINDEX: // %bb.0: // %entry
-; POSTINDEX-NEXT: incb x0, all, mul #4
-; POSTINDEX-NEXT: ptrue p0.s
+; POSTINDEX-NEXT: ptrue p0.b
+; POSTINDEX-NEXT: rdvl x9, #4
; POSTINDEX-NEXT: mov x8, xzr
-; POSTINDEX-NEXT: mov x9, #8 // =0x8
+; POSTINDEX-NEXT: ptrue p1.s
+; POSTINDEX-NEXT: orr x9, x9, #0x20
; POSTINDEX-NEXT: .LBB3_1: // %for.body
; POSTINDEX-NEXT: // =>This Inner Loop Header: Depth=1
-; POSTINDEX-NEXT: ldr z0, [x0, #-4, mul vl]
-; POSTINDEX-NEXT: ldr z1, [x0]
-; POSTINDEX-NEXT: ld1w { z2.s }, p0/z, [x0, x9, lsl #2]
+; POSTINDEX-NEXT: ldr z0, [x0]
+; POSTINDEX-NEXT: ldr z1, [x0, #4, mul vl]
+; POSTINDEX-NEXT: ld1b { z2.b }, p0/z, [x0, x9]
; POSTINDEX-NEXT: incb x0
; POSTINDEX-NEXT: add z0.s, z0.s, z1.s
; POSTINDEX-NEXT: add z0.s, z0.s, z2.s
-; POSTINDEX-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
+; POSTINDEX-NEXT: st1w { z0.s }, p1, [x1, x8, lsl #2]
; POSTINDEX-NEXT: incw x8
; POSTINDEX-NEXT: cmp x2, x8
; POSTINDEX-NEXT: b.ne .LBB3_1
@@ -242,18 +243,18 @@ for.exit:
define void @mixed_offsets_fixed_then_scalable(ptr %src, ptr %dst, i64 %count) #0 {
; COMMON-LABEL: mixed_offsets_fixed_then_scalable:
; COMMON: // %bb.0: // %entry
-; COMMON-NEXT: mov x9, x0
; COMMON-NEXT: ptrue p0.s
+; COMMON-NEXT: rdvl x9, #4
; COMMON-NEXT: mov x8, xzr
-; COMMON-NEXT: incb x9, all, mul #4
+; COMMON-NEXT: ptrue p1.b
+; COMMON-NEXT: orr x9, x9, #0x20
; COMMON-NEXT: mov x10, #8 // =0x8
-; COMMON-NEXT: add x9, x9, #32
; COMMON-NEXT: .LBB4_1: // %for.body
; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
-; COMMON-NEXT: add x11, x0, x8, lsl #2
-; COMMON-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; COMMON-NEXT: ld1w { z2.s }, p0/z, [x9, x8, lsl #2]
-; COMMON-NEXT: ld1w { z1.s }, p0/z, [x11, x10, lsl #2]
+; COMMON-NEXT: ldr z0, [x0]
+; COMMON-NEXT: ld1w { z1.s }, p0/z, [x0, x10, lsl #2]
+; COMMON-NEXT: ld1b { z2.b }, p1/z, [x0, x9]
+; COMMON-NEXT: incb x0
; COMMON-NEXT: add z0.s, z0.s, z1.s
; COMMON-NEXT: add z0.s, z0.s, z2.s
; COMMON-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]
@@ -388,102 +389,33 @@ for.exit:
; expression for %gep.part1 (and use mul vl addressing from a common base for
; both loads and stores).
define void @vscale_complex_base_address(ptr %ptr, i32 %0, i64 %n) local_unnamed_addr #0 {
-; BASE-LABEL: vscale_complex_base_address:
-; BASE: // %bb.0: // %entry
-; BASE-NEXT: mov w8, #21846 // =0x5556
-; BASE-NEXT: sub w9, w1, #1
-; BASE-NEXT: ptrue p0.s
-; BASE-NEXT: movk w8, #21845, lsl #16
-; BASE-NEXT: mov x10, xzr
-; BASE-NEXT: smull x8, w9, w8
-; BASE-NEXT: lsr x9, x8, #32
-; BASE-NEXT: add x8, x9, x8, lsr #63
-; BASE-NEXT: adrp x9, :got:global_data
-; BASE-NEXT: ldr x9, [x9, :got_lo12:global_data]
-; BASE-NEXT: add w8, w8, w8, lsl #1
-; BASE-NEXT: add w8, w8, #4
-; BASE-NEXT: add x8, x9, w8, sxtw #2
-; BASE-NEXT: add x8, x8, #324
-; BASE-NEXT: mov x9, x8
-; BASE-NEXT: incb x8
-; BASE-NEXT: .LBB6_1: // %vector.body
-; BASE-NEXT: // =>This Inner Loop Header: Depth=1
-; BASE-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
-; BASE-NEXT: ld1w { z1.s }, p0/z, [x8, x10, lsl #2]
-; BASE-NEXT: sub z0.s, z0.s, #10 // =0xa
-; BASE-NEXT: sub z1.s, z1.s, #10 // =0xa
-; BASE-NEXT: st1w { z0.s }, p0, [x9, x10, lsl #2]
-; BASE-NEXT: st1w { z1.s }, p0, [x8, x10, lsl #2]
-; BASE-NEXT: inch x10
-; BASE-NEXT: cmp x2, x10
-; BASE-NEXT: b.ne .LBB6_1
-; BASE-NEXT: // %bb.2: // %exit
-; BASE-NEXT: ret
-;
-; PREINDEX-LABEL: vscale_complex_base_address:
-; PREINDEX: // %bb.0: // %entry
-; PREINDEX-NEXT: mov w8, #21846 // =0x5556
-; PREINDEX-NEXT: sub w9, w1, #1
-; PREINDEX-NEXT: ptrue p0.s
-; PREINDEX-NEXT: movk w8, #21845, lsl #16
-; PREINDEX-NEXT: mov x10, xzr
-; PREINDEX-NEXT: smull x8, w9, w8
-; PREINDEX-NEXT: lsr x9, x8, #32
-; PREINDEX-NEXT: add x8, x9, x8, lsr #63
-; PREINDEX-NEXT: adrp x9, :got:global_data
-; PREINDEX-NEXT: ldr x9, [x9, :got_lo12:global_data]
-; PREINDEX-NEXT: add w8, w8, w8, lsl #1
-; PREINDEX-NEXT: add w8, w8, #4
-; PREINDEX-NEXT: add x8, x9, w8, sxtw #2
-; PREINDEX-NEXT: add x8, x8, #324
-; PREINDEX-NEXT: mov x9, x8
-; PREINDEX-NEXT: incb x8
-; PREINDEX-NEXT: .LBB6_1: // %vector.body
-; PREINDEX-NEXT: // =>This Inner Loop Header: Depth=1
-; PREINDEX-NEXT: ld1w { z0.s }, p0/z, [x9, x10, lsl #2]
-; PREINDEX-NEXT: ld1w { z1.s }, p0/z, [x8, x10, lsl #2]
-; PREINDEX-NEXT: sub z0.s, z0.s, #10 // =0xa
-; PREINDEX-NEXT: sub z1.s, z1.s, #10 // =0xa
-; PREINDEX-NEXT: st1w { z0.s }, p0, [x9, x10, lsl #2]
-; PREINDEX-NEXT: st1w { z1.s }, p0, [x8, x10, lsl #2]
-; PREINDEX-NEXT: inch x10
-; PREINDEX-NEXT: cmp x2, x10
-; PREINDEX-NEXT: b.ne .LBB6_1
-; PREINDEX-NEXT: // %bb.2: // %exit
-; PREINDEX-NEXT: ret
-;
-; POSTINDEX-LABEL: vscale_complex_base_address:
-; POSTINDEX: // %bb.0: // %entry
-; POSTINDEX-NEXT: mov w10, #21846 // =0x5556
-; POSTINDEX-NEXT: sub w9, w1, #1
-; POSTINDEX-NEXT: ptrue p0.s
-; POSTINDEX-NEXT: movk w10, #21845, lsl #16
-; POSTINDEX-NEXT: mov x8, xzr
-; POSTINDEX-NEXT: smull x9, w9, w10
-; POSTINDEX-NEXT: lsr x10, x9, #32
-; POSTINDEX-NEXT: add x9, x10, x9, lsr #63
-; POSTINDEX-NEXT: adrp x10, :got:global_data
-; POSTINDEX-NEXT: ldr x10, [x10, :got_lo12:global_data]
-; POSTINDEX-NEXT: add w9, w9, w9, lsl #1
-; POSTINDEX-NEXT: add x10, x10, #324
-; POSTINDEX-NEXT: add w9, w9, #4
-; POSTINDEX-NEXT: sbfiz x11, x9, #2, #32
-; POSTINDEX-NEXT: add x9, x10, x11
-; POSTINDEX-NEXT: incb x11
-; POSTINDEX-NEXT: add x10, x10, x11
-; POSTINDEX-NEXT: .LBB6_1: // %vector.body
-; POSTINDEX-NEXT: // =>This Inner Loop Header: Depth=1
-; POSTINDEX-NEXT: ld1w { z0.s }, p0/z, [x9, x8, lsl #2]
-; POSTINDEX-NEXT: ld1w { z1.s }, p0/z, [x10, x8, lsl #2]
-; POSTINDEX-NEXT: sub z0.s, z0.s, #10 // =0xa
-; POSTINDEX-NEXT: sub z1.s, z1.s, #10 // =0xa
-; POSTINDEX-NEXT: st1w { z0.s }, p0, [x9, x8, lsl #2]
-; POSTINDEX-NEXT: st1w { z1.s }, p0, [x10, x8, lsl #2]
-; POSTINDEX-NEXT: inch x8
-; POSTINDEX-NEXT: cmp x2, x8
-; POSTINDEX-NEXT: b.ne .LBB6_1
-; POSTINDEX-NEXT: // %bb.2: // %exit
-; POSTINDEX-NEXT: ret
+; COMMON-LABEL: vscale_complex_base_address:
+; COMMON: // %bb.0: // %entry
+; COMMON-NEXT: mov w8, #21846 // =0x5556
+; COMMON-NEXT: sub w9, w1, #1
+; COMMON-NEXT: movk w8, #21845, lsl #16
+; COMMON-NEXT: smull x8, w9, w8
+; COMMON-NEXT: lsr x9, x8, #32
+; COMMON-NEXT: add x8, x9, x8, lsr #63
+; COMMON-NEXT: adrp x9, :got:global_data
+; COMMON-NEXT: ldr x9, [x9, :got_lo12:global_data]
+; COMMON-NEXT: add w8, w8, w8, lsl #1
+; COMMON-NEXT: add w8, w8, #4
+; COMMON-NEXT: add x8, x9, w8, sxtw #2
+; COMMON-NEXT: add x8, x8, #324
+; COMMON-NEXT: .LBB6_1: // %vector.body
+; COMMON-NEXT: // =>This Inner Loop Header: Depth=1
+; COMMON-NEXT: ldr z0, [x8]
+; COMMON-NEXT: ldr z1, [x8, #1, mul vl]
+; COMMON-NEXT: dech x2
+; COMMON-NEXT: sub z0.s, z0.s, #10 // =0xa
+; COMMON-NEXT: sub z1.s, z1.s, #10 // =0xa
+; COMMON-NEXT: str z0, [x8]
+; COMMON-NEXT: str z1, [x8, #1, mul vl]
+; COMMON-NEXT: incb x8, all, mul #2
+; COMMON-NEXT: cbnz x2, .LBB6_1
+; COMMON-NEXT: // %bb.2: // %exit
+; COMMON-NEXT: ret
entry:
%1 = add i32 %0, 3
%2 = add i32 %0, -1
More information about the llvm-commits
mailing list