[llvm] [LLVM][SelectionDAG] Implement unrolling of scalable vectors. (PR #216985)
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 2 09:04:58 PDT 2026
https://github.com/paulwalker-arm updated https://github.com/llvm/llvm-project/pull/216985
>From b5edacae195082bf1c041a1693979985ae863d07 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker at arm.com>
Date: Thu, 13 Aug 2026 17:21:35 +0100
Subject: [PATCH 1/2] [LLVM][SelectionDAG] Implement unrolling of scalable
vectors.
We cannot typically unroll scalable vectors because they have an
unknown-at-compile-time vector length. However, when a function has
the vscale_range attribute, we can unroll based on their maximum
possible length.
---
llvm/include/llvm/CodeGen/SelectionDAG.h | 10 +
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 65 +-
.../test/CodeGen/AArch64/sve-vector-unroll.ll | 641 ++++++++++++++++++
3 files changed, 709 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-unroll.ll
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index fc72f2a2ba0c6..243e7f87ae1a4 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -2782,6 +2782,16 @@ class SelectionDAG {
LLVM_ABI SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
SDValue InChain, const SDLoc &DLoc);
+ /// Returns the maximum runtime number of elements in VT if known, or 0
+ /// otherwise.
+ unsigned getMaxRuntimeNumElements(EVT VT) const;
+
+ /// Returns a vector constructed from the scalar values in order. The number
+ /// of scalars must match the maximum runtime length of VT, but only the first
+ /// actual runtime length scalars are included in the result.
+ SDValue buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
+ ArrayRef<SDValue> Scalars);
+
private:
#ifndef NDEBUG
void verifyNode(SDNode *N) const;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 2218e5f59752f..027aa11c35924 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -115,6 +115,11 @@ static cl::opt<unsigned>
cl::desc("DAG combiner limit number of steps when searching DAG "
"for predecessor nodes"));
+static cl::opt<int> VScaleUnrollLimit(
+ "vscale-unroll-limit",
+ cl::desc("Maximum vscale for which vector unrolling is allowed."),
+ cl::Hidden, cl::init(64));
+
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
}
@@ -14275,8 +14280,12 @@ SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
EVT VT = N->getValueType(0);
EVT EltVT = VT.getVectorElementType();
- unsigned NE = VT.getVectorNumElements();
+ unsigned NE = getMaxRuntimeNumElements(VT);
+
+ if (VT.isScalableVector() && (NE == 0 || ResNE != 0))
+ reportFatalUsageError("Cannot unroll scalable vector!");
+ assert(NE && "Nothing to unroll!");
SDLoc dl(N);
// If ResNE is 0, fully unroll the vector op.
@@ -14312,10 +14321,12 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
Scalars1.push_back(getUNDEF(EltVT1));
}
- EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
- EVT VecVT1 = EVT::getVectorVT(*getContext(), EltVT1, ResNE);
- SDValue Vec0 = getBuildVector(VecVT, dl, Scalars0);
- SDValue Vec1 = getBuildVector(VecVT1, dl, Scalars1);
+ ElementCount ResEC = VT.isScalableVector() ? VT.getVectorElementCount()
+ : ElementCount::getFixed(ResNE);
+ EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResEC);
+ EVT VecVT1 = EVT::getVectorVT(*getContext(), EltVT1, ResEC);
+ SDValue Vec0 = buildVectorFromUnrolledParts(VecVT, dl, Scalars0);
+ SDValue Vec1 = buildVectorFromUnrolledParts(VecVT1, dl, Scalars1);
return getMergeValues({Vec0, Vec1}, dl);
}
@@ -14379,8 +14390,10 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
for (; i < ResNE; ++i)
Scalars.push_back(getUNDEF(EltVT));
- EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
- return getBuildVector(VecVT, dl, Scalars);
+ EVT VecVT = VT.isScalableVector()
+ ? VT
+ : EVT::getVectorVT(*getContext(), EltVT, ResNE);
+ return buildVectorFromUnrolledParts(VecVT, dl, Scalars);
}
std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
@@ -15325,6 +15338,44 @@ void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
SDEI[To] = std::move(NEI);
}
+unsigned SelectionDAG::getMaxRuntimeNumElements(EVT VT) const {
+ assert(VT.isVector() && "Can only unroll vector types!");
+ if (VT.isFixedLengthVector())
+ return VT.getVectorNumElements();
+
+ const MachineFunction &MF = getMachineFunction();
+ const Function &F = MF.getFunction();
+
+ APInt MaxVScale = getVScaleRange(&F, sizeof(unsigned) * 8).getUnsignedMax();
+ if (MaxVScale.ugt(VScaleUnrollLimit))
+ return 0;
+
+ bool Overflow;
+ APInt MinNElts(sizeof(unsigned) * 8, VT.getVectorMinNumElements());
+ APInt MaxNElts = MinNElts.umul_ov(MaxVScale, Overflow);
+ if (Overflow)
+ return 0;
+
+ return MaxNElts.getZExtValue();
+}
+
+SDValue SelectionDAG::buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
+ ArrayRef<SDValue> Scalars) {
+ assert(Scalars.size() == getMaxRuntimeNumElements(VT) &&
+ "Element count mismatch!");
+ if (VT.isFixedLengthVector())
+ return getBuildVector(VT, DL, Scalars);
+
+ SDValue Vec = getPOISON(VT);
+ for (unsigned I = 0, E = Scalars.size(); I != E; ++I) {
+ // Iterate in reverse so result remains poison until we encounter a lane
+ // that exists, after which all lower-numbered lanes must also exist.
+ unsigned IdxVal = E - I - 1;
+ Vec = getInsertVectorElt(DL, Vec, Scalars[IdxVal], IdxVal);
+ }
+ return Vec;
+}
+
#ifndef NDEBUG
static void checkForCyclesHelper(const SDNode *N,
SmallPtrSetImpl<const SDNode*> &Visited,
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-unroll.ll b/llvm/test/CodeGen/AArch64/sve-vector-unroll.ll
new file mode 100644
index 0000000000000..621b39751e6f3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-unroll.ll
@@ -0,0 +1,641 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define <vscale x 2 x double> @unroll_single_result_op_vscale_1_1(<vscale x 2 x double> %a, <vscale x 2 x double> %b) vscale_range(1,1) #0 {
+; CHECK-LABEL: unroll_single_result_op_vscale_1_1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-18
+; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: stp q23, q22, [sp, #32] // 32-byte Folded Spill
+; CHECK-NEXT: stp q21, q20, [sp, #64] // 32-byte Folded Spill
+; CHECK-NEXT: stp q19, q18, [sp, #96] // 32-byte Folded Spill
+; CHECK-NEXT: stp q17, q16, [sp, #128] // 32-byte Folded Spill
+; CHECK-NEXT: stp q15, q14, [sp, #160] // 32-byte Folded Spill
+; CHECK-NEXT: stp q13, q12, [sp, #192] // 32-byte Folded Spill
+; CHECK-NEXT: stp q11, q10, [sp, #224] // 32-byte Folded Spill
+; CHECK-NEXT: stp q9, q8, [sp, #256] // 32-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: stp q0, q1, [sp] // 32-byte Folded Spill
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: str z0, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldp q0, q1, [sp] // 32-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[1]
+; CHECK-NEXT: mov z1.d, z1.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: mov z0.d, p0/m, z1.d
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldp q23, q22, [sp, #32] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q21, q20, [sp, #64] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q19, q18, [sp, #96] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q17, q16, [sp, #128] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q15, q14, [sp, #160] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q13, q12, [sp, #192] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q11, q10, [sp, #224] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldp q9, q8, [sp, #256] // 32-byte Folded Reload
+; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #18
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %res = call <vscale x 2 x double> @llvm.pow(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x double> @unroll_single_result_op_vscale_2_2(<vscale x 2 x double> %a, <vscale x 2 x double> %b) vscale_range(2,2) #0 {
+; CHECK-LABEL: unroll_single_result_op_vscale_2_2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-18
+; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: mov w8, #1 // =0x1
+; CHECK-NEXT: index z4.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z2.d, x8
+; CHECK-NEXT: mov w9, #2 // =0x2
+; CHECK-NEXT: str z1, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z3.d, x9
+; CHECK-NEXT: str z0, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z0.d, z0.d[2]
+; CHECK-NEXT: mov z1.d, z1.d[2]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: cmpeq p1.d, p0/z, z4.d, z2.d
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: str p1, [sp, #39, mul vl] // 2-byte Spill
+; CHECK-NEXT: cmpeq p1.d, p0/z, z4.d, z3.d
+; CHECK-NEXT: str p1, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: fmov d8, d0
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[3]
+; CHECK-NEXT: mov z1.d, z1.d[3]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ldr p0, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: mov z1.d, z1.d[1]
+; CHECK-NEXT: mov z0.d, p0/m, d8
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: str z0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: ldr p0, [sp, #39, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr z1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: str z1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: ldr z1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #18
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %res = call <vscale x 2 x double> @llvm.pow(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x double> @unroll_single_result_op_vscale_1_2(<vscale x 2 x double> %a, <vscale x 2 x double> %b) vscale_range(1,2) #0 {
+; CHECK-LABEL: unroll_single_result_op_vscale_1_2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-18
+; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: mov w8, #1 // =0x1
+; CHECK-NEXT: index z4.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z2.d, x8
+; CHECK-NEXT: mov w9, #2 // =0x2
+; CHECK-NEXT: str z1, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z3.d, x9
+; CHECK-NEXT: str z0, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z0.d, z0.d[2]
+; CHECK-NEXT: mov z1.d, z1.d[2]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: cmpeq p1.d, p0/z, z4.d, z2.d
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: str p1, [sp, #39, mul vl] // 2-byte Spill
+; CHECK-NEXT: cmpeq p1.d, p0/z, z4.d, z3.d
+; CHECK-NEXT: str p1, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: fmov d8, d0
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[3]
+; CHECK-NEXT: mov z1.d, z1.d[3]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ldr p0, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: mov z1.d, z1.d[1]
+; CHECK-NEXT: mov z0.d, p0/m, d8
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: str z0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: ldr p0, [sp, #39, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr z1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: str z1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: ldr z1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #18
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %res = call <vscale x 2 x double> @llvm.pow(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x double> @unroll_single_result_op_vscale_2_4(<vscale x 2 x double> %a, <vscale x 2 x double> %b) vscale_range(2,4) #0 {
+; CHECK-LABEL: unroll_single_result_op_vscale_2_4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-18
+; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-4
+; CHECK-NEXT: mov w8, #5 // =0x5
+; CHECK-NEXT: index z4.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z3.d, x8
+; CHECK-NEXT: mov w9, #6 // =0x6
+; CHECK-NEXT: str z1, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z2.d, x9
+; CHECK-NEXT: str z0, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z0.d, z0.d[6]
+; CHECK-NEXT: mov z1.d, z1.d[6]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: cmpeq p1.d, p0/z, z4.d, z3.d
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: str p1, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: cmpeq p1.d, p0/z, z4.d, z2.d
+; CHECK-NEXT: str p1, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: fmov d8, d0
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[7]
+; CHECK-NEXT: mov z1.d, z1.d[7]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ldr p0, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, d0
+; CHECK-NEXT: mov z1.d, z1.d[5]
+; CHECK-NEXT: mov z0.d, p0/m, d8
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: str z0, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[5]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: mov w8, #4 // =0x4
+; CHECK-NEXT: index z2.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z1.d, x8
+; CHECK-NEXT: cmpeq p1.d, p0/z, z2.d, z1.d
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p0, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: str p1, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z1, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[4]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: mov z1.d, z1.d[4]
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: mov w8, #3 // =0x3
+; CHECK-NEXT: index z2.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z1.d, x8
+; CHECK-NEXT: cmpeq p1.d, p0/z, z2.d, z1.d
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p0, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: str p1, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z1, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[3]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: mov z1.d, z1.d[3]
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: mov w8, #2 // =0x2
+; CHECK-NEXT: index z2.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z1.d, x8
+; CHECK-NEXT: cmpeq p1.d, p0/z, z2.d, z1.d
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p0, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: str p1, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z1, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[2]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: mov z1.d, z1.d[2]
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: mov w8, #1 // =0x1
+; CHECK-NEXT: index z2.d, #0, #1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: mov z1.d, x8
+; CHECK-NEXT: cmpeq p1.d, p0/z, z2.d, z1.d
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p0, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: str p1, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z1, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: mov z1.d, z1.d[1]
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p0, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z1.d, p0/m, d0
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: str z1, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT: bl pow
+; CHECK-NEXT: ldr z1, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d
+; CHECK-NEXT: addvl sp, sp, #4
+; CHECK-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #18
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %res = call <vscale x 2 x double> @llvm.pow(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+ ret <vscale x 2 x double> %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x i32> } @unroll_two_result_op_vscale_1_2(<vscale x 2 x double> %a) vscale_range(1,2) #0 {
+; CHECK-LABEL: unroll_two_result_op_vscale_1_2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-18
+; CHECK-NEXT: str p15, [sp, #4, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p14, [sp, #5, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p13, [sp, #6, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p12, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p11, [sp, #8, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p10, [sp, #9, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p9, [sp, #10, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p8, [sp, #11, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p7, [sp, #12, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p6, [sp, #13, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p5, [sp, #14, mul vl] // 2-byte Spill
+; CHECK-NEXT: str p4, [sp, #15, mul vl] // 2-byte Spill
+; CHECK-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: sub sp, sp, #32
+; CHECK-NEXT: addvl sp, sp, #-4
+; CHECK-NEXT: mov w8, #1 // =0x1
+; CHECK-NEXT: index z1.d, #0, #1
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov z2.d, x8
+; CHECK-NEXT: mov w9, #2 // =0x2
+; CHECK-NEXT: add x8, sp, #32
+; CHECK-NEXT: mov z3.d, x9
+; CHECK-NEXT: str z0, [x8, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z0.d, z0.d[2]
+; CHECK-NEXT: add x0, sp, #8
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: cmpeq p0.d, p1/z, z1.d, z2.d
+; CHECK-NEXT: str p0, [x8, #31, mul vl] // 2-byte Spill
+; CHECK-NEXT: cmpeq p0.d, p1/z, z1.d, z3.d
+; CHECK-NEXT: str p0, [x8, #30, mul vl] // 2-byte Spill
+; CHECK-NEXT: bl frexp
+; CHECK-NEXT: add x8, sp, #32
+; CHECK-NEXT: fmov d8, d0
+; CHECK-NEXT: mov x0, sp
+; CHECK-NEXT: ldr z0, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z0.d, z0.d[3]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: bl frexp
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: add x9, sp, #32
+; CHECK-NEXT: ldr x8, [sp, #8]
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: add x0, sp, #16
+; CHECK-NEXT: ld1rd { z1.d }, p0/z, [sp]
+; CHECK-NEXT: ldr p0, [x9, #30, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z1.d, p0/m, x8
+; CHECK-NEXT: add x8, sp, #32
+; CHECK-NEXT: str z0, [x8] // 16-byte Folded Spill
+; CHECK-NEXT: ldr z0, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: str z1, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: mov z0.d, z0.d[1]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: bl frexp
+; CHECK-NEXT: add x9, sp, #32
+; CHECK-NEXT: ldr x8, [sp, #16]
+; CHECK-NEXT: fmov d9, d0
+; CHECK-NEXT: ldr p0, [x9, #31, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr z1, [x9, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: add x0, sp, #24
+; CHECK-NEXT: mov z1.d, p0/m, x8
+; CHECK-NEXT: add x8, sp, #32
+; CHECK-NEXT: ldr z0, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: str z1, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT: bl frexp
+; CHECK-NEXT: add x8, sp, #32
+; CHECK-NEXT: add x9, sp, #32
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p0, [x8, #30, mul vl] // 2-byte Reload
+; CHECK-NEXT: mov z2.d, d1
+; CHECK-NEXT: ldr z1, [x9, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: mov z2.d, p0/m, d8
+; CHECK-NEXT: ldr p0, [x8, #31, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr x8, [sp, #24]
+; CHECK-NEXT: mov z2.d, p0/m, d9
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: mov z1.d, p0/m, x8
+; CHECK-NEXT: sel z0.d, p0, z0.d, z2.d
+; CHECK-NEXT: add sp, sp, #32
+; CHECK-NEXT: addvl sp, sp, #4
+; CHECK-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Reload
+; CHECK-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Reload
+; CHECK-NEXT: addvl sp, sp, #18
+; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+ %res = call { <vscale x 2 x double>, <vscale x 2 x i32> } @llvm.frexp(<vscale x 2 x double> %a)
+ ret { <vscale x 2 x double>, <vscale x 2 x i32> } %res
+}
+
+attributes #0 = { nounwind }
>From 587402f0f2f9f0f74172982b5af2ff2a7b63eb78 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker at arm.com>
Date: Wed, 2 Sep 2026 16:03:50 +0000
Subject: [PATCH 2/2] Improved reverse iteration.
---
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 027aa11c35924..e4be867b68c4b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -15367,12 +15367,11 @@ SDValue SelectionDAG::buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL,
return getBuildVector(VT, DL, Scalars);
SDValue Vec = getPOISON(VT);
- for (unsigned I = 0, E = Scalars.size(); I != E; ++I) {
- // Iterate in reverse so result remains poison until we encounter a lane
- // that exists, after which all lower-numbered lanes must also exist.
- unsigned IdxVal = E - I - 1;
+ // Iterate in reverse so result remains poison until we encounter a lane that
+ // exists, after which all lower-numbered lanes must also exist.
+ for (unsigned IdxVal : reverse(seq(Scalars.size())))
Vec = getInsertVectorElt(DL, Vec, Scalars[IdxVal], IdxVal);
- }
+
return Vec;
}
More information about the llvm-commits
mailing list