[llvm] [SeparateConstOffsetFromGEP] Share scalable vector GEP bases (PR #214769)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 08:09:58 PDT 2026
https://github.com/jacob-crawley created https://github.com/llvm/llvm-project/pull/214769
Recognize scalable vector GEPs with a common varying index and loop-invariant splat offsets. Compute the varying GEP once and express each remaining offset as a scalar byte offset, avoiding repeated vector address calculations in unrolled loops.
>From 4595e371a1c1c25c1a6d451c4b936d80adf5a6ef Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 7 Aug 2026 15:08:21 +0000
Subject: [PATCH] [SeparateConstOffsetFromGEP] Share scalable vector GEP bases
Recognize scalable vector GEPs with a common varying index and
loop-invariant splat offsets. Compute the varying GEP once and express
each remaining offset as a scalar byte offset, avoiding repeated vector
address calculations in unrolled loops.
---
.../Scalar/SeparateConstOffsetFromGEP.cpp | 228 ++++++++++++++++++
llvm/test/CodeGen/AArch64/sve-vector-gep.ll | 34 +++
.../scalable-vector-gep-common-base.ll | 67 +++++
3 files changed, 329 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-vector-gep.ll
create mode 100644 llvm/test/Transforms/SeparateConstOffsetFromGEP/AArch64/scalable-vector-gep-common-base.ll
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 4870b8c888279..481e57cf55981 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -138,6 +138,7 @@
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
@@ -156,6 +157,7 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
+#include "llvm/IR/ValueHandle.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
@@ -394,6 +396,24 @@ class SeparateConstOffsetFromGEP {
return {B, A};
}
+ struct VectorGEPOffsetTerm {
+ Value *Scalar;
+ bool IsSub;
+ };
+
+ struct VectorGEPCandidate {
+ GetElementPtrInst *GEP;
+ Value *VaryingIndex;
+ SmallVector<VectorGEPOffsetTerm, 4> OffsetTerms;
+ uint64_t Stride;
+ };
+
+ bool collectVectorGEPCandidate(GetElementPtrInst *GEP,
+ VectorGEPCandidate &Candidate);
+ static bool haveSameVectorGEPOffset(const VectorGEPCandidate &LHS,
+ const VectorGEPCandidate &RHS);
+ bool shareScalableVectorGEPBase(BasicBlock &BB);
+
/// Tries to split the given GEP into a variadic base and a constant offset,
/// and returns true if the splitting succeeds.
bool splitGEP(GetElementPtrInst *GEP);
@@ -1172,6 +1192,211 @@ bool SeparateConstOffsetFromGEP::reorderGEP(GetElementPtrInst *GEP,
return true;
}
+bool SeparateConstOffsetFromGEP::haveSameVectorGEPOffset(
+ const VectorGEPCandidate &LHS, const VectorGEPCandidate &RHS) {
+ if (LHS.OffsetTerms.size() != RHS.OffsetTerms.size())
+ return false;
+
+ for (unsigned I = 0; I != LHS.OffsetTerms.size(); ++I) {
+ const VectorGEPOffsetTerm < = LHS.OffsetTerms[I];
+ const VectorGEPOffsetTerm &RT = RHS.OffsetTerms[I];
+
+ if (LT.Scalar != RT.Scalar || LT.IsSub != RT.IsSub)
+ return false;
+ }
+
+ return true;
+}
+
+/// Collect a scalable vector GEP whose index is a varying value plus or minus
+/// loop invariant splats. The invariant terms become scalar byte offsets.
+bool SeparateConstOffsetFromGEP::collectVectorGEPCandidate(
+ GetElementPtrInst *GEP, VectorGEPCandidate &Candidate) {
+ if (GEP->getNumIndices() != 1 || !GEP->getPointerOperandType()->isPointerTy())
+ return false;
+
+ auto *GEPType = dyn_cast<VectorType>(GEP->getType());
+ if (!GEPType || !GEPType->getElementCount().isScalable())
+ return false;
+
+ if (GEP->isInBounds() || GEP->hasNoUnsignedWrap() ||
+ GEP->hasNoUnsignedSignedWrap())
+ return false;
+
+ Value *Index = GEP->getOperand(1);
+ auto *IndexType = dyn_cast<VectorType>(Index->getType());
+ if (!IndexType || !IndexType->getElementCount().isScalable() ||
+ !IndexType->getElementType()->isIntegerTy())
+ return false;
+
+ Type *PointerIndexType = DL->getIndexType(GEP->getPointerOperandType());
+ if (IndexType->getElementType() != PointerIndexType)
+ return false;
+
+ TypeSize ElementSize = DL->getTypeAllocSize(GEP->getSourceElementType());
+ if (ElementSize.isScalable())
+ return false;
+
+ Loop *L = LI->getLoopFor(GEP->getParent());
+ if (!L)
+ return false;
+
+ SmallVector<VectorGEPOffsetTerm, 4> OffsetTerms;
+ Value *VaryingIndex = Index;
+
+ // Peel loop-invariant splats from an add/sub chain.
+ while (auto *BO = dyn_cast<BinaryOperator>(VaryingIndex)) {
+ unsigned Opcode = BO->getOpcode();
+ if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
+ break;
+
+ Value *NextIndex = nullptr;
+ Value *ScalarOffset = nullptr;
+ bool IsSub = false;
+
+ auto GetInvariantSplat = [&](Value *V) -> Value * {
+ Value *Splat = getSplatValue(V);
+ if (!Splat || Splat->getType() != IndexType->getElementType() ||
+ !L->isLoopInvariant(Splat))
+ return nullptr;
+ return Splat;
+ };
+
+ unsigned SplatOperand = 1;
+ Value *Splat = GetInvariantSplat(BO->getOperand(SplatOperand));
+
+ if (!Splat && Opcode == Instruction::Add) {
+ SplatOperand = 0;
+ Splat = GetInvariantSplat(BO->getOperand(SplatOperand));
+ }
+
+ if (Splat) {
+ NextIndex = BO->getOperand(1 - SplatOperand);
+ ScalarOffset = Splat;
+ IsSub = Opcode == Instruction::Sub;
+ }
+
+ if (!NextIndex)
+ break;
+
+ OffsetTerms.push_back({ScalarOffset, IsSub});
+ VaryingIndex = NextIndex;
+ }
+
+ if (OffsetTerms.empty())
+ return false;
+
+ Candidate = {GEP, VaryingIndex, std::move(OffsetTerms),
+ ElementSize.getFixedValue()};
+
+ return true;
+}
+
+/// Find compatible scalable vector GEPs in \p BB that share a base pointer and
+/// varying index. Rewrite them to share one scaled vector GEP, with each
+/// original GEP represented by a scalar byte offset from the common address.
+bool SeparateConstOffsetFromGEP::shareScalableVectorGEPBase(BasicBlock &BB) {
+ SmallVector<VectorGEPCandidate, 8> Candidates;
+
+ for (Instruction &I : BB) {
+ auto *GEP = dyn_cast<GetElementPtrInst>(&I);
+ if (!GEP)
+ continue;
+
+ VectorGEPCandidate Candidate;
+ if (collectVectorGEPCandidate(GEP, Candidate))
+ Candidates.push_back(std::move(Candidate));
+ }
+
+ SmallVector<bool, 8> Rewritten(Candidates.size(), false);
+ SmallVector<WeakTrackingVH, 8> DeadIndices;
+ bool Changed = false;
+
+ for (unsigned I = 0; I != Candidates.size(); ++I) {
+ if (Rewritten[I])
+ continue;
+
+ SmallVector<unsigned, 4> Group;
+ Group.push_back(I);
+
+ const VectorGEPCandidate &Leader = Candidates[I];
+ for (unsigned J = I + 1; J != Candidates.size(); ++J) {
+ if (Rewritten[J])
+ continue;
+
+ const VectorGEPCandidate &Other = Candidates[J];
+ if (Other.GEP->getPointerOperand() == Leader.GEP->getPointerOperand() &&
+ Other.GEP->getSourceElementType() ==
+ Leader.GEP->getSourceElementType() &&
+ Other.VaryingIndex == Leader.VaryingIndex)
+ Group.push_back(J);
+ }
+
+ if (Group.size() < 2)
+ continue;
+
+ bool HasDistinctOffset = false;
+ for (unsigned K = 1; K != Group.size(); ++K) {
+ if (!haveSameVectorGEPOffset(Leader, Candidates[Group[K]])) {
+ HasDistinctOffset = true;
+ break;
+ }
+ }
+
+ if (!HasDistinctOffset)
+ continue;
+
+ GetElementPtrInst *FirstGEP = Leader.GEP;
+ IRBuilder<> BaseBuilder(FirstGEP);
+ BaseBuilder.SetCurrentDebugLocation(FirstGEP->getDebugLoc());
+
+ // Computing the varying portion once avoids a vector multiply-add for every
+ // unrolled part. The remaining uniform offsets can be calculated scalarly.
+ Value *CommonBase = BaseBuilder.CreateGEP(
+ FirstGEP->getSourceElementType(), FirstGEP->getPointerOperand(),
+ Leader.VaryingIndex, "vector.gep.base");
+
+ for (unsigned CandidateIndex : Group) {
+ VectorGEPCandidate &Current = Candidates[CandidateIndex];
+ GetElementPtrInst *GEP = Current.GEP;
+
+ IRBuilder<> Builder(GEP);
+ Builder.SetCurrentDebugLocation(GEP->getDebugLoc());
+
+ Type *OffsetType =
+ cast<VectorType>(GEP->getOperand(1)->getType())->getElementType();
+ Value *Offset = ConstantInt::get(OffsetType, 0);
+
+ for (const VectorGEPOffsetTerm &Term : Current.OffsetTerms) {
+ Offset =
+ Term.IsSub
+ ? Builder.CreateSub(Offset, Term.Scalar, "vector.gep.offset")
+ : Builder.CreateAdd(Offset, Term.Scalar, "vector.gep.offset");
+ }
+
+ Value *ByteOffset = Builder.CreateMul(
+ Offset, ConstantInt::get(OffsetType, Current.Stride),
+ "vector.gep.byte.offset");
+ Value *NewGEP =
+ Builder.CreatePtrAdd(CommonBase, ByteOffset, GEP->getName());
+
+ if (auto *NewI = dyn_cast<Instruction>(NewGEP)) {
+ NewI->copyMetadata(*GEP);
+ NewI->takeName(GEP);
+ }
+
+ DeadIndices.emplace_back(GEP->getOperand(1));
+ GEP->replaceAllUsesWith(NewGEP);
+ GEP->eraseFromParent();
+ Rewritten[CandidateIndex] = true;
+ Changed = true;
+ }
+ }
+
+ RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadIndices, TLI);
+ return Changed;
+}
+
bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
// Skip vector GEPs.
if (GEP->getType()->isVectorTy())
@@ -1414,6 +1639,9 @@ bool SeparateConstOffsetFromGEP::run(Function &F) {
Changed |= splitGEP(GEP);
// No need to split GEP ConstantExprs because all its indices are constant
// already.
+
+ if (LowerGEP)
+ Changed |= shareScalableVectorGEPBase(*B);
}
Changed |= reuniteExts(F);
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-gep.ll b/llvm/test/CodeGen/AArch64/sve-vector-gep.ll
new file mode 100644
index 0000000000000..10ff1ed76b7e0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-vector-gep.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "movprfx" --filter "m(ad|la).*z" --filter "(add|sub).*z" --version 6
+; RUN: opt -S -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=4 -scalable-vectorization=on < %s | llc -O3 -aarch64-enable-gep-opt=true | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @scalable_vector_geps(i64 %n, ptr %base, ptr %out) #0 {
+; CHECK-LABEL: scalable_vector_geps:
+; CHECK: movprfx z7, z0
+; CHECK: sub z7.d, z7.d, #1 // =0x1
+; CHECK: movprfx z7, z2
+; CHECK: mla z7.d, p0/m, z0.d, z3.d
+; CHECK: sub z0.d, z0.d, z1.d
+; CHECK: movprfx z16, z7
+; CHECK: sub z16.d, z16.d, #56 // =0x38
+; CHECK: add z17.d, z7.d, z4.d
+; CHECK: add z18.d, z7.d, z5.d
+; CHECK: add z7.d, z7.d, z6.d
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %n, %entry ], [ %next, %loop ]
+ %next = add nsw i64 %iv, -1
+ %src = getelementptr [56 x i8], ptr %base, i64 %next
+ %dst = getelementptr inbounds nuw [8 x i8], ptr %out, i64 %next
+ store ptr %src, ptr %dst, align 8
+ %continue = icmp samesign ugt i64 %iv, 1
+ br i1 %continue, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+attributes #0 = { nounwind "target-cpu"="neoverse-v2" }
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AArch64/scalable-vector-gep-common-base.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AArch64/scalable-vector-gep-common-base.ll
new file mode 100644
index 0000000000000..35246c73b634c
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AArch64/scalable-vector-gep-common-base.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='separate-const-offset-from-gep<lower-gep>' < %s | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+define void @scalable_gep_common_base(ptr %base, ptr %out0, ptr %out1, i64 %offset, i1 %cond) {
+; CHECK-LABEL: define void @scalable_gep_common_base(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], i64 [[OFFSET:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[INSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[OFFSET]], i64 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[INSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = sub <vscale x 2 x i64> [[IV]], [[SPLAT]]
+; CHECK-NEXT: [[VECTOR_GEP_BASE:%.*]] = getelementptr [56 x i8], ptr [[BASE]], <vscale x 2 x i64> [[IV]]
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, <vscale x 2 x ptr> [[VECTOR_GEP_BASE]], i64 -56
+; CHECK-NEXT: [[VECTOR_GEP_OFFSET:%.*]] = sub i64 -1, [[OFFSET]]
+; CHECK-NEXT: [[VECTOR_GEP_BYTE_OFFSET:%.*]] = mul i64 [[VECTOR_GEP_OFFSET]], 56
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, <vscale x 2 x ptr> [[VECTOR_GEP_BASE]], i64 [[VECTOR_GEP_BYTE_OFFSET]]
+; CHECK-NEXT: store <vscale x 2 x ptr> [[GEP0]], ptr [[OUT0]], align 16
+; CHECK-NEXT: store <vscale x 2 x ptr> [[GEP1]], ptr [[OUT1]], align 16
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %insert = insertelement <vscale x 2 x i64> poison, i64 %offset, i64 0
+ %splat = shufflevector <vscale x 2 x i64> %insert,
+ <vscale x 2 x i64> poison,
+ <vscale x 2 x i32> zeroinitializer
+ br label %loop
+
+loop:
+ %iv = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %next, %loop ]
+ %index0 = add <vscale x 2 x i64> %iv, splat (i64 -1)
+ %next = sub <vscale x 2 x i64> %iv, %splat
+ %index1 = add <vscale x 2 x i64> %next, splat (i64 -1)
+ %gep0 = getelementptr [56 x i8], ptr %base, <vscale x 2 x i64> %index0
+ %gep1 = getelementptr [56 x i8], ptr %base, <vscale x 2 x i64> %index1
+ store <vscale x 2 x ptr> %gep0, ptr %out0
+ store <vscale x 2 x ptr> %gep1, ptr %out1
+
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @identical_scalable_geps(ptr %base, ptr %out0, ptr %out1,
+; CHECK-LABEL: define void @identical_scalable_geps(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <vscale x 2 x i64> [[IV:%.*]]) {
+; CHECK-NEXT: [[INDEX:%.*]] = add <vscale x 2 x i64> [[IV]], splat (i64 -1)
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr [56 x i8], ptr [[BASE]], <vscale x 2 x i64> [[INDEX]]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr [56 x i8], ptr [[BASE]], <vscale x 2 x i64> [[INDEX]]
+; CHECK-NEXT: store <vscale x 2 x ptr> [[GEP0]], ptr [[OUT0]], align 16
+; CHECK-NEXT: store <vscale x 2 x ptr> [[GEP1]], ptr [[OUT1]], align 16
+; CHECK-NEXT: ret void
+;
+ <vscale x 2 x i64> %iv) {
+ %index = add <vscale x 2 x i64> %iv, splat (i64 -1)
+ %gep0 = getelementptr [56 x i8], ptr %base, <vscale x 2 x i64> %index
+ %gep1 = getelementptr [56 x i8], ptr %base, <vscale x 2 x i64> %index
+ store <vscale x 2 x ptr> %gep0, ptr %out0
+ store <vscale x 2 x ptr> %gep1, ptr %out1
+ ret void
+}
More information about the llvm-commits
mailing list