[llvm] [InstCombine] Fold extractelement of GEPs into a GEP of extractelements (PR #208767)
Vigneshwar Jayakumar via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 10 09:17:03 PDT 2026
https://github.com/VigneshwarJ created https://github.com/llvm/llvm-project/pull/208767
Fold, for a non-constant index N:
extractelement <gep(P0, Idxs), gep(P1, Idxs), ...>, N
-> gep (extractelement <P0, P1, ...>, N), Idxs
when the single-use GEPs building the vector share the same index operands and differ only in their pointer operand, it scalarizes the GEP and exposes the (often loop-invariant) pointer selection so passes such as LICM can hoist it, while the common indices stay with the GEP.
>From 74baffdc1d319b3f3bb67645f79dfb811f561b22 Mon Sep 17 00:00:00 2001
From: vigneshwar jayakumar <vigneshwar.jayakumar at amd.com>
Date: Fri, 10 Jul 2026 10:01:40 -0500
Subject: [PATCH] [InstCombine] Fold extractelement of GEPs into a GEP of
extractelements
Fold, for a non-constant index N:
extractelement <gep(P0, Idxs), gep(P1, Idxs), ...>, N
-> gep (extractelement <P0, P1, ...>, N), Idxs
when the single-use GEPs building the vector share the same index operands and
differ only in their pointer operand, it scalarizes the GEP and exposes the
(often loop-invariant) pointer selection so passes such as LICM can hoist it,
while the common indices stay with the GEP.
---
.../InstCombine/InstCombineVectorOps.cpp | 107 ++++++++++++++++++
.../InstCombine/extractelement-gep.ll | 89 +++++++++++++++
.../PhaseOrdering/extractelement-gep-licm.ll | 46 ++++++++
3 files changed, 242 insertions(+)
create mode 100644 llvm/test/Transforms/InstCombine/extractelement-gep.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/extractelement-gep-licm.ll
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 86f64363d4186..383eeab5a05fc 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -410,6 +410,105 @@ static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
IndexC->getValue().zextOrTrunc(64));
}
+/// Collect the per-lane values of a fixed-length vector fully defined by an
+/// insertelement chain into poison/undef with constant lane indices. Returns
+/// false if the vector is not built this way or any lane is left undefined.
+static bool collectInsertedVectorElements(Value *V,
+ SmallVectorImpl<Value *> &Elts) {
+ auto *VecTy = dyn_cast<FixedVectorType>(V->getType());
+ if (!VecTy)
+ return false;
+
+ Elts.assign(VecTy->getNumElements(), nullptr);
+ // Walk the chain outermost-first; the first value seen for a lane wins.
+ for (Value *Cur = V; auto *IE = dyn_cast<InsertElementInst>(Cur);
+ Cur = IE->getOperand(0)) {
+ // Intermediate chain nodes must be single-use so the whole build becomes
+ // dead once the extractelement(s) fold. The outermost node (Cur == V) may
+ // feed several extractelements; the caller checks it has no other users.
+ if (Cur != V && !IE->hasOneUse())
+ return false;
+ auto *Lane = dyn_cast<ConstantInt>(IE->getOperand(2));
+ if (!Lane || Lane->getValue().uge(Elts.size()))
+ return false;
+ if (Value *&Slot = Elts[Lane->getZExtValue()]; !Slot)
+ Slot = IE->getOperand(1);
+ }
+ return all_of(Elts, [](Value *E) { return E; });
+}
+
+/// Fold an extractelement of a vector of GEPs into a scalar GEP -- the GEP
+/// analog of the extractelement-of-unop/binop/cmp scalarization folds below:
+/// extractelement (<gep P0, Idxs>, <gep P1, Idxs>, ...), N
+/// --> gep (extractelement <P0, P1, ...>, N), Idxs
+/// The GEPs must be single-use and share the same source element type and index
+/// operands, differing only in the pointer operand. This scalarizes the GEP and
+/// exposes a (frequently loop-invariant) pointer selection that LICM can hoist,
+/// while the common indices stay with the GEP. Only run for a non-constant
+/// index; the constant-index case is handled by the insertelement fold above.
+static Instruction *foldExtractOfGEPVector(ExtractElementInst &EI,
+ InstCombiner::BuilderTy &Builder) {
+ auto *VecTy = dyn_cast<FixedVectorType>(EI.getVectorOperandType());
+ if (!VecTy || !VecTy->getElementType()->isPointerTy())
+ return nullptr;
+
+ // The source vector must feed only extractelements, otherwise the
+ // insertelement build (and the GEPs) stay live after folding and we would
+ // duplicate IR. Multiple extractelement users are fine -- they each fold and
+ // the build dies once the last one does.
+ if (!all_of(EI.getVectorOperand()->users(),
+ [](User *U) { return isa<ExtractElementInst>(U); }))
+ return nullptr;
+
+ SmallVector<Value *, 4> Elts;
+ if (!collectInsertedVectorElements(EI.getVectorOperand(), Elts))
+ return nullptr;
+
+ auto *FirstGEP = dyn_cast<GetElementPtrInst>(Elts[0]);
+ if (!FirstGEP || !FirstGEP->hasOneUse())
+ return nullptr;
+
+ // Bail on GEPs that index into a vector; creating one is discouraged.
+ Type *SrcElemTy = FirstGEP->getSourceElementType();
+ if (SrcElemTy->isVectorTy())
+ return nullptr;
+
+ SmallVector<Value *, 4> Indices(FirstGEP->idx_begin(), FirstGEP->idx_end());
+ if (Indices.empty())
+ return nullptr;
+
+ GEPNoWrapFlags NW = FirstGEP->getNoWrapFlags();
+ SmallVector<Value *, 4> BasePtrs;
+ BasePtrs.reserve(Elts.size());
+ BasePtrs.push_back(FirstGEP->getPointerOperand());
+
+ for (Value *Elt : drop_begin(Elts)) {
+ auto *GEP = dyn_cast<GetElementPtrInst>(Elt);
+ if (!GEP || !GEP->hasOneUse() || GEP->getSourceElementType() != SrcElemTy ||
+ GEP->getNumIndices() != Indices.size())
+ return nullptr;
+ for (auto [K, Idx] : enumerate(Indices))
+ if (GEP->getOperand(1 + K) != Idx)
+ return nullptr;
+ // The result is whichever arm the index selects at runtime, so a no-wrap
+ // flag holds only if it holds for every arm.
+ NW = NW & GEP->getNoWrapFlags();
+ BasePtrs.push_back(GEP->getPointerOperand());
+ }
+
+ // Build <P0, P1, ...>, extract the selected base, then re-apply the common
+ // GEP. Constant base pointers fold the vector + extract to a loop-invariant
+ // value that LICM can hoist.
+ Value *BaseVec = PoisonValue::get(
+ FixedVectorType::get(BasePtrs[0]->getType(), BasePtrs.size()));
+ for (auto [Lane, BasePtr] : enumerate(BasePtrs))
+ BaseVec =
+ Builder.CreateInsertElement(BaseVec, BasePtr, Builder.getInt64(Lane));
+ Value *Picked = Builder.CreateExtractElement(BaseVec, EI.getIndexOperand());
+
+ return GetElementPtrInst::Create(SrcElemTy, Picked, Indices, NW);
+}
+
Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
Value *SrcVec = EI.getVectorOperand();
Value *Index = EI.getIndexOperand();
@@ -417,6 +516,14 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
SQ.getWithInstruction(&EI)))
return replaceInstUsesWith(EI, V);
+ // extractelement (<gep(P0, Idxs), gep(P1, Idxs)>), N (non-constant N)
+ // --> gep (extractelement <P0, P1>, N), Idxs
+ // Scalarizes the GEP so a loop-invariant pointer selection can be hoisted
+ // (the indices stay with the GEP).
+ if (!isa<Constant>(Index))
+ if (Instruction *R = foldExtractOfGEPVector(EI, Builder))
+ return R;
+
// extractelt (select %x, %vec1, %vec2), %const ->
// select %x, %vec1[%const], %vec2[%const]
// TODO: Support constant folding of multiple select operands:
diff --git a/llvm/test/Transforms/InstCombine/extractelement-gep.ll b/llvm/test/Transforms/InstCombine/extractelement-gep.ll
new file mode 100644
index 0000000000000..1a842d49196e0
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/extractelement-gep.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; Fold an extractelement of a vector of GEPs that differ only in their pointer
+; operand into a scalar GEP of the extracted base pointer.
+
+define ptr @extractelt_of_gep_vector(ptr %p0, ptr %p1, i32 %n, i64 %off) {
+; CHECK-LABEL: define ptr @extractelt_of_gep_vector(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], i32 [[N:%.*]], i64 [[OFF:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[P0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP1]], ptr [[P1]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 [[N]]
+; CHECK-NEXT: [[E:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[OFF]]
+; CHECK-NEXT: ret ptr [[E]]
+;
+ %g0 = getelementptr i8, ptr %p0, i64 %off
+ %g1 = getelementptr i8, ptr %p1, i64 %off
+ %v0 = insertelement <2 x ptr> poison, ptr %g0, i64 0
+ %v1 = insertelement <2 x ptr> %v0, ptr %g1, i64 1
+ %e = extractelement <2 x ptr> %v1, i32 %n
+ ret ptr %e
+}
+
+; Negative: GEPs differ in their index operands, so there is no common GEP.
+define ptr @extractelt_of_gep_vector_mismatched_index(ptr %p0, ptr %p1, i32 %n, i64 %o0, i64 %o1) {
+; CHECK-LABEL: define ptr @extractelt_of_gep_vector_mismatched_index(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], i32 [[N:%.*]], i64 [[O0:%.*]], i64 [[O1:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[O0]]
+; CHECK-NEXT: [[G1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[O1]]
+; CHECK-NEXT: [[V0:%.*]] = insertelement <2 x ptr> poison, ptr [[G0]], i64 0
+; CHECK-NEXT: [[V1:%.*]] = insertelement <2 x ptr> [[V0]], ptr [[G1]], i64 1
+; CHECK-NEXT: [[E:%.*]] = extractelement <2 x ptr> [[V1]], i32 [[N]]
+; CHECK-NEXT: ret ptr [[E]]
+;
+ %g0 = getelementptr i8, ptr %p0, i64 %o0
+ %g1 = getelementptr i8, ptr %p1, i64 %o1
+ %v0 = insertelement <2 x ptr> poison, ptr %g0, i64 0
+ %v1 = insertelement <2 x ptr> %v0, ptr %g1, i64 1
+ %e = extractelement <2 x ptr> %v1, i32 %n
+ ret ptr %e
+}
+
+; Negative: the vector has a non-extractelement use, so folding would duplicate IR.
+define ptr @extractelt_of_gep_vector_multiuse(ptr %p0, ptr %p1, i32 %n, i64 %off, ptr %sink) {
+; CHECK-LABEL: define ptr @extractelt_of_gep_vector_multiuse(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], i32 [[N:%.*]], i64 [[OFF:%.*]], ptr [[SINK:%.*]]) {
+; CHECK-NEXT: [[G0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[OFF]]
+; CHECK-NEXT: [[G1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[OFF]]
+; CHECK-NEXT: [[V0:%.*]] = insertelement <2 x ptr> poison, ptr [[G0]], i64 0
+; CHECK-NEXT: [[V1:%.*]] = insertelement <2 x ptr> [[V0]], ptr [[G1]], i64 1
+; CHECK-NEXT: [[E:%.*]] = extractelement <2 x ptr> [[V1]], i32 [[N]]
+; CHECK-NEXT: store <2 x ptr> [[V1]], ptr [[SINK]], align 16
+; CHECK-NEXT: ret ptr [[E]]
+;
+ %g0 = getelementptr i8, ptr %p0, i64 %off
+ %g1 = getelementptr i8, ptr %p1, i64 %off
+ %v0 = insertelement <2 x ptr> poison, ptr %g0, i64 0
+ %v1 = insertelement <2 x ptr> %v0, ptr %g1, i64 1
+ %e = extractelement <2 x ptr> %v1, i32 %n
+ store <2 x ptr> %v1, ptr %sink
+ ret ptr %e
+}
+
+; The same vector feeding multiple extractelements still folds.
+define void @extractelt_of_gep_vector_two_extracts(ptr %p0, ptr %p1, i32 %n, i32 %m, i64 %off) {
+; CHECK-LABEL: define void @extractelt_of_gep_vector_two_extracts(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]], i32 [[N:%.*]], i32 [[M:%.*]], i64 [[OFF:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[P0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP1]], ptr [[P1]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x ptr> [[TMP2]], i32 [[N]]
+; CHECK-NEXT: [[E0:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[OFF]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x ptr> poison, ptr [[P0]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> [[TMP4]], ptr [[P1]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x ptr> [[TMP5]], i32 [[M]]
+; CHECK-NEXT: [[E1:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[OFF]]
+; CHECK-NEXT: call void @use2(ptr [[E0]], ptr [[E1]])
+; CHECK-NEXT: ret void
+;
+ %g0 = getelementptr i8, ptr %p0, i64 %off
+ %g1 = getelementptr i8, ptr %p1, i64 %off
+ %v0 = insertelement <2 x ptr> poison, ptr %g0, i64 0
+ %v1 = insertelement <2 x ptr> %v0, ptr %g1, i64 1
+ %e0 = extractelement <2 x ptr> %v1, i32 %n
+ %e1 = extractelement <2 x ptr> %v1, i32 %m
+ call void @use2(ptr %e0, ptr %e1)
+ ret void
+}
+
+declare void @use2(ptr, ptr)
diff --git a/llvm/test/Transforms/PhaseOrdering/extractelement-gep-licm.ll b/llvm/test/Transforms/PhaseOrdering/extractelement-gep-licm.ll
new file mode 100644
index 0000000000000..40df4b9161ea1
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/extractelement-gep-licm.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='instcombine,loop-simplify,loop-mssa(licm)' -S < %s | FileCheck %s
+
+; InstCombine scalarizes the extractelement of a vector of GEPs, exposing the
+; loop-invariant base pointer selection so that LICM can hoist it out of the
+; loop, leaving only the varying-offset GEP and the load inside the loop.
+
+ at g = external addrspace(3) global [0 x i8]
+declare void @use(i8)
+
+define void @extractelement_gep_hoist(i32 %idx, i64 %n) {
+; CHECK-LABEL: define void @extractelement_gep_hoist(
+; CHECK-SAME: i32 [[IDX:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x ptr addrspace(3)> <ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @g, i64 100), ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @g, i64 200)>, i32 [[IDX]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[OFF:%.*]] = shl i64 [[I]], 6
+; CHECK-NEXT: [[SEL:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 [[OFF]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr addrspace(3) [[SEL]], align 1
+; CHECK-NEXT: call void @use(i8 [[VAL]])
+; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %off = shl i64 %i, 6
+ %a = getelementptr i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @g, i64 100), i64 %off
+ %b = getelementptr i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @g, i64 200), i64 %off
+ %v0 = insertelement <2 x ptr addrspace(3)> poison, ptr addrspace(3) %a, i64 0
+ %v1 = insertelement <2 x ptr addrspace(3)> %v0, ptr addrspace(3) %b, i64 1
+ %sel = extractelement <2 x ptr addrspace(3)> %v1, i32 %idx
+ %val = load i8, ptr addrspace(3) %sel
+ call void @use(i8 %val)
+ %i.next = add i64 %i, 1
+ %done = icmp eq i64 %i.next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list