[llvm] [InstCombine] Sink widening casts used only by extractelements (PR #201529)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 4 01:46:14 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Barbara Mitic (barbara-amd)
<details>
<summary>Changes</summary>
InstCombine already canonicalizes extractelement(cast X) into cast(extractelement X), but only when the cast has a single use. Extend this to casts with multiple uses, provided the cast is a widening cast (sext/zext/fpext) and every user is an extractelement. In that case the operand vector is narrower than the result and the wide cast has no other users, so sinking it into each extract is at least as cheap and removes the wide cast entirely. Narrowing and fp-to-int casts keep the existing single-use restriction, where multi-use scalarization could duplicate bitcasts or turn a vector op into scalar ops in a loop. The existing gate (bitcast and loop-profitability checks) is unchanged and moved into a canSinkCastThroughExtract() helper for clarity.
---
Full diff: https://github.com/llvm/llvm-project/pull/201529.diff
4 Files Affected:
- (modified) llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp (+35-12)
- (modified) llvm/test/Transforms/InstCombine/ExtractCast.ll (+48)
- (modified) llvm/test/Transforms/LoopVectorize/histograms.ll (+4-3)
- (modified) llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll (+4-6)
``````````diff
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 3a7fbbcb468da..7a34c8f0a0166 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -410,6 +410,39 @@ static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
IndexC->getValue().zextOrTrunc(64));
}
+/// Decide whether extractelement(cast X), Index can be rewritten as
+/// cast(extractelement X, Index), where \p Cast is the source of the
+/// extractelement \p EI.
+static bool canSinkCastThroughExtract(CastInst *Cast, ExtractElementInst &EI,
+ Value *Index) {
+ // Bitcasts can change the number of vector elements; they are handled by
+ // foldBitcastExtElt instead.
+ if (Cast->getOpcode() == Instruction::BitCast)
+ return false;
+
+ // Sinking would move the cast next to the extractelement. Avoid sinking it
+ // into a loop: only do so when the index is constant or the cast and the
+ // extractelement are in the same basic block.
+ if (EI.getParent() != Cast->getParent() && !isa<ConstantInt>(Index))
+ return false;
+
+ // A single-use cast is always replaced (its only use is this extract).
+ if (Cast->hasOneUse())
+ return true;
+
+ // With multiple uses, only sink widening casts (sext/zext/fpext) used solely
+ // by extractelements: the narrower operand keeps this cheap and fully
+ // replaces the wide cast. Other casts are left to the single-use case.
+ switch (Cast->getOpcode()) {
+ case Instruction::SExt:
+ case Instruction::ZExt:
+ case Instruction::FPExt:
+ return all_of(Cast->users(), IsaPred<ExtractElementInst>);
+ default:
+ return false;
+ }
+}
+
Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
Value *SrcVec = EI.getVectorOperand();
Value *Index = EI.getIndexOperand();
@@ -600,18 +633,8 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
}
}
} else if (auto *CI = dyn_cast<CastInst>(I)) {
- // Canonicalize extractelement(cast) -> cast(extractelement).
- // Bitcasts can change the number of vector elements, and they cost
- // nothing.
- // If the CI has only one use, but that use is inside a loop, this
- // canonicalization is not profitable because it would turn a vector
- // operation into scalar operations inside the loop. Apply the transform
- // when:
- // - the index is constant and CI has one use, or
- // - the CI and EI are in the same basic block, so the cast won't be sunk
- // into a loop.
- if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast) &&
- (EI.getParent() == CI->getParent() || isa<ConstantInt>(Index))) {
+ // Canonicalize extractelement(cast X) -> cast(extractelement X).
+ if (canSinkCastThroughExtract(CI, EI, Index)) {
Value *EE = Builder.CreateExtractElement(CI->getOperand(0), Index);
return CastInst::Create(CI->getOpcode(), EE, EI.getType());
}
diff --git a/llvm/test/Transforms/InstCombine/ExtractCast.ll b/llvm/test/Transforms/InstCombine/ExtractCast.ll
index d5ad40f0caf8b..880e3171fea2c 100644
--- a/llvm/test/Transforms/InstCombine/ExtractCast.ll
+++ b/llvm/test/Transforms/InstCombine/ExtractCast.ll
@@ -23,3 +23,51 @@ define i32 @b(<4 x float> %I) {
ret i32 %K
}
+define { i32, i32 } @sext_all_extracts(<2 x i16> %v) {
+; CHECK-LABEL: @sext_all_extracts(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i16> [[V:%.*]], i64 0
+; CHECK-NEXT: [[X:%.*]] = sext i16 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[V]], i64 1
+; CHECK-NEXT: [[Y:%.*]] = sext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[R0:%.*]] = insertvalue { i32, i32 } poison, i32 [[X]], 0
+; CHECK-NEXT: [[R1:%.*]] = insertvalue { i32, i32 } [[R0]], i32 [[Y]], 1
+; CHECK-NEXT: ret { i32, i32 } [[R1]]
+;
+ %e = sext <2 x i16> %v to <2 x i32>
+ %x = extractelement <2 x i32> %e, i64 0
+ %y = extractelement <2 x i32> %e, i64 1
+ %r0 = insertvalue { i32, i32 } poison, i32 %x, 0
+ %r1 = insertvalue { i32, i32 } %r0, i32 %y, 1
+ ret { i32, i32 } %r1
+}
+
+define { i32, i32 } @zext_all_extracts(<2 x i16> %v) {
+; CHECK-LABEL: @zext_all_extracts(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i16> [[V:%.*]], i64 0
+; CHECK-NEXT: [[X:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[V]], i64 1
+; CHECK-NEXT: [[Y:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[R0:%.*]] = insertvalue { i32, i32 } poison, i32 [[X]], 0
+; CHECK-NEXT: [[R1:%.*]] = insertvalue { i32, i32 } [[R0]], i32 [[Y]], 1
+; CHECK-NEXT: ret { i32, i32 } [[R1]]
+;
+ %e = zext <2 x i16> %v to <2 x i32>
+ %x = extractelement <2 x i32> %e, i64 0
+ %y = extractelement <2 x i32> %e, i64 1
+ %r0 = insertvalue { i32, i32 } poison, i32 %x, 0
+ %r1 = insertvalue { i32, i32 } %r0, i32 %y, 1
+ ret { i32, i32 } %r1
+}
+
+define i32 @sext_mixed_use(<2 x i16> %v, ptr %p) {
+; CHECK-LABEL: @sext_mixed_use(
+; CHECK-NEXT: [[E:%.*]] = sext <2 x i16> [[V:%.*]] to <2 x i32>
+; CHECK-NEXT: [[X:%.*]] = extractelement <2 x i32> [[E]], i64 0
+; CHECK-NEXT: store <2 x i32> [[E]], ptr [[P:%.*]], align 8
+; CHECK-NEXT: ret i32 [[X]]
+;
+ %e = sext <2 x i16> %v to <2 x i32>
+ %x = extractelement <2 x i32> %e, i64 0
+ store <2 x i32> %e, ptr %p
+ ret i32 %x
+}
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 5850ac3195c39..92bcbb49f2ccc 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -14,9 +14,10 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP9]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP3]], i64 0
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
index 8210fd4113013..96000f2329255 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
@@ -5,12 +5,10 @@ define noundef i32 @load_ext_extract(ptr %src) {
; CHECK-LABEL: define noundef range(i32 0, 1021) i32 @load_ext_extract(
; CHECK-SAME: ptr readonly captures(none) [[SRC:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], <i32 0, i32 8, i32 16, i32 24>
-; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i32> [[TMP2]], <i32 255, i32 255, i32 255, i32 -1>
-; CHECK-NEXT: [[ADD3:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[SRC]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[ADD3:%.*]] = zext nneg i16 [[TMP2]] to i32
; CHECK-NEXT: ret i32 [[ADD3]]
;
entry:
``````````
</details>
https://github.com/llvm/llvm-project/pull/201529
More information about the llvm-commits
mailing list