[llvm] [InstCombine] Sink widening casts used only by extractelements (PR #201529)
Barbara Mitic via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 06:43:27 PDT 2026
https://github.com/barbara-amd updated https://github.com/llvm/llvm-project/pull/201529
>From b64e58c1907906dbbb6235a6ad9cc8a17696c40f Mon Sep 17 00:00:00 2001
From: Barbara Mitic <Barbara.Mitic at amd.com>
Date: Wed, 3 Jun 2026 15:25:22 +0200
Subject: [PATCH 1/3] [InstCombine] Sink widening casts used only by
extractelements
InstCombine already canonicalizes extractelement(cast X) into
cast(extractelement X), but only when the cast has a single use. Extend
this to casts with multiple uses, provided the cast is a widening cast
(sext/zext/fpext) and every user is an extractelement that can itself
be sunk. Requiring every user to be sinkable guarantees the wide cast
is fully removed: the narrower operand keeps each scalar cast cheap,
and once all extracts are rewritten the wide cast has no remaining
users. If any user cannot be sunk (e.g. a variable-index extract in a
loop), the wide cast would survive next to the new scalar casts, so the
transform is skipped. Narrowing and fp-to-int casts keep the existing
single-use restriction, where multi-use scalarization could duplicate
bitcasts or turn a vector op into scalar ops in a loop. The existing
gate (bitcast and loop-profitability checks) is unchanged and moved
into a canSinkCastThroughExtract() helper for clarity.
---
.../InstCombine/InstCombineVectorOps.cpp | 53 ++++++++++---
.../Transforms/InstCombine/ExtractCast.ll | 77 +++++++++++++++++++
.../Transforms/LoopVectorize/histograms.ll | 6 +-
.../AArch64/scalarize-load-ext-extract.ll | 10 +--
4 files changed, 126 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 226253a7de3f8..b3392d1904908 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -410,6 +410,45 @@ static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
IndexC->getValue().zextOrTrunc(64));
}
+/// Decide whether extractelement(cast X), Index can be rewritten as
+/// cast(extractelement X, Index), where \p Cast is the source of the
+/// extractelement \p EI.
+static bool canSinkCastThroughExtract(CastInst *Cast, ExtractElementInst &EI,
+ Value *Index) {
+ // Bitcasts can change the number of vector elements; they are handled by
+ // foldBitcastExtElt instead.
+ if (Cast->getOpcode() == Instruction::BitCast)
+ return false;
+
+ // Sinking would move the cast next to the extractelement. Avoid sinking it
+ // into a loop: only do so when the index is constant or the cast and the
+ // extractelement are in the same basic block.
+ if (EI.getParent() != Cast->getParent() && !isa<ConstantInt>(Index))
+ return false;
+
+ // A single-use cast is always replaced (its only use is this extract).
+ if (Cast->hasOneUse())
+ return true;
+
+ // With multiple uses, only sink widening casts (sext/zext/fpext) whose every
+ // user is a sinkable extractelement (same block as the cast or a constant
+ // index). This ensures the wide cast is fully removed rather than left
+ // alongside the new scalar casts. Other casts are left to the single-use
+ // case.
+ switch (Cast->getOpcode()) {
+ case Instruction::SExt:
+ case Instruction::ZExt:
+ case Instruction::FPExt:
+ return all_of(Cast->users(), [&](User *U) {
+ auto *EE = dyn_cast<ExtractElementInst>(U);
+ return EE && (EE->getParent() == Cast->getParent() ||
+ isa<ConstantInt>(EE->getIndexOperand()));
+ });
+ default:
+ return false;
+ }
+}
+
Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
Value *SrcVec = EI.getVectorOperand();
Value *Index = EI.getIndexOperand();
@@ -600,18 +639,8 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
}
}
} else if (auto *CI = dyn_cast<CastInst>(I)) {
- // Canonicalize extractelement(cast) -> cast(extractelement).
- // Bitcasts can change the number of vector elements, and they cost
- // nothing.
- // If the CI has only one use, but that use is inside a loop, this
- // canonicalization is not profitable because it would turn a vector
- // operation into scalar operations inside the loop. Apply the transform
- // when:
- // - the index is constant and CI has one use, or
- // - the CI and EI are in the same basic block, so the cast won't be sunk
- // into a loop.
- if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast) &&
- (EI.getParent() == CI->getParent() || isa<ConstantInt>(Index))) {
+ // Canonicalize extractelement(cast X) -> cast(extractelement X).
+ if (canSinkCastThroughExtract(CI, EI, Index)) {
Value *EE = Builder.CreateExtractElement(CI->getOperand(0), Index);
return CastInst::Create(CI->getOpcode(), EE, EI.getType());
}
diff --git a/llvm/test/Transforms/InstCombine/ExtractCast.ll b/llvm/test/Transforms/InstCombine/ExtractCast.ll
index d5ad40f0caf8b..e6b880fc3cd42 100644
--- a/llvm/test/Transforms/InstCombine/ExtractCast.ll
+++ b/llvm/test/Transforms/InstCombine/ExtractCast.ll
@@ -23,3 +23,80 @@ define i32 @b(<4 x float> %I) {
ret i32 %K
}
+define { i32, i32 } @sext_all_extracts(<2 x i16> %v) {
+; CHECK-LABEL: @sext_all_extracts(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i16> [[V:%.*]], i64 0
+; CHECK-NEXT: [[X:%.*]] = sext i16 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[V]], i64 1
+; CHECK-NEXT: [[Y:%.*]] = sext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[R0:%.*]] = insertvalue { i32, i32 } poison, i32 [[X]], 0
+; CHECK-NEXT: [[R1:%.*]] = insertvalue { i32, i32 } [[R0]], i32 [[Y]], 1
+; CHECK-NEXT: ret { i32, i32 } [[R1]]
+;
+ %e = sext <2 x i16> %v to <2 x i32>
+ %x = extractelement <2 x i32> %e, i64 0
+ %y = extractelement <2 x i32> %e, i64 1
+ %r0 = insertvalue { i32, i32 } poison, i32 %x, 0
+ %r1 = insertvalue { i32, i32 } %r0, i32 %y, 1
+ ret { i32, i32 } %r1
+}
+
+define { i32, i32 } @zext_all_extracts(<2 x i16> %v) {
+; CHECK-LABEL: @zext_all_extracts(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i16> [[V:%.*]], i64 0
+; CHECK-NEXT: [[X:%.*]] = zext i16 [[TMP1]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[V]], i64 1
+; CHECK-NEXT: [[Y:%.*]] = zext i16 [[TMP2]] to i32
+; CHECK-NEXT: [[R0:%.*]] = insertvalue { i32, i32 } poison, i32 [[X]], 0
+; CHECK-NEXT: [[R1:%.*]] = insertvalue { i32, i32 } [[R0]], i32 [[Y]], 1
+; CHECK-NEXT: ret { i32, i32 } [[R1]]
+;
+ %e = zext <2 x i16> %v to <2 x i32>
+ %x = extractelement <2 x i32> %e, i64 0
+ %y = extractelement <2 x i32> %e, i64 1
+ %r0 = insertvalue { i32, i32 } poison, i32 %x, 0
+ %r1 = insertvalue { i32, i32 } %r0, i32 %y, 1
+ ret { i32, i32 } %r1
+}
+
+define i32 @sext_mixed_use(<2 x i16> %v, ptr %p) {
+; CHECK-LABEL: @sext_mixed_use(
+; CHECK-NEXT: [[E:%.*]] = sext <2 x i16> [[V:%.*]] to <2 x i32>
+; CHECK-NEXT: [[X:%.*]] = extractelement <2 x i32> [[E]], i64 0
+; CHECK-NEXT: store <2 x i32> [[E]], ptr [[P:%.*]], align 8
+; CHECK-NEXT: ret i32 [[X]]
+;
+ %e = sext <2 x i16> %v to <2 x i32>
+ %x = extractelement <2 x i32> %e, i64 0
+ store <2 x i32> %e, ptr %p
+ ret i32 %x
+}
+
+define void @zext_unsinkable_user(<2 x i32> %v, i64 %idx, i1 %cond, ptr %p, ptr %q) {
+; CHECK-LABEL: @zext_unsinkable_user(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[E:%.*]] = zext <2 x i32> [[V:%.*]] to <2 x i64>
+; CHECK-NEXT: [[C:%.*]] = extractelement <2 x i64> [[E]], i64 0
+; CHECK-NEXT: store i64 [[C]], ptr [[P:%.*]], align 4
+; CHECK-NEXT: br i1 [[COND:%.*]], label [[OTHER:%.*]], label [[EXIT:%.*]]
+; CHECK: other:
+; CHECK-NEXT: [[A:%.*]] = extractelement <2 x i64> [[E]], i64 [[IDX:%.*]]
+; CHECK-NEXT: store i64 [[A]], ptr [[Q:%.*]], align 4
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %e = zext <2 x i32> %v to <2 x i64>
+ %c = extractelement <2 x i64> %e, i64 0
+ store i64 %c, ptr %p
+ br i1 %cond, label %other, label %exit
+
+other:
+ %a = extractelement <2 x i64> %e, i64 %idx
+ store i64 %a, ptr %q
+ br label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index f4988bb01dfa4..6f20bf2005fc5 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -14,8 +14,10 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP9]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP4]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
index 8af7ae1b0ac77..e3c5bfbd43c0d 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
@@ -5,12 +5,10 @@ define noundef i32 @load_ext_extract(ptr %src) {
; CHECK-LABEL: define noundef range(i32 0, 1021) i32 @load_ext_extract(
; CHECK-SAME: ptr nofree readonly captures(none) [[SRC:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], <i32 0, i32 8, i32 16, i32 24>
-; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i32> [[TMP2]], <i32 255, i32 255, i32 255, i32 -1>
-; CHECK-NEXT: [[ADD3:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[SRC]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
+; CHECK-NEXT: [[ADD3:%.*]] = zext nneg i16 [[TMP2]] to i32
; CHECK-NEXT: ret i32 [[ADD3]]
;
entry:
>From 81bc367929dd2af19c98d741105a49ccc222f383 Mon Sep 17 00:00:00 2001
From: Barbara Mitic <Barbara.Mitic at amd.com>
Date: Wed, 29 Jul 2026 11:05:47 +0200
Subject: [PATCH 2/3] Revert "[InstCombine] Sink widening casts used only by
extractelements"
This reverts commit bf4da16f60f4636e02abb5bf48da99b8df3257b7.
---
.../InstCombine/InstCombineVectorOps.cpp | 53 +++----------
.../Transforms/InstCombine/ExtractCast.ll | 77 -------------------
.../Transforms/LoopVectorize/histograms.ll | 6 +-
.../AArch64/scalarize-load-ext-extract.ll | 10 ++-
4 files changed, 20 insertions(+), 126 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index b3392d1904908..226253a7de3f8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -410,45 +410,6 @@ static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
IndexC->getValue().zextOrTrunc(64));
}
-/// Decide whether extractelement(cast X), Index can be rewritten as
-/// cast(extractelement X, Index), where \p Cast is the source of the
-/// extractelement \p EI.
-static bool canSinkCastThroughExtract(CastInst *Cast, ExtractElementInst &EI,
- Value *Index) {
- // Bitcasts can change the number of vector elements; they are handled by
- // foldBitcastExtElt instead.
- if (Cast->getOpcode() == Instruction::BitCast)
- return false;
-
- // Sinking would move the cast next to the extractelement. Avoid sinking it
- // into a loop: only do so when the index is constant or the cast and the
- // extractelement are in the same basic block.
- if (EI.getParent() != Cast->getParent() && !isa<ConstantInt>(Index))
- return false;
-
- // A single-use cast is always replaced (its only use is this extract).
- if (Cast->hasOneUse())
- return true;
-
- // With multiple uses, only sink widening casts (sext/zext/fpext) whose every
- // user is a sinkable extractelement (same block as the cast or a constant
- // index). This ensures the wide cast is fully removed rather than left
- // alongside the new scalar casts. Other casts are left to the single-use
- // case.
- switch (Cast->getOpcode()) {
- case Instruction::SExt:
- case Instruction::ZExt:
- case Instruction::FPExt:
- return all_of(Cast->users(), [&](User *U) {
- auto *EE = dyn_cast<ExtractElementInst>(U);
- return EE && (EE->getParent() == Cast->getParent() ||
- isa<ConstantInt>(EE->getIndexOperand()));
- });
- default:
- return false;
- }
-}
-
Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
Value *SrcVec = EI.getVectorOperand();
Value *Index = EI.getIndexOperand();
@@ -639,8 +600,18 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
}
}
} else if (auto *CI = dyn_cast<CastInst>(I)) {
- // Canonicalize extractelement(cast X) -> cast(extractelement X).
- if (canSinkCastThroughExtract(CI, EI, Index)) {
+ // Canonicalize extractelement(cast) -> cast(extractelement).
+ // Bitcasts can change the number of vector elements, and they cost
+ // nothing.
+ // If the CI has only one use, but that use is inside a loop, this
+ // canonicalization is not profitable because it would turn a vector
+ // operation into scalar operations inside the loop. Apply the transform
+ // when:
+ // - the index is constant and CI has one use, or
+ // - the CI and EI are in the same basic block, so the cast won't be sunk
+ // into a loop.
+ if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast) &&
+ (EI.getParent() == CI->getParent() || isa<ConstantInt>(Index))) {
Value *EE = Builder.CreateExtractElement(CI->getOperand(0), Index);
return CastInst::Create(CI->getOpcode(), EE, EI.getType());
}
diff --git a/llvm/test/Transforms/InstCombine/ExtractCast.ll b/llvm/test/Transforms/InstCombine/ExtractCast.ll
index e6b880fc3cd42..d5ad40f0caf8b 100644
--- a/llvm/test/Transforms/InstCombine/ExtractCast.ll
+++ b/llvm/test/Transforms/InstCombine/ExtractCast.ll
@@ -23,80 +23,3 @@ define i32 @b(<4 x float> %I) {
ret i32 %K
}
-define { i32, i32 } @sext_all_extracts(<2 x i16> %v) {
-; CHECK-LABEL: @sext_all_extracts(
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i16> [[V:%.*]], i64 0
-; CHECK-NEXT: [[X:%.*]] = sext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[V]], i64 1
-; CHECK-NEXT: [[Y:%.*]] = sext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[R0:%.*]] = insertvalue { i32, i32 } poison, i32 [[X]], 0
-; CHECK-NEXT: [[R1:%.*]] = insertvalue { i32, i32 } [[R0]], i32 [[Y]], 1
-; CHECK-NEXT: ret { i32, i32 } [[R1]]
-;
- %e = sext <2 x i16> %v to <2 x i32>
- %x = extractelement <2 x i32> %e, i64 0
- %y = extractelement <2 x i32> %e, i64 1
- %r0 = insertvalue { i32, i32 } poison, i32 %x, 0
- %r1 = insertvalue { i32, i32 } %r0, i32 %y, 1
- ret { i32, i32 } %r1
-}
-
-define { i32, i32 } @zext_all_extracts(<2 x i16> %v) {
-; CHECK-LABEL: @zext_all_extracts(
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i16> [[V:%.*]], i64 0
-; CHECK-NEXT: [[X:%.*]] = zext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[V]], i64 1
-; CHECK-NEXT: [[Y:%.*]] = zext i16 [[TMP2]] to i32
-; CHECK-NEXT: [[R0:%.*]] = insertvalue { i32, i32 } poison, i32 [[X]], 0
-; CHECK-NEXT: [[R1:%.*]] = insertvalue { i32, i32 } [[R0]], i32 [[Y]], 1
-; CHECK-NEXT: ret { i32, i32 } [[R1]]
-;
- %e = zext <2 x i16> %v to <2 x i32>
- %x = extractelement <2 x i32> %e, i64 0
- %y = extractelement <2 x i32> %e, i64 1
- %r0 = insertvalue { i32, i32 } poison, i32 %x, 0
- %r1 = insertvalue { i32, i32 } %r0, i32 %y, 1
- ret { i32, i32 } %r1
-}
-
-define i32 @sext_mixed_use(<2 x i16> %v, ptr %p) {
-; CHECK-LABEL: @sext_mixed_use(
-; CHECK-NEXT: [[E:%.*]] = sext <2 x i16> [[V:%.*]] to <2 x i32>
-; CHECK-NEXT: [[X:%.*]] = extractelement <2 x i32> [[E]], i64 0
-; CHECK-NEXT: store <2 x i32> [[E]], ptr [[P:%.*]], align 8
-; CHECK-NEXT: ret i32 [[X]]
-;
- %e = sext <2 x i16> %v to <2 x i32>
- %x = extractelement <2 x i32> %e, i64 0
- store <2 x i32> %e, ptr %p
- ret i32 %x
-}
-
-define void @zext_unsinkable_user(<2 x i32> %v, i64 %idx, i1 %cond, ptr %p, ptr %q) {
-; CHECK-LABEL: @zext_unsinkable_user(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[E:%.*]] = zext <2 x i32> [[V:%.*]] to <2 x i64>
-; CHECK-NEXT: [[C:%.*]] = extractelement <2 x i64> [[E]], i64 0
-; CHECK-NEXT: store i64 [[C]], ptr [[P:%.*]], align 4
-; CHECK-NEXT: br i1 [[COND:%.*]], label [[OTHER:%.*]], label [[EXIT:%.*]]
-; CHECK: other:
-; CHECK-NEXT: [[A:%.*]] = extractelement <2 x i64> [[E]], i64 [[IDX:%.*]]
-; CHECK-NEXT: store i64 [[A]], ptr [[Q:%.*]], align 4
-; CHECK-NEXT: br label [[EXIT]]
-; CHECK: exit:
-; CHECK-NEXT: ret void
-;
-entry:
- %e = zext <2 x i32> %v to <2 x i64>
- %c = extractelement <2 x i64> %e, i64 0
- store i64 %c, ptr %p
- br i1 %cond, label %other, label %exit
-
-other:
- %a = extractelement <2 x i64> %e, i64 %idx
- store i64 %a, ptr %q
- br label %exit
-
-exit:
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 6f20bf2005fc5..f4988bb01dfa4 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -14,10 +14,8 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i8], ptr [[INDICES]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[BUCKETS]], i64 [[TMP4]]
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
index e3c5bfbd43c0d..8af7ae1b0ac77 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-load-ext-extract.ll
@@ -5,10 +5,12 @@ define noundef i32 @load_ext_extract(ptr %src) {
; CHECK-LABEL: define noundef range(i32 0, 1021) i32 @load_ext_extract(
; CHECK-SAME: ptr nofree readonly captures(none) [[SRC:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[SRC]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i16>
-; CHECK-NEXT: [[TMP2:%.*]] = tail call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> [[TMP1]])
-; CHECK-NEXT: [[ADD3:%.*]] = zext nneg i16 [[TMP2]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP14]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], <i32 0, i32 8, i32 16, i32 24>
+; CHECK-NEXT: [[TMP5:%.*]] = and <4 x i32> [[TMP2]], <i32 255, i32 255, i32 255, i32 -1>
+; CHECK-NEXT: [[ADD3:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
; CHECK-NEXT: ret i32 [[ADD3]]
;
entry:
>From d8a6a3a13d4b12dc521f2e747552ba427d2aeab8 Mon Sep 17 00:00:00 2001
From: Barbara Mitic <Barbara.Mitic at amd.com>
Date: Wed, 29 Jul 2026 14:51:52 +0200
Subject: [PATCH 3/3] [VectorCombine] Scalarize multi-use casts feeding only
extractelements
Extend InstCombine's existing canonicalization
extractelement (castop X), Idx --> castop (extractelement X, Idx)
from single-use casts to multi-use ones. Every user of the cast must be
an extractelement, which guarantees the vector cast is fully replaced
rather than surviving next to the new scalar casts.
Unlike the single-use case, which InstCombine does unconditionally, the
multi-use rewrite trades one vector op for N scalar ops per lane and
needs a cost model to justify it. InstCombine has none, so this lives
in VectorCombine instead: any non-bitcast cast is accepted, and
profitability is left entirely to the cost model.
---
.../Transforms/Vectorize/VectorCombine.cpp | 96 ++++
.../AArch64/scalarize-cast-extract.ll | 70 +++
.../AArch64/scalarize-cast-extract.ll | 185 +++++++
.../AMDGPU/scalarize-cast-extract.ll | 61 +++
.../X86/scalarize-cast-extract.ll | 474 ++++++++++++++++++
5 files changed, 886 insertions(+)
create mode 100644 llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-cast-extract.ll
create mode 100644 llvm/test/Transforms/VectorCombine/AArch64/scalarize-cast-extract.ll
create mode 100644 llvm/test/Transforms/VectorCombine/AMDGPU/scalarize-cast-extract.ll
create mode 100644 llvm/test/Transforms/VectorCombine/X86/scalarize-cast-extract.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 726f564b1aad9..baf92eb41f6d8 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -56,6 +56,7 @@ STATISTIC(NumShufOfBitcast, "Number of shuffles moved after bitcast");
STATISTIC(NumScalarOps, "Number of scalar unary + binary ops formed");
STATISTIC(NumScalarCmp, "Number of scalar compares formed");
STATISTIC(NumScalarIntrinsic, "Number of scalar intrinsic calls formed");
+STATISTIC(NumScalarCast, "Number of scalar casts formed");
static cl::opt<bool> DisableVectorCombine(
"disable-vector-combine", cl::init(false), cl::Hidden,
@@ -149,6 +150,7 @@ class VectorCombine {
bool foldShuffleFromReductions(Instruction &I);
bool foldShuffleChainsToReduce(Instruction &I);
bool foldCastFromReductions(Instruction &I);
+ bool scalarizeCastExtract(Instruction &I);
bool foldSignBitReductionCmp(Instruction &I);
bool foldReductionZeroTest(Instruction &I);
bool foldICmpEqZeroVectorReduce(Instruction &I);
@@ -4360,6 +4362,98 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
return true;
}
+/// Try to scalarize a multi-use cast whose users are all extractelements:
+/// extractelement (castop X), Idx --> castop (extractelement X, Idx)
+///
+/// InstCombine canonicalizes this for single-use casts. Multi-use cast
+/// needs a cost model, since it turns one vector cast into N scalar casts;
+/// every user must be an extractelement or the vector cast would survive
+/// anyway.
+bool VectorCombine::scalarizeCastExtract(Instruction &I) {
+ if (!TTI.allowVectorElementIndexingUsingGEP())
+ return false;
+
+ auto *Cast = dyn_cast<CastInst>(&I);
+ if (!Cast || Cast->hasOneUse())
+ return false;
+
+ // Every cast except bitcast is elementwise, so moving one past an
+ // extractelement is always valid and profitability is left to the cost model
+ // below. A bitcast can change the lane count, so it's excluded here;
+ // InstCombine's foldBitcastExtElt() handles that case,
+ Instruction::CastOps Opc = Cast->getOpcode();
+ if (Opc == Instruction::BitCast)
+ return false;
+
+ // TODO: Fixed-width only; scalable vectors need separate cost validation.
+ auto *SrcTy = dyn_cast<FixedVectorType>(Cast->getOperand(0)->getType());
+ auto *DstTy = dyn_cast<FixedVectorType>(Cast->getType());
+ if (!SrcTy || !DstTy)
+ return false;
+
+ // Every user has to be an extractelement, or the vector cast would survive
+ // alongside the new scalar casts. A variable index is only allowed in the
+ // cast's own block: elsewhere it may sit in a loop, and unlike a constant
+ // index it can't be hoisted back out if the cast was loop-invariant.
+ SmallVector<ExtractElementInst *, 4> Extracts;
+ for (User *U : Cast->users()) {
+ auto *EE = dyn_cast<ExtractElementInst>(U);
+ if (!EE)
+ return false;
+ if (!isa<ConstantInt>(EE->getIndexOperand()) &&
+ EE->getParent() != Cast->getParent())
+ return false;
+ if (!EE->use_empty())
+ Extracts.push_back(EE);
+ }
+ if (Extracts.empty())
+ return false;
+
+ // OldCost: the vector cast + its extracts.
+ // NewCost: a narrow extract + scalar cast per lane.
+ InstructionCost OldCost = TTI.getCastInstrCost(
+ Opc, DstTy, SrcTy, TTI::getCastContextHint(Cast), CostKind, Cast);
+ // The scalar cast reads the new narrow extract, so unlike the vector cast it
+ // has no load or store to fold into.
+ InstructionCost ScalarCastCost = TTI.getCastInstrCost(
+ Opc, DstTy->getElementType(), SrcTy->getElementType(),
+ TTI::CastContextHint::None, CostKind);
+ InstructionCost NewCost = 0;
+ for (ExtractElementInst *EE : Extracts) {
+ // Only an in-range constant maps to a known lane; anything else is costed
+ // as an unknown index.
+ unsigned Idx = -1U;
+ if (auto *IdxC = dyn_cast<ConstantInt>(EE->getIndexOperand()))
+ if (IdxC->getValue().ult(DstTy->getNumElements()))
+ Idx = IdxC->getZExtValue();
+ OldCost += TTI.getVectorInstrCost(*EE, DstTy, CostKind, Idx);
+ NewCost += TTI.getVectorInstrCost(Instruction::ExtractElement, SrcTy,
+ CostKind, Idx) +
+ ScalarCastCost;
+ }
+
+ LLVM_DEBUG(dbgs() << "Found a multi-use cast feeding extracts: " << *Cast
+ << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
+ << "\n");
+
+ if (!NewCost.isValid() || NewCost >= OldCost)
+ return false;
+
+ for (ExtractElementInst *EE : Extracts) {
+ Builder.SetInsertPoint(EE);
+ Value *Extract = Builder.CreateExtractElement(Cast->getOperand(0),
+ EE->getIndexOperand());
+ Worklist.pushValue(Extract);
+ Value *ScalarCast = Builder.CreateCast(Opc, Extract, EE->getType());
+ if (auto *SC = dyn_cast<Instruction>(ScalarCast))
+ SC->copyIRFlags(Cast);
+ replaceValue(*EE, *ScalarCast, /*Erase=*/false);
+ ++NumScalarCast;
+ }
+
+ return true;
+}
+
/// Determine if its more efficient to fold:
/// reduce(trunc(x)) -> trunc(reduce(x)).
/// reduce(sext(x)) -> sext(reduce(x)).
@@ -6517,6 +6611,8 @@ bool VectorCombine::run() {
return true;
if (scalarizeExtExtract(I))
return true;
+ if (scalarizeCastExtract(I))
+ return true;
if (scalarizeVPIntrinsic(I))
return true;
if (foldInterleaveIntrinsics(I))
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-cast-extract.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-cast-extract.ll
new file mode 100644
index 0000000000000..2b33f5757dea5
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/scalarize-cast-extract.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -O3 -mtriple=arm64-apple-darwinos -S %s | FileCheck %s
+
+define i64 @load_ext_extract_gep(ptr %p, ptr %tab) {
+; CHECK-LABEL: define i64 @load_ext_extract_gep(
+; CHECK-SAME: ptr nofree readonly captures(none) [[P:%.*]], ptr nofree readonly captures(none) [[TAB:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[P]], align 16
+; CHECK-NEXT: [[I0:%.*]] = zext i16 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 14
+; CHECK-NEXT: [[TMP2:%.*]] = load i16, ptr [[TMP1]], align 2
+; CHECK-NEXT: [[I7:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[G0:%.*]] = getelementptr [8 x i8], ptr [[TAB]], i64 [[I0]]
+; CHECK-NEXT: [[G7:%.*]] = getelementptr [8 x i8], ptr [[TAB]], i64 [[I7]]
+; CHECK-NEXT: [[L0:%.*]] = load i64, ptr [[G0]], align 8
+; CHECK-NEXT: [[L7:%.*]] = load i64, ptr [[G7]], align 8
+; CHECK-NEXT: [[S:%.*]] = add i64 [[L7]], [[L0]]
+; CHECK-NEXT: ret i64 [[S]]
+;
+entry:
+ %v = load <8 x i16>, ptr %p, align 16
+ %e = zext <8 x i16> %v to <8 x i64>
+ %i0 = extractelement <8 x i64> %e, i64 0
+ %i7 = extractelement <8 x i64> %e, i64 7
+ %g0 = getelementptr i64, ptr %tab, i64 %i0
+ %g7 = getelementptr i64, ptr %tab, i64 %i7
+ %l0 = load i64, ptr %g0, align 8
+ %l7 = load i64, ptr %g7, align 8
+ %s = add i64 %l0, %l7
+ ret i64 %s
+}
+
+define void @loop_invariant_cast(<8 x i16> %v, i64 %n) {
+; CHECK-LABEL: define void @loop_invariant_cast(
+; CHECK-SAME: <8 x i16> [[V:%.*]], i64 [[N:%.*]]) local_unnamed_addr {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: tail call void @use.i64(i64 [[A]])
+; CHECK-NEXT: tail call void @use.i64(i64 [[B]])
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %e = zext <8 x i16> %v to <8 x i64>
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare void @use.i64(i64)
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/scalarize-cast-extract.ll b/llvm/test/Transforms/VectorCombine/AArch64/scalarize-cast-extract.ll
new file mode 100644
index 0000000000000..4a57ddc4d61b5
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/scalarize-cast-extract.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p vector-combine -mtriple=arm64-apple-darwinos -S %s | FileCheck %s
+
+declare void @use.i64(i64)
+declare void @use.i32(i32)
+
+define void @zext_v8i16_to_v8i64(<8 x i16> %v) {
+; CHECK-LABEL: define void @zext_v8i16_to_v8i64(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = zext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @sext_v8i16_2_lanes_used(<8 x i16> %v) {
+; CHECK-LABEL: define void @sext_v8i16_2_lanes_used(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 1
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 1
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @sext_v8i16_4_lanes_used(<8 x i16> %v) {
+; CHECK-LABEL: define void @sext_v8i16_4_lanes_used(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 1
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i16> [[V]], i64 2
+; CHECK-NEXT: [[C:%.*]] = sext i16 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[D:%.*]] = sext i16 [[TMP4]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: call void @use.i64(i64 [[C]])
+; CHECK-NEXT: call void @use.i64(i64 [[D]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 1
+ %c = extractelement <8 x i64> %e, i64 2
+ %d = extractelement <8 x i64> %e, i64 3
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ call void @use.i64(i64 %c)
+ call void @use.i64(i64 %d)
+ ret void
+}
+
+define void @neg_sext_v8i16_all_lanes_used(<8 x i16> %v) {
+; CHECK-LABEL: define void @neg_sext_v8i16_all_lanes_used(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[E:%.*]] = sext <8 x i16> [[V]] to <8 x i64>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i64> [[E]], i64 0
+; CHECK-NEXT: [[B:%.*]] = extractelement <8 x i64> [[E]], i64 1
+; CHECK-NEXT: [[C:%.*]] = extractelement <8 x i64> [[E]], i64 2
+; CHECK-NEXT: [[D:%.*]] = extractelement <8 x i64> [[E]], i64 3
+; CHECK-NEXT: [[F:%.*]] = extractelement <8 x i64> [[E]], i64 4
+; CHECK-NEXT: [[G:%.*]] = extractelement <8 x i64> [[E]], i64 5
+; CHECK-NEXT: [[H:%.*]] = extractelement <8 x i64> [[E]], i64 6
+; CHECK-NEXT: [[I:%.*]] = extractelement <8 x i64> [[E]], i64 7
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: call void @use.i64(i64 [[C]])
+; CHECK-NEXT: call void @use.i64(i64 [[D]])
+; CHECK-NEXT: call void @use.i64(i64 [[F]])
+; CHECK-NEXT: call void @use.i64(i64 [[G]])
+; CHECK-NEXT: call void @use.i64(i64 [[H]])
+; CHECK-NEXT: call void @use.i64(i64 [[I]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 1
+ %c = extractelement <8 x i64> %e, i64 2
+ %d = extractelement <8 x i64> %e, i64 3
+ %f = extractelement <8 x i64> %e, i64 4
+ %g = extractelement <8 x i64> %e, i64 5
+ %h = extractelement <8 x i64> %e, i64 6
+ %i = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ call void @use.i64(i64 %c)
+ call void @use.i64(i64 %d)
+ call void @use.i64(i64 %f)
+ call void @use.i64(i64 %g)
+ call void @use.i64(i64 %h)
+ call void @use.i64(i64 %i)
+ ret void
+}
+
+define void @ext_extract_keeps_priority(<4 x i8> %v) {
+; CHECK-LABEL: define void @ext_extract_keeps_priority(
+; CHECK-SAME: <4 x i8> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i8> [[V]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i8> [[TMP1]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 24
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[TMP6]], 255
+; CHECK-NEXT: [[TMP8:%.*]] = and i32 [[TMP2]], 255
+; CHECK-NEXT: [[E:%.*]] = zext <4 x i8> [[V]] to <4 x i32>
+; CHECK-NEXT: [[A:%.*]] = extractelement <4 x i32> [[E]], i64 0
+; CHECK-NEXT: [[B:%.*]] = extractelement <4 x i32> [[E]], i64 1
+; CHECK-NEXT: [[C:%.*]] = extractelement <4 x i32> [[E]], i64 2
+; CHECK-NEXT: [[D:%.*]] = extractelement <4 x i32> [[E]], i64 3
+; CHECK-NEXT: call void @use.i32(i32 [[TMP8]])
+; CHECK-NEXT: call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT: call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT: call void @use.i32(i32 [[TMP3]])
+; CHECK-NEXT: ret void
+;
+ %e = zext <4 x i8> %v to <4 x i32>
+ %a = extractelement <4 x i32> %e, i64 0
+ %b = extractelement <4 x i32> %e, i64 1
+ %c = extractelement <4 x i32> %e, i64 2
+ %d = extractelement <4 x i32> %e, i64 3
+ call void @use.i32(i32 %a)
+ call void @use.i32(i32 %b)
+ call void @use.i32(i32 %c)
+ call void @use.i32(i32 %d)
+ ret void
+}
+
+define void @sext_v8i8_to_v8i64(<8 x i8> %v) {
+; CHECK-LABEL: define void @sext_v8i8_to_v8i64(
+; CHECK-SAME: <8 x i8> [[V:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i8> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i8 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = sext i8 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i8> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @neg_scalable(<vscale x 4 x i16> %v) {
+; CHECK-LABEL: define void @neg_scalable(
+; CHECK-SAME: <vscale x 4 x i16> [[V:%.*]]) {
+; CHECK-NEXT: [[E:%.*]] = zext <vscale x 4 x i16> [[V]] to <vscale x 4 x i64>
+; CHECK-NEXT: [[A:%.*]] = extractelement <vscale x 4 x i64> [[E]], i64 0
+; CHECK-NEXT: [[B:%.*]] = extractelement <vscale x 4 x i64> [[E]], i64 1
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = zext <vscale x 4 x i16> %v to <vscale x 4 x i64>
+ %a = extractelement <vscale x 4 x i64> %e, i64 0
+ %b = extractelement <vscale x 4 x i64> %e, i64 1
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/AMDGPU/scalarize-cast-extract.ll b/llvm/test/Transforms/VectorCombine/AMDGPU/scalarize-cast-extract.ll
new file mode 100644
index 0000000000000..c30bac8fdce67
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AMDGPU/scalarize-cast-extract.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=vector-combine -S %s | FileCheck %s
+
+declare void @use.i64(i64)
+declare void @use.double(double)
+
+define void @sext_8i8_to_8i64(<8 x i8> %v) {
+; CHECK-LABEL: define void @sext_8i8_to_8i64(
+; CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i8> [[V]], i64 0
+; CHECK-NEXT: [[X:%.*]] = sext i8 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i8> [[V]], i64 7
+; CHECK-NEXT: [[Y:%.*]] = sext i8 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[X]])
+; CHECK-NEXT: call void @use.i64(i64 [[Y]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i8> %v to <8 x i64>
+ %x = extractelement <8 x i64> %e, i64 0
+ %y = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %x)
+ call void @use.i64(i64 %y)
+ ret void
+}
+
+define void @fpext_8f32_to_8f64(<8 x float> %v) {
+; CHECK-LABEL: define void @fpext_8f32_to_8f64(
+; CHECK-SAME: <8 x float> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x float> [[V]], i64 0
+; CHECK-NEXT: [[X:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[V]], i64 7
+; CHECK-NEXT: [[Y:%.*]] = fpext float [[TMP2]] to double
+; CHECK-NEXT: call void @use.double(double [[X]])
+; CHECK-NEXT: call void @use.double(double [[Y]])
+; CHECK-NEXT: ret void
+;
+ %e = fpext <8 x float> %v to <8 x double>
+ %x = extractelement <8 x double> %e, i64 0
+ %y = extractelement <8 x double> %e, i64 7
+ call void @use.double(double %x)
+ call void @use.double(double %y)
+ ret void
+}
+
+define void @neg_zext_8i8_to_8i64(<8 x i8> %v) {
+; CHECK-LABEL: define void @neg_zext_8i8_to_8i64(
+; CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = zext <8 x i8> [[V]] to <8 x i64>
+; CHECK-NEXT: [[X:%.*]] = extractelement <8 x i64> [[E]], i64 0
+; CHECK-NEXT: [[Y:%.*]] = extractelement <8 x i64> [[E]], i64 7
+; CHECK-NEXT: call void @use.i64(i64 [[X]])
+; CHECK-NEXT: call void @use.i64(i64 [[Y]])
+; CHECK-NEXT: ret void
+;
+ %e = zext <8 x i8> %v to <8 x i64>
+ %x = extractelement <8 x i64> %e, i64 0
+ %y = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %x)
+ call void @use.i64(i64 %y)
+ ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/scalarize-cast-extract.ll b/llvm/test/Transforms/VectorCombine/X86/scalarize-cast-extract.ll
new file mode 100644
index 0000000000000..8c60f0648ed77
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/scalarize-cast-extract.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p vector-combine -mtriple=x86_64-- -mcpu=x86-64 -S %s | FileCheck %s
+
+declare void @use.i16(i16)
+declare void @use.i64(i64)
+declare void @use.double(double)
+declare void @use.v8i64(<8 x i64>)
+
+define void @sext_v8i16_const_idx(<8 x i16> %v) {
+; CHECK-LABEL: define void @sext_v8i16_const_idx(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[C:%.*]] = sext i16 [[TMP3]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: call void @use.i64(i64 [[C]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 3
+ %c = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ call void @use.i64(i64 %c)
+ ret void
+}
+
+define void @zext_v8i16_const_idx(<8 x i16> %v) {
+; CHECK-LABEL: define void @zext_v8i16_const_idx(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[B:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[C:%.*]] = zext i16 [[TMP3]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: call void @use.i64(i64 [[C]])
+; CHECK-NEXT: ret void
+;
+ %e = zext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 3
+ %c = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ call void @use.i64(i64 %c)
+ ret void
+}
+
+define void @fpext_v8f32_const_idx(<8 x float> %v) {
+; CHECK-LABEL: define void @fpext_v8f32_const_idx(
+; CHECK-SAME: <8 x float> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x float> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[V]], i64 3
+; CHECK-NEXT: [[B:%.*]] = fpext float [[TMP2]] to double
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[V]], i64 7
+; CHECK-NEXT: [[C:%.*]] = fpext float [[TMP3]] to double
+; CHECK-NEXT: call void @use.double(double [[A]])
+; CHECK-NEXT: call void @use.double(double [[B]])
+; CHECK-NEXT: call void @use.double(double [[C]])
+; CHECK-NEXT: ret void
+;
+ %e = fpext <8 x float> %v to <8 x double>
+ %a = extractelement <8 x double> %e, i64 0
+ %b = extractelement <8 x double> %e, i64 3
+ %c = extractelement <8 x double> %e, i64 7
+ call void @use.double(double %a)
+ call void @use.double(double %b)
+ call void @use.double(double %c)
+ ret void
+}
+
+define void @sext_v8i16_all_lanes_used(<8 x i16> %v) {
+; CHECK-LABEL: define void @sext_v8i16_all_lanes_used(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 1
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i16> [[V]], i64 2
+; CHECK-NEXT: [[C:%.*]] = sext i16 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[D:%.*]] = sext i16 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[V]], i64 4
+; CHECK-NEXT: [[F:%.*]] = sext i16 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[V]], i64 5
+; CHECK-NEXT: [[G:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[V]], i64 6
+; CHECK-NEXT: [[H:%.*]] = sext i16 [[TMP7]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[I:%.*]] = sext i16 [[TMP8]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: call void @use.i64(i64 [[C]])
+; CHECK-NEXT: call void @use.i64(i64 [[D]])
+; CHECK-NEXT: call void @use.i64(i64 [[F]])
+; CHECK-NEXT: call void @use.i64(i64 [[G]])
+; CHECK-NEXT: call void @use.i64(i64 [[H]])
+; CHECK-NEXT: call void @use.i64(i64 [[I]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 1
+ %c = extractelement <8 x i64> %e, i64 2
+ %d = extractelement <8 x i64> %e, i64 3
+ %f = extractelement <8 x i64> %e, i64 4
+ %g = extractelement <8 x i64> %e, i64 5
+ %h = extractelement <8 x i64> %e, i64 6
+ %i = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ call void @use.i64(i64 %c)
+ call void @use.i64(i64 %d)
+ call void @use.i64(i64 %f)
+ call void @use.i64(i64 %g)
+ call void @use.i64(i64 %h)
+ call void @use.i64(i64 %i)
+ ret void
+}
+
+define void @trunc_v8i64(<8 x i64> %v) {
+; CHECK-LABEL: define void @trunc_v8i64(
+; CHECK-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i64> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i64> [[V]], i64 3
+; CHECK-NEXT: [[B:%.*]] = trunc i64 [[TMP2]] to i16
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i64> [[V]], i64 7
+; CHECK-NEXT: [[C:%.*]] = trunc i64 [[TMP3]] to i16
+; CHECK-NEXT: call void @use.i16(i16 [[A]])
+; CHECK-NEXT: call void @use.i16(i16 [[B]])
+; CHECK-NEXT: call void @use.i16(i16 [[C]])
+; CHECK-NEXT: ret void
+;
+ %e = trunc <8 x i64> %v to <8 x i16>
+ %a = extractelement <8 x i16> %e, i64 0
+ %b = extractelement <8 x i16> %e, i64 3
+ %c = extractelement <8 x i16> %e, i64 7
+ call void @use.i16(i16 %a)
+ call void @use.i16(i16 %b)
+ call void @use.i16(i16 %c)
+ ret void
+}
+
+define void @sitofp_v8i64(<8 x i64> %v) {
+; CHECK-LABEL: define void @sitofp_v8i64(
+; CHECK-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i64> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sitofp i64 [[TMP1]] to double
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i64> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = sitofp i64 [[TMP2]] to double
+; CHECK-NEXT: call void @use.double(double [[A]])
+; CHECK-NEXT: call void @use.double(double [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sitofp <8 x i64> %v to <8 x double>
+ %a = extractelement <8 x double> %e, i64 0
+ %b = extractelement <8 x double> %e, i64 7
+ call void @use.double(double %a)
+ call void @use.double(double %b)
+ ret void
+}
+
+define void @variable_idx_same_block(<8 x i16> %v, i64 %idx) {
+; CHECK-LABEL: define void @variable_idx_same_block(
+; CHECK-SAME: <8 x i16> [[V:%.*]], i64 [[IDX:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 [[IDX]]
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 %idx
+ %b = extractelement <8 x i64> %e, i64 3
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @zext_nneg_preserved(<8 x i16> %v) {
+; CHECK-LABEL: define void @zext_nneg_preserved(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = zext nneg i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = zext nneg i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = zext nneg <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @dead_extract_among_live(<8 x i16> %v) {
+; CHECK-LABEL: define void @dead_extract_among_live(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %dead = extractelement <8 x i64> %e, i64 1
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @duplicate_lane(<8 x i16> %v) {
+; CHECK-LABEL: define void @duplicate_lane(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 3
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 3
+ %b = extractelement <8 x i64> %e, i64 3
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @extracts_feed_stores(<8 x i16> %v, ptr %p, ptr %q) {
+; CHECK-LABEL: define void @extracts_feed_stores(
+; CHECK-SAME: <8 x i16> [[V:%.*]], ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP2]] to i64
+; CHECK-NEXT: store i64 [[A]], ptr [[P]], align 8
+; CHECK-NEXT: store i64 [[B]], ptr [[Q]], align 8
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ store i64 %a, ptr %p
+ store i64 %b, ptr %q
+ ret void
+}
+
+define void @cast_of_load(ptr %p) {
+; CHECK-LABEL: define void @cast_of_load(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <8 x i16>, ptr [[P]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %v = load <8 x i16>, ptr %p
+ %e = zext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @const_idx_in_loop(<8 x i16> %v, i64 %n) {
+; CHECK-LABEL: define void @const_idx_in_loop(
+; CHECK-SAME: <8 x i16> [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <8 x i16> [[V]], i64 0
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = zext i16 [[TMP1]] to i64
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %e = zext <8 x i16> %v to <8 x i64>
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %a = extractelement <8 x i64> %e, i64 0
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @oob_wide_index(<8 x i16> %v) {
+; CHECK-LABEL: define void @oob_wide_index(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <8 x i16> [[V]], i64 7
+; CHECK-NEXT: [[B:%.*]] = sext i16 [[TMP1]] to i64
+; CHECK-NEXT: call void @use.i64(i64 poison)
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i128 36893488147419103232
+ %b = extractelement <8 x i64> %e, i64 7
+ call void @use.i64(i64 %a)
+ call void @use.i64(i64 %b)
+ ret void
+}
+
+define void @neg_cheap_cast(<8 x i8> %v) {
+; CHECK-LABEL: define void @neg_cheap_cast(
+; CHECK-SAME: <8 x i8> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = zext <8 x i8> [[V]] to <8 x i16>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i16> [[E]], i64 0
+; CHECK-NEXT: [[B:%.*]] = extractelement <8 x i16> [[E]], i64 1
+; CHECK-NEXT: call void @use.i16(i16 [[A]])
+; CHECK-NEXT: call void @use.i16(i16 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = zext <8 x i8> %v to <8 x i16>
+ %a = extractelement <8 x i16> %e, i64 0
+ %b = extractelement <8 x i16> %e, i64 1
+ call void @use.i16(i16 %a)
+ call void @use.i16(i16 %b)
+ ret void
+}
+
+define void @neg_single_use(<8 x i16> %v) {
+; CHECK-LABEL: define void @neg_single_use(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = sext <8 x i16> [[V]] to <8 x i64>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i64> [[E]], i64 0
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ call void @use.i64(i64 %a)
+ ret void
+}
+
+define void @neg_non_extract_user(<8 x i16> %v) {
+; CHECK-LABEL: define void @neg_non_extract_user(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = sext <8 x i16> [[V]] to <8 x i64>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i64> [[E]], i64 0
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: call void @use.v8i64(<8 x i64> [[E]])
+; CHECK-NEXT: ret void
+;
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ call void @use.i64(i64 %a)
+ call void @use.v8i64(<8 x i64> %e)
+ ret void
+}
+
+define void @neg_bitcast(<4 x i32> %v) {
+; CHECK-LABEL: define void @neg_bitcast(
+; CHECK-SAME: <4 x i32> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = bitcast <4 x i32> [[V]] to <8 x i16>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i16> [[E]], i64 0
+; CHECK-NEXT: [[B:%.*]] = extractelement <8 x i16> [[E]], i64 7
+; CHECK-NEXT: call void @use.i16(i16 [[A]])
+; CHECK-NEXT: call void @use.i16(i16 [[B]])
+; CHECK-NEXT: ret void
+;
+ %e = bitcast <4 x i32> %v to <8 x i16>
+ %a = extractelement <8 x i16> %e, i64 0
+ %b = extractelement <8 x i16> %e, i64 7
+ call void @use.i16(i16 %a)
+ call void @use.i16(i16 %b)
+ ret void
+}
+
+define void @neg_variable_idx_other_block(<8 x i16> %v, i64 %idx, i1 %cond) {
+; CHECK-LABEL: define void @neg_variable_idx_other_block(
+; CHECK-SAME: <8 x i16> [[V:%.*]], i64 [[IDX:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[E:%.*]] = sext <8 x i16> [[V]] to <8 x i64>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i64> [[E]], i64 0
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: br i1 [[COND]], label %[[OTHER:.*]], label %[[EXIT:.*]]
+; CHECK: [[OTHER]]:
+; CHECK-NEXT: [[B:%.*]] = extractelement <8 x i64> [[E]], i64 [[IDX]]
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ call void @use.i64(i64 %a)
+ br i1 %cond, label %other, label %exit
+
+other:
+ %b = extractelement <8 x i64> %e, i64 %idx
+ call void @use.i64(i64 %b)
+ br label %exit
+
+exit:
+ ret void
+}
+
+define void @neg_variable_idx_in_loop(<8 x i16> %v, i64 %n) {
+; CHECK-LABEL: define void @neg_variable_idx_in_loop(
+; CHECK-SAME: <8 x i16> [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[E:%.*]] = sext <8 x i16> [[V]] to <8 x i64>
+; CHECK-NEXT: [[A:%.*]] = extractelement <8 x i64> [[E]], i64 0
+; CHECK-NEXT: call void @use.i64(i64 [[A]])
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[B:%.*]] = extractelement <8 x i64> [[E]], i64 [[IV]]
+; CHECK-NEXT: call void @use.i64(i64 [[B]])
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %e = sext <8 x i16> %v to <8 x i64>
+ %a = extractelement <8 x i64> %e, i64 0
+ call void @use.i64(i64 %a)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %b = extractelement <8 x i64> %e, i64 %iv
+ call void @use.i64(i64 %b)
+ %iv.next = add i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list