[llvm] [VPlan] Sink single-scalar VPI to replicate regions (PR #208906)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 02:09:45 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/208906
>From 751c1c8b7e458cadae9bccfe2ee6a05caed87a7f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 11 Jul 2026 12:32:25 +0100
Subject: [PATCH 1/2] [VPlan] Sink single-scalar VPI to replicate regions
VPlanUnroll currently only knows how to handle single-scalar
VPInstructions that are sunk into replicate regions: currently the
newly-introduced scalar-casts are sunk. The patch also generalizes the
existing special cases correctly using vputils::doesGeneratePerAllLanes.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 20 +++---
.../AArch64/force-target-instruction-cost.ll | 11 ++--
.../LoopVectorize/X86/induction-costs.ll | 11 ++--
.../X86/predicated-replicate-feeding-cast.ll | 15 +++--
llvm/test/Transforms/LoopVectorize/as_cast.ll | 21 ++++---
.../LoopVectorize/cast-induction.ll | 4 +-
.../Transforms/LoopVectorize/induction.ll | 61 ++++++++++++-------
.../LoopVectorize/single-value-blend-phis.ll | 5 +-
8 files changed, 89 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 32ce8392ed08d..69c786eef9bfa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -311,22 +311,22 @@ static bool sinkScalarOperands(VPlan &Plan) {
SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
VPBasicBlock *SinkTo, VPValue *Op) {
- auto *Candidate =
- dyn_cast_or_null<VPSingleDefRecipe>(Op->getDefiningRecipe());
- if (!Candidate)
- return;
-
- // We only know how to sink VPReplicateRecipes and VPScalarIVStepsRecipes
- // for now.
- if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Candidate))
+ auto *Candidate = dyn_cast<VPSingleDefRecipe>(Op);
+ if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
+ VPInstruction>(Candidate))
return;
if (Candidate->getParent() == SinkTo ||
vputils::cannotHoistOrSinkRecipe(*Candidate, /*Sinking=*/true))
return;
- if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
- if (!ScalarVFOnly && RepR->isSingleScalar())
+ if (ScalarVFOnly && vputils::doesGeneratePerAllLanes(Candidate))
+ return;
+
+ // Due to a limitation in VPlanUnroll, we currently know only how to sink
+ // single-scalar VPInstructions.
+ if (auto *VPI = dyn_cast<VPInstruction>(Candidate))
+ if (!vputils::isSingleScalar(VPI))
return;
WorkList.insert({SinkTo, Candidate});
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 07fc18e46c4fe..8851695e8c579 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -601,10 +601,10 @@ define void @forced_scalar_instr(ptr %gep.dst) {
; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; COMMON-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; COMMON-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 4)
-; COMMON-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
; COMMON-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; COMMON: [[PRED_STORE_IF]]:
+; COMMON-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[INDEX]]
; COMMON-NEXT: [[TMP6:%.*]] = or i32 [[TMP0]], 1
; COMMON-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4
@@ -613,8 +613,9 @@ define void @forced_scalar_instr(ptr %gep.dst) {
; COMMON-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; COMMON-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; COMMON: [[PRED_STORE_IF1]]:
+; COMMON-NEXT: [[TMP23:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
-; COMMON-NEXT: [[TMP9:%.*]] = add i32 [[TMP0]], 1
+; COMMON-NEXT: [[TMP9:%.*]] = add i32 [[TMP23]], 1
; COMMON-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP8]]
; COMMON-NEXT: [[TMP11:%.*]] = or i32 [[TMP9]], 1
; COMMON-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4
@@ -623,8 +624,9 @@ define void @forced_scalar_instr(ptr %gep.dst) {
; COMMON-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; COMMON-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; COMMON: [[PRED_STORE_IF3]]:
+; COMMON-NEXT: [[TMP24:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 2
-; COMMON-NEXT: [[TMP14:%.*]] = add i32 [[TMP0]], 2
+; COMMON-NEXT: [[TMP14:%.*]] = add i32 [[TMP24]], 2
; COMMON-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP13]]
; COMMON-NEXT: [[TMP16:%.*]] = or i32 [[TMP14]], 1
; COMMON-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4
@@ -633,8 +635,9 @@ define void @forced_scalar_instr(ptr %gep.dst) {
; COMMON-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; COMMON-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; COMMON: [[PRED_STORE_IF5]]:
+; COMMON-NEXT: [[TMP25:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 3
-; COMMON-NEXT: [[TMP19:%.*]] = add i32 [[TMP0]], 3
+; COMMON-NEXT: [[TMP19:%.*]] = add i32 [[TMP25]], 3
; COMMON-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[GEP_DST]], i64 [[TMP18]]
; COMMON-NEXT: [[TMP21:%.*]] = or i32 [[TMP19]], 1
; COMMON-NEXT: store i32 [[TMP21]], ptr [[TMP20]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 165d81b285a19..8efac9b10d5c2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -585,31 +585,34 @@ define void @wide_iv_trunc(ptr %dst, i64 %N) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
; CHECK-NEXT: store i32 [[TMP1]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[TMP12]], 1
; CHECK-NEXT: store i32 [[TMP6]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; CHECK: [[PRED_STORE_IF3]]:
-; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP1]], 2
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP13]], 2
; CHECK-NEXT: store i32 [[TMP8]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; CHECK: [[PRED_STORE_CONTINUE4]]:
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_IF5]]:
-; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP1]], 3
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP14]], 3
; CHECK-NEXT: store i32 [[TMP10]], ptr [[DST]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; CHECK: [[PRED_STORE_CONTINUE6]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
index 63bfa10ff2934..2ab0f685308f2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
@@ -23,17 +23,22 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[A]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_SDIV_IF]] ]
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
; CHECK: [[PRED_SDIV_IF1]]:
+; CHECK-NEXT: [[TMP8:%.*]] = sdiv i16 1, [[A]]
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP8]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[C1]], i16 0, i16 [[TMP6]]
-; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP11]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ [[TMP7]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <2 x i16> zeroinitializer, <2 x i16> [[TMP10]]
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
; CHECK: [[PRED_SDIV_IF3]]:
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP11]] to i8
; CHECK-NEXT: [[TMP13:%.*]] = sdiv i8 [[TMP12]], [[B]]
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i8> poison, i8 [[TMP13]], i64 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
@@ -41,7 +46,9 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i8> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP14]], %[[PRED_SDIV_IF3]] ]
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_IF5]]:
-; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP12]], [[B]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i16 [[TMP16]] to i8
+; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP17]], [[B]]
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> [[TMP15]], i8 [[TMP18]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_CONTINUE6]]:
diff --git a/llvm/test/Transforms/LoopVectorize/as_cast.ll b/llvm/test/Transforms/LoopVectorize/as_cast.ll
index 7d93501da4411..d2b281d133fb9 100644
--- a/llvm/test/Transforms/LoopVectorize/as_cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/as_cast.ll
@@ -7,7 +7,6 @@ define void @loop_invariant_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[IN]] to ptr
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
@@ -17,6 +16,7 @@ define void @loop_invariant_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[IN]] to ptr
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP3]]
; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP5]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
@@ -24,7 +24,8 @@ define void @loop_invariant_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr addrspace(1) [[IN]] to ptr
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP6]]
; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
@@ -64,20 +65,20 @@ define void @loop_varying_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[TMP0]], 6
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP5:%.*]] = addrspacecast ptr addrspace(1) [[TMP4]] to ptr
-; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr addrspace(1) [[TMP7]] to ptr
; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = addrspacecast ptr addrspace(1) [[TMP7]] to ptr
+; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_IF1]]:
-; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[IN]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = addrspacecast ptr addrspace(1) [[TMP11]] to ptr
+; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP12]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
index b087ed0e0b412..76088b630635d 100644
--- a/llvm/test/Transforms/LoopVectorize/cast-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -276,16 +276,16 @@ define void @cast_induction_tail_folding(ptr %A) {
; IC2-NEXT: [[INDEX1:%.*]] = add i32 [[INDEX]], 1
; IC2-NEXT: [[TMP2:%.*]] = icmp ule i32 [[INDEX]], 2
; IC2-NEXT: [[TMP3:%.*]] = icmp ule i32 [[INDEX1]], 2
-; IC2-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64
-; IC2-NEXT: [[TMP6:%.*]] = sext i32 [[INDEX1]] to i64
; IC2-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; IC2: [[PRED_STORE_IF]]:
+; IC2-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64
; IC2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
; IC2-NEXT: store i32 [[INDEX]], ptr [[TMP5]], align 4
; IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
; IC2: [[PRED_STORE_CONTINUE]]:
; IC2-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; IC2: [[PRED_STORE_IF1]]:
+; IC2-NEXT: [[TMP6:%.*]] = sext i32 [[INDEX1]] to i64
; IC2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
; IC2-NEXT: store i32 [[INDEX1]], ptr [[TMP7]], align 4
; IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index f2ede41effb54..6a0ab3015c61e 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -4909,11 +4909,11 @@ define i32 @PR32419(i32 %a, i16 %b) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_UREM_CONTINUE2]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; CHECK: pred.urem.if:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP1]]
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0
; CHECK-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -4922,7 +4922,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2]]
; CHECK: pred.urem.if1:
-; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP18]], 1
; CHECK-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP11]], i64 1
; CHECK-NEXT: br label [[PRED_UREM_CONTINUE2]]
@@ -4951,12 +4952,12 @@ define i32 @PR32419(i32 %a, i16 %b) {
; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE2:%.*]] ]
; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[PRED_UREM_CONTINUE2]] ]
; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ]
-; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16
+; IND-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], -20
; IND-NEXT: [[TMP2:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; IND-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; IND-NEXT: br i1 [[TMP3]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; IND: pred.urem.if:
-; IND-NEXT: [[TMP4:%.*]] = add i16 [[TMP1]], -20
+; IND-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP1]] to i16
; IND-NEXT: [[TMP5:%.*]] = urem i16 [[B:%.*]], [[TMP4]]
; IND-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; IND-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -4965,7 +4966,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
; IND-NEXT: br i1 [[TMP8]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2]]
; IND: pred.urem.if1:
-; IND-NEXT: [[TMP9:%.*]] = add i16 [[TMP1]], -19
+; IND-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP1]] to i16
+; IND-NEXT: [[TMP9:%.*]] = or disjoint i16 [[TMP16]], 1
; IND-NEXT: [[TMP10:%.*]] = urem i16 [[B]], [[TMP9]]
; IND-NEXT: [[TMP11:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP10]], i64 1
; IND-NEXT: br label [[PRED_UREM_CONTINUE2]]
@@ -4995,13 +4997,13 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[PRED_UREM_CONTINUE7]] ]
; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_UREM_CONTINUE7]] ]
; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE7]] ]
-; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16
+; UNROLL-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], -20
; UNROLL-NEXT: [[TMP2:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; UNROLL-NEXT: [[TMP3:%.*]] = icmp ne <2 x i16> [[VEC_IND]], splat (i16 -2)
; UNROLL-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; UNROLL-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; UNROLL: pred.urem.if:
-; UNROLL-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], -20
+; UNROLL-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP1]] to i16
; UNROLL-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]]
; UNROLL-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -5010,7 +5012,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
; UNROLL-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]]
; UNROLL: pred.urem.if2:
-; UNROLL-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], -19
+; UNROLL-NEXT: [[TMP29:%.*]] = trunc i32 [[TMP1]] to i16
+; UNROLL-NEXT: [[TMP10:%.*]] = or disjoint i16 [[TMP29]], 1
; UNROLL-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]]
; UNROLL-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP11]], i64 1
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE3]]
@@ -5019,7 +5022,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; UNROLL-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]]
; UNROLL: pred.urem.if4:
-; UNROLL-NEXT: [[TMP15:%.*]] = add i16 [[TMP1]], -18
+; UNROLL-NEXT: [[TMP30:%.*]] = trunc i32 [[TMP1]] to i16
+; UNROLL-NEXT: [[TMP15:%.*]] = or disjoint i16 [[TMP30]], 2
; UNROLL-NEXT: [[TMP16:%.*]] = urem i16 [[B]], [[TMP15]]
; UNROLL-NEXT: [[TMP17:%.*]] = insertelement <2 x i16> poison, i16 [[TMP16]], i64 0
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE5]]
@@ -5028,7 +5032,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
; UNROLL-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7]]
; UNROLL: pred.urem.if6:
-; UNROLL-NEXT: [[TMP20:%.*]] = add i16 [[TMP1]], -17
+; UNROLL-NEXT: [[TMP31:%.*]] = trunc i32 [[TMP1]] to i16
+; UNROLL-NEXT: [[TMP20:%.*]] = or disjoint i16 [[TMP31]], 3
; UNROLL-NEXT: [[TMP21:%.*]] = urem i16 [[B]], [[TMP20]]
; UNROLL-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP21]], i64 1
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE7]]
@@ -5064,12 +5069,12 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE7]] ]
; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
-; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = icmp ne <2 x i16> [[STEP_ADD]], zeroinitializer
; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; UNROLL-NO-IC-NEXT: br i1 [[TMP6]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; UNROLL-NO-IC: pred.urem.if:
+; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = urem i16 [[B:%.*]], [[TMP1]]
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -5078,7 +5083,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]]
; UNROLL-NO-IC: pred.urem.if2:
-; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i16 [[TMP1]], 1
+; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i16 [[TMP32]], 1
; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = urem i16 [[B]], [[TMP12]]
; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <2 x i16> [[TMP10]], i16 [[TMP13]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE3]]
@@ -5087,7 +5093,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP5]], i64 0
; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]]
; UNROLL-NO-IC: pred.urem.if4:
-; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add i16 [[TMP1]], 2
+; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add i16 [[TMP33]], 2
; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = urem i16 [[B]], [[TMP17]]
; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP18]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE5]]
@@ -5096,7 +5103,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP5]], i64 1
; UNROLL-NO-IC-NEXT: br i1 [[TMP21]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7]]
; UNROLL-NO-IC: pred.urem.if6:
-; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add i16 [[TMP1]], 3
+; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
+; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add i16 [[TMP34]], 3
; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = urem i16 [[B]], [[TMP22]]
; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = insertelement <2 x i16> [[TMP20]], i16 [[TMP23]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE7]]
@@ -5130,13 +5138,13 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_UREM_CONTINUE15]] ]
; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_UREM_CONTINUE15]] ]
; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -20, i16 -19, i16 -18, i16 -17>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE15]] ]
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16
+; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], -20
; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp ne <4 x i16> [[VEC_IND]], zeroinitializer
; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ne <4 x i16> [[VEC_IND]], splat (i16 -4)
; INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; INTERLEAVE-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; INTERLEAVE: pred.urem.if:
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], -20
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP1]] to i16
; INTERLEAVE-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]]
; INTERLEAVE-NEXT: [[TMP7:%.*]] = insertelement <4 x i16> poison, i16 [[TMP6]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -5145,7 +5153,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]]
; INTERLEAVE: pred.urem.if2:
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], -19
+; INTERLEAVE-NEXT: [[TMP49:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = or disjoint i16 [[TMP49]], 1
; INTERLEAVE-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]]
; INTERLEAVE-NEXT: [[TMP12:%.*]] = insertelement <4 x i16> [[TMP8]], i16 [[TMP11]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE3]]
@@ -5154,7 +5163,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]]
; INTERLEAVE: pred.urem.if4:
-; INTERLEAVE-NEXT: [[TMP15:%.*]] = add i16 [[TMP1]], -18
+; INTERLEAVE-NEXT: [[TMP52:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP15:%.*]] = or disjoint i16 [[TMP52]], 2
; INTERLEAVE-NEXT: [[TMP16:%.*]] = urem i16 [[B]], [[TMP15]]
; INTERLEAVE-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP13]], i16 [[TMP16]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE5]]
@@ -5163,7 +5173,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
; INTERLEAVE-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7:%.*]]
; INTERLEAVE: pred.urem.if6:
-; INTERLEAVE-NEXT: [[TMP20:%.*]] = add i16 [[TMP1]], -17
+; INTERLEAVE-NEXT: [[TMP53:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP20:%.*]] = or disjoint i16 [[TMP53]], 3
; INTERLEAVE-NEXT: [[TMP21:%.*]] = urem i16 [[B]], [[TMP20]]
; INTERLEAVE-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP21]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE7]]
@@ -5172,7 +5183,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
; INTERLEAVE-NEXT: br i1 [[TMP24]], label [[PRED_UREM_IF8:%.*]], label [[PRED_UREM_CONTINUE9:%.*]]
; INTERLEAVE: pred.urem.if8:
-; INTERLEAVE-NEXT: [[TMP25:%.*]] = add i16 [[TMP1]], -16
+; INTERLEAVE-NEXT: [[TMP54:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP25:%.*]] = add i16 [[TMP54]], 4
; INTERLEAVE-NEXT: [[TMP26:%.*]] = urem i16 [[B]], [[TMP25]]
; INTERLEAVE-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> poison, i16 [[TMP26]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE9]]
@@ -5181,7 +5193,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
; INTERLEAVE-NEXT: br i1 [[TMP29]], label [[PRED_UREM_IF10:%.*]], label [[PRED_UREM_CONTINUE11:%.*]]
; INTERLEAVE: pred.urem.if10:
-; INTERLEAVE-NEXT: [[TMP30:%.*]] = add i16 [[TMP1]], -15
+; INTERLEAVE-NEXT: [[TMP55:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP30:%.*]] = add i16 [[TMP55]], 5
; INTERLEAVE-NEXT: [[TMP31:%.*]] = urem i16 [[B]], [[TMP30]]
; INTERLEAVE-NEXT: [[TMP32:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP31]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE11]]
@@ -5190,7 +5203,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
; INTERLEAVE-NEXT: br i1 [[TMP34]], label [[PRED_UREM_IF12:%.*]], label [[PRED_UREM_CONTINUE13:%.*]]
; INTERLEAVE: pred.urem.if12:
-; INTERLEAVE-NEXT: [[TMP35:%.*]] = add i16 [[TMP1]], -14
+; INTERLEAVE-NEXT: [[TMP56:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP35:%.*]] = add i16 [[TMP56]], 6
; INTERLEAVE-NEXT: [[TMP36:%.*]] = urem i16 [[B]], [[TMP35]]
; INTERLEAVE-NEXT: [[TMP37:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP36]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE13]]
@@ -5199,7 +5213,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
; INTERLEAVE-NEXT: br i1 [[TMP39]], label [[PRED_UREM_IF14:%.*]], label [[PRED_UREM_CONTINUE15]]
; INTERLEAVE: pred.urem.if14:
-; INTERLEAVE-NEXT: [[TMP40:%.*]] = add i16 [[TMP1]], -13
+; INTERLEAVE-NEXT: [[TMP57:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP40:%.*]] = add i16 [[TMP57]], 7
; INTERLEAVE-NEXT: [[TMP41:%.*]] = urem i16 [[B]], [[TMP40]]
; INTERLEAVE-NEXT: [[TMP42:%.*]] = insertelement <4 x i16> [[TMP38]], i16 [[TMP41]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE15]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll b/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
index dd060ab5de799..12380f0be8c22 100644
--- a/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
@@ -202,11 +202,11 @@ define void @single_incoming_needs_predication(i64 %a, i64 %b) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE2]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i16
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i16
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[TMP5]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0
@@ -216,7 +216,8 @@ define void @single_incoming_needs_predication(i64 %a, i64 %b) {
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = trunc i64 [[INDEX]] to i16
+; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP17]], 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP11]], align 1
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP12]], i64 1
>From 2d85bb1d748bfde2265996ee86d1bc7292f8cb47 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sun, 19 Jul 2026 08:59:34 +0100
Subject: [PATCH 2/2] [VPlan] Draft of fixes
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 8 ++-
.../X86/predicated-replicate-feeding-cast.ll | 15 ++---
llvm/test/Transforms/LoopVectorize/as_cast.ll | 5 +-
.../Transforms/LoopVectorize/induction.ll | 61 +++++++------------
.../LoopVectorize/single-value-blend-phis.ll | 5 +-
5 files changed, 36 insertions(+), 58 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 69c786eef9bfa..de2a9764e9921 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -307,16 +307,18 @@ static bool sinkScalarOperands(VPlan &Plan) {
auto Iter = vp_depth_first_deep(Plan.getEntry());
bool ScalarVFOnly = Plan.hasScalarVFOnly();
bool Changed = false;
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
- auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
- VPBasicBlock *SinkTo, VPValue *Op) {
+ auto InsertIfValidSinkCandidate = [&](VPBasicBlock *SinkTo, VPValue *Op) {
auto *Candidate = dyn_cast<VPSingleDefRecipe>(Op);
if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
VPInstruction>(Candidate))
return;
if (Candidate->getParent() == SinkTo ||
+ all_of(Candidate->operands(),
+ [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
vputils::cannotHoistOrSinkRecipe(*Candidate, /*Sinking=*/true))
return;
@@ -326,7 +328,7 @@ static bool sinkScalarOperands(VPlan &Plan) {
// Due to a limitation in VPlanUnroll, we currently know only how to sink
// single-scalar VPInstructions.
if (auto *VPI = dyn_cast<VPInstruction>(Candidate))
- if (!vputils::isSingleScalar(VPI))
+ if (!HeaderMask || !vputils::isSingleScalar(VPI))
return;
WorkList.insert({SinkTo, Candidate});
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
index 2ab0f685308f2..63bfa10ff2934 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-replicate-feeding-cast.ll
@@ -23,22 +23,17 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[TMP5:%.*]] = sdiv i16 1, [[A]]
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_SDIV_IF]] ]
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: [[TMP8:%.*]] = sdiv i16 1, [[A]]
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP8]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x i16> [ [[TMP7]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C1]], <2 x i16> zeroinitializer, <2 x i16> [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[C1]], i16 0, i16 [[TMP6]]
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP11]] to i8
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
; CHECK: [[PRED_SDIV_IF3]]:
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = trunc i16 [[TMP11]] to i8
; CHECK-NEXT: [[TMP13:%.*]] = sdiv i8 [[TMP12]], [[B]]
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i8> poison, i8 [[TMP13]], i64 0
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
@@ -46,9 +41,7 @@ define i8 @predicated_replicate_feeding_cast(i16 %n, i1 %c1, i1 %c2, i16 %a, i8
; CHECK-NEXT: [[TMP15:%.*]] = phi <2 x i8> [ poison, %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP14]], %[[PRED_SDIV_IF3]] ]
; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_IF5]]:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i16> [[PREDPHI]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i16 [[TMP16]] to i8
-; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP17]], [[B]]
+; CHECK-NEXT: [[TMP18:%.*]] = sdiv i8 [[TMP12]], [[B]]
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> [[TMP15]], i8 [[TMP18]], i64 1
; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
; CHECK: [[PRED_SDIV_CONTINUE6]]:
diff --git a/llvm/test/Transforms/LoopVectorize/as_cast.ll b/llvm/test/Transforms/LoopVectorize/as_cast.ll
index d2b281d133fb9..81b09121f4077 100644
--- a/llvm/test/Transforms/LoopVectorize/as_cast.ll
+++ b/llvm/test/Transforms/LoopVectorize/as_cast.ll
@@ -7,6 +7,7 @@ define void @loop_invariant_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[IN]] to ptr
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
@@ -16,7 +17,6 @@ define void @loop_invariant_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr addrspace(1) [[IN]] to ptr
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP3]]
; CHECK-NEXT: store i64 [[TMP3]], ptr [[TMP5]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
@@ -24,8 +24,7 @@ define void @loop_invariant_as_cast(ptr addrspace(1) %in) {
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = addrspacecast ptr addrspace(1) [[IN]] to ptr
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP6]]
; CHECK-NEXT: store i64 [[TMP6]], ptr [[TMP8]], align 4
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; CHECK: [[PRED_STORE_CONTINUE2]]:
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 6a0ab3015c61e..f2ede41effb54 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -4909,11 +4909,11 @@ define i32 @PR32419(i32 %a, i16 %b) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_UREM_CONTINUE2]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; CHECK: pred.urem.if:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; CHECK-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP1]]
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0
; CHECK-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -4922,8 +4922,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2]]
; CHECK: pred.urem.if1:
-; CHECK-NEXT: [[TMP18:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP18]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], 1
; CHECK-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP11]], i64 1
; CHECK-NEXT: br label [[PRED_UREM_CONTINUE2]]
@@ -4952,12 +4951,12 @@ define i32 @PR32419(i32 %a, i16 %b) {
; IND-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UREM_CONTINUE2:%.*]] ]
; IND-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[PRED_UREM_CONTINUE2]] ]
; IND-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ]
-; IND-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], -20
+; IND-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16
; IND-NEXT: [[TMP2:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; IND-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; IND-NEXT: br i1 [[TMP3]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; IND: pred.urem.if:
-; IND-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP1]] to i16
+; IND-NEXT: [[TMP4:%.*]] = add i16 [[TMP1]], -20
; IND-NEXT: [[TMP5:%.*]] = urem i16 [[B:%.*]], [[TMP4]]
; IND-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i64 0
; IND-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -4966,8 +4965,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; IND-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
; IND-NEXT: br i1 [[TMP8]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2]]
; IND: pred.urem.if1:
-; IND-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP1]] to i16
-; IND-NEXT: [[TMP9:%.*]] = or disjoint i16 [[TMP16]], 1
+; IND-NEXT: [[TMP9:%.*]] = add i16 [[TMP1]], -19
; IND-NEXT: [[TMP10:%.*]] = urem i16 [[B]], [[TMP9]]
; IND-NEXT: [[TMP11:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP10]], i64 1
; IND-NEXT: br label [[PRED_UREM_CONTINUE2]]
@@ -4997,13 +4995,13 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[PRED_UREM_CONTINUE7]] ]
; UNROLL-NEXT: [[VEC_PHI1:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_UREM_CONTINUE7]] ]
; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE7]] ]
-; UNROLL-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], -20
+; UNROLL-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16
; UNROLL-NEXT: [[TMP2:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; UNROLL-NEXT: [[TMP3:%.*]] = icmp ne <2 x i16> [[VEC_IND]], splat (i16 -2)
; UNROLL-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; UNROLL-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; UNROLL: pred.urem.if:
-; UNROLL-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP1]] to i16
+; UNROLL-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], -20
; UNROLL-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]]
; UNROLL-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -5012,8 +5010,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
; UNROLL-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]]
; UNROLL: pred.urem.if2:
-; UNROLL-NEXT: [[TMP29:%.*]] = trunc i32 [[TMP1]] to i16
-; UNROLL-NEXT: [[TMP10:%.*]] = or disjoint i16 [[TMP29]], 1
+; UNROLL-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], -19
; UNROLL-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]]
; UNROLL-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP11]], i64 1
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE3]]
@@ -5022,8 +5019,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
; UNROLL-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]]
; UNROLL: pred.urem.if4:
-; UNROLL-NEXT: [[TMP30:%.*]] = trunc i32 [[TMP1]] to i16
-; UNROLL-NEXT: [[TMP15:%.*]] = or disjoint i16 [[TMP30]], 2
+; UNROLL-NEXT: [[TMP15:%.*]] = add i16 [[TMP1]], -18
; UNROLL-NEXT: [[TMP16:%.*]] = urem i16 [[B]], [[TMP15]]
; UNROLL-NEXT: [[TMP17:%.*]] = insertelement <2 x i16> poison, i16 [[TMP16]], i64 0
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE5]]
@@ -5032,8 +5028,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
; UNROLL-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7]]
; UNROLL: pred.urem.if6:
-; UNROLL-NEXT: [[TMP31:%.*]] = trunc i32 [[TMP1]] to i16
-; UNROLL-NEXT: [[TMP20:%.*]] = or disjoint i16 [[TMP31]], 3
+; UNROLL-NEXT: [[TMP20:%.*]] = add i16 [[TMP1]], -17
; UNROLL-NEXT: [[TMP21:%.*]] = urem i16 [[B]], [[TMP20]]
; UNROLL-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP21]], i64 1
; UNROLL-NEXT: br label [[PRED_UREM_CONTINUE7]]
@@ -5069,12 +5064,12 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE7]] ]
; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
+; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = icmp ne <2 x i16> [[STEP_ADD]], zeroinitializer
; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
; UNROLL-NO-IC-NEXT: br i1 [[TMP6]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; UNROLL-NO-IC: pred.urem.if:
-; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = urem i16 [[B:%.*]], [[TMP1]]
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -5083,8 +5078,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
; UNROLL-NO-IC-NEXT: br i1 [[TMP11]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]]
; UNROLL-NO-IC: pred.urem.if2:
-; UNROLL-NO-IC-NEXT: [[TMP32:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i16 [[TMP32]], 1
+; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = add i16 [[TMP1]], 1
; UNROLL-NO-IC-NEXT: [[TMP13:%.*]] = urem i16 [[B]], [[TMP12]]
; UNROLL-NO-IC-NEXT: [[TMP14:%.*]] = insertelement <2 x i16> [[TMP10]], i16 [[TMP13]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE3]]
@@ -5093,8 +5087,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP5]], i64 0
; UNROLL-NO-IC-NEXT: br i1 [[TMP16]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]]
; UNROLL-NO-IC: pred.urem.if4:
-; UNROLL-NO-IC-NEXT: [[TMP33:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add i16 [[TMP33]], 2
+; UNROLL-NO-IC-NEXT: [[TMP17:%.*]] = add i16 [[TMP1]], 2
; UNROLL-NO-IC-NEXT: [[TMP18:%.*]] = urem i16 [[B]], [[TMP17]]
; UNROLL-NO-IC-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP18]], i64 0
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE5]]
@@ -5103,8 +5096,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; UNROLL-NO-IC-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP5]], i64 1
; UNROLL-NO-IC-NEXT: br i1 [[TMP21]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7]]
; UNROLL-NO-IC: pred.urem.if6:
-; UNROLL-NO-IC-NEXT: [[TMP34:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add i16 [[TMP34]], 3
+; UNROLL-NO-IC-NEXT: [[TMP22:%.*]] = add i16 [[TMP1]], 3
; UNROLL-NO-IC-NEXT: [[TMP23:%.*]] = urem i16 [[B]], [[TMP22]]
; UNROLL-NO-IC-NEXT: [[TMP24:%.*]] = insertelement <2 x i16> [[TMP20]], i16 [[TMP23]], i64 1
; UNROLL-NO-IC-NEXT: br label [[PRED_UREM_CONTINUE7]]
@@ -5138,13 +5130,13 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_UREM_CONTINUE15]] ]
; INTERLEAVE-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[PRED_UREM_CONTINUE15]] ]
; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 -20, i16 -19, i16 -18, i16 -17>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE15]] ]
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], -20
+; INTERLEAVE-NEXT: [[TMP1:%.*]] = trunc i32 [[INDEX]] to i16
; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp ne <4 x i16> [[VEC_IND]], zeroinitializer
; INTERLEAVE-NEXT: [[TMP3:%.*]] = icmp ne <4 x i16> [[VEC_IND]], splat (i16 -4)
; INTERLEAVE-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; INTERLEAVE-NEXT: br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
; INTERLEAVE: pred.urem.if:
-; INTERLEAVE-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP1]] to i16
+; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i16 [[TMP1]], -20
; INTERLEAVE-NEXT: [[TMP6:%.*]] = urem i16 [[B:%.*]], [[TMP5]]
; INTERLEAVE-NEXT: [[TMP7:%.*]] = insertelement <4 x i16> poison, i16 [[TMP6]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE]]
@@ -5153,8 +5145,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
; INTERLEAVE-NEXT: br i1 [[TMP9]], label [[PRED_UREM_IF2:%.*]], label [[PRED_UREM_CONTINUE3:%.*]]
; INTERLEAVE: pred.urem.if2:
-; INTERLEAVE-NEXT: [[TMP49:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP10:%.*]] = or disjoint i16 [[TMP49]], 1
+; INTERLEAVE-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], -19
; INTERLEAVE-NEXT: [[TMP11:%.*]] = urem i16 [[B]], [[TMP10]]
; INTERLEAVE-NEXT: [[TMP12:%.*]] = insertelement <4 x i16> [[TMP8]], i16 [[TMP11]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE3]]
@@ -5163,8 +5154,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
; INTERLEAVE-NEXT: br i1 [[TMP14]], label [[PRED_UREM_IF4:%.*]], label [[PRED_UREM_CONTINUE5:%.*]]
; INTERLEAVE: pred.urem.if4:
-; INTERLEAVE-NEXT: [[TMP52:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP15:%.*]] = or disjoint i16 [[TMP52]], 2
+; INTERLEAVE-NEXT: [[TMP15:%.*]] = add i16 [[TMP1]], -18
; INTERLEAVE-NEXT: [[TMP16:%.*]] = urem i16 [[B]], [[TMP15]]
; INTERLEAVE-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP13]], i16 [[TMP16]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE5]]
@@ -5173,8 +5163,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
; INTERLEAVE-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF6:%.*]], label [[PRED_UREM_CONTINUE7:%.*]]
; INTERLEAVE: pred.urem.if6:
-; INTERLEAVE-NEXT: [[TMP53:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP20:%.*]] = or disjoint i16 [[TMP53]], 3
+; INTERLEAVE-NEXT: [[TMP20:%.*]] = add i16 [[TMP1]], -17
; INTERLEAVE-NEXT: [[TMP21:%.*]] = urem i16 [[B]], [[TMP20]]
; INTERLEAVE-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[TMP21]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE7]]
@@ -5183,8 +5172,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
; INTERLEAVE-NEXT: br i1 [[TMP24]], label [[PRED_UREM_IF8:%.*]], label [[PRED_UREM_CONTINUE9:%.*]]
; INTERLEAVE: pred.urem.if8:
-; INTERLEAVE-NEXT: [[TMP54:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP25:%.*]] = add i16 [[TMP54]], 4
+; INTERLEAVE-NEXT: [[TMP25:%.*]] = add i16 [[TMP1]], -16
; INTERLEAVE-NEXT: [[TMP26:%.*]] = urem i16 [[B]], [[TMP25]]
; INTERLEAVE-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> poison, i16 [[TMP26]], i64 0
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE9]]
@@ -5193,8 +5181,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1
; INTERLEAVE-NEXT: br i1 [[TMP29]], label [[PRED_UREM_IF10:%.*]], label [[PRED_UREM_CONTINUE11:%.*]]
; INTERLEAVE: pred.urem.if10:
-; INTERLEAVE-NEXT: [[TMP55:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP30:%.*]] = add i16 [[TMP55]], 5
+; INTERLEAVE-NEXT: [[TMP30:%.*]] = add i16 [[TMP1]], -15
; INTERLEAVE-NEXT: [[TMP31:%.*]] = urem i16 [[B]], [[TMP30]]
; INTERLEAVE-NEXT: [[TMP32:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP31]], i64 1
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE11]]
@@ -5203,8 +5190,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2
; INTERLEAVE-NEXT: br i1 [[TMP34]], label [[PRED_UREM_IF12:%.*]], label [[PRED_UREM_CONTINUE13:%.*]]
; INTERLEAVE: pred.urem.if12:
-; INTERLEAVE-NEXT: [[TMP56:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP35:%.*]] = add i16 [[TMP56]], 6
+; INTERLEAVE-NEXT: [[TMP35:%.*]] = add i16 [[TMP1]], -14
; INTERLEAVE-NEXT: [[TMP36:%.*]] = urem i16 [[B]], [[TMP35]]
; INTERLEAVE-NEXT: [[TMP37:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP36]], i64 2
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE13]]
@@ -5213,8 +5199,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
; INTERLEAVE-NEXT: [[TMP39:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3
; INTERLEAVE-NEXT: br i1 [[TMP39]], label [[PRED_UREM_IF14:%.*]], label [[PRED_UREM_CONTINUE15]]
; INTERLEAVE: pred.urem.if14:
-; INTERLEAVE-NEXT: [[TMP57:%.*]] = trunc i32 [[TMP1]] to i16
-; INTERLEAVE-NEXT: [[TMP40:%.*]] = add i16 [[TMP57]], 7
+; INTERLEAVE-NEXT: [[TMP40:%.*]] = add i16 [[TMP1]], -13
; INTERLEAVE-NEXT: [[TMP41:%.*]] = urem i16 [[B]], [[TMP40]]
; INTERLEAVE-NEXT: [[TMP42:%.*]] = insertelement <4 x i16> [[TMP38]], i16 [[TMP41]], i64 3
; INTERLEAVE-NEXT: br label [[PRED_UREM_CONTINUE15]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll b/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
index 12380f0be8c22..dd060ab5de799 100644
--- a/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
@@ -202,11 +202,11 @@ define void @single_incoming_needs_predication(i64 %a, i64 %b) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE2]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i16
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i16
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[TMP5]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i64 0
@@ -216,8 +216,7 @@ define void @single_incoming_needs_predication(i64 %a, i64 %b) {
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
; CHECK: pred.load.if1:
-; CHECK-NEXT: [[TMP17:%.*]] = trunc i64 [[INDEX]] to i16
-; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP17]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP11]], align 1
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP12]], i64 1
More information about the llvm-commits
mailing list