[llvm-branch-commits] [llvm] Backport vectorised-alloca bugfix to LLVM23 (PR #236602)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 28 16:53:07 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Jeremy Morse (jmorse)
<details>
<summary>Changes</summary>
See #<!-- -->230450 , this commit contains three commits:
* Florians pre-landing of a test for a vectorized-alloca issue,
* Florians fix for it,
* A test-update commit that refreshes two tests which have different behaviour as a result of this change.
The two commits from Florian are clean cherry-picks from the main branch, which IMHO don't warrant too much further review. However I'm completely unfamiliar with the test-changes and can't really vouch for them -- they definitely need review.
It'd be nice to have this fixed in llvm23 given how it's a (rare) miscompile.
---
Patch is 74.24 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/236602.diff
5 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+3-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+3-2)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll (+288-282)
- (added) llvm/test/Transforms/LoopVectorize/alloca-in-loop.ll (+224)
- (modified) llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll (+6-3)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 48ba81dacefc7b..e94c80e98b273c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2537,9 +2537,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
// The issue with (Insert|Extract)Value is that the index of the
// insert/extract is not a proper operand in LLVM IR, and hence also not in
- // VPlan.
+ // VPlan. Allocas must not be merged, as each creates a distinct allocation.
if (!C || (!C->first && (C->second == Instruction::InsertValue ||
- C->second == Instruction::ExtractValue)))
+ C->second == Instruction::ExtractValue ||
+ C->second == Instruction::Alloca)))
return false;
// During CSE, we can only handle non-memory recipes, as memory can alias.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 7b4ffce87434c2..05bc010f6784cc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -454,8 +454,9 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
.Case([](const VPReplicateRecipe *R) {
// Be conservative about side-effects, except for the
// known-side-effecting assumes and stores, which we know will be
- // uniform.
- return R->isSingleScalar() &&
+ // uniform. Each alloca creates a distinct allocation, so allocas are
+ // never uniform.
+ return R->isSingleScalar() && R->getOpcode() != Instruction::Alloca &&
(!R->mayHaveSideEffects() ||
isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
all_of(R->operands(), isUniformAcrossVFsAndUFs);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 07fc18e46c4fe4..3c8d118f539752 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -57,16 +57,16 @@ define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) {
; COST1: [[VECTOR_PH]]:
; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 32
; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]]
-; COST1-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]]
+; COST1-NEXT: [[TMP6:%.*]] = sub i64 [[START]], [[N_VEC]]
; COST1-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
-; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX]]
-; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 16
-; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1
+; COST1-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST1-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP5]]
+; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 16
+; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP2]], align 1
; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP0]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 32
; COST1-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST1-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
@@ -110,7 +110,7 @@ define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) {
; COST10: [[VECTOR_PH]]:
; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 16
; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]]
-; COST10-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]]
+; COST10-NEXT: [[TMP4:%.*]] = sub i64 [[START]], [[N_VEC]]
; COST10-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
@@ -185,73 +185,73 @@ define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y.
; COMMON-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT2]], <2 x i1> poison, <2 x i32> zeroinitializer
; COMMON-NEXT: [[TMP0:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer
; COMMON-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP0]], splat (i1 true)
-; COMMON-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT3]], splat (i1 true)
+; COMMON-NEXT: [[TMP2:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT3]], splat (i1 true)
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
-; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE17:.*]] ]
+; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE33:.*]] ]
; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[X_PTR]], i64 [[INDEX]]
; COMMON-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
-; COMMON-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer
-; COMMON-NEXT: [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true)
-; COMMON-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i64 0
-; COMMON-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; COMMON: [[PRED_STORE_IF]]:
-; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8
-; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; COMMON: [[PRED_STORE_CONTINUE]]:
-; COMMON-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP7]], i64 1
-; COMMON-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
-; COMMON: [[PRED_STORE_IF4]]:
-; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8
-; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE5]]
-; COMMON: [[PRED_STORE_CONTINUE5]]:
-; COMMON-NEXT: [[TMP10:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer
-; COMMON-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP10]], i64 0
+; COMMON-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer
+; COMMON-NEXT: [[TMP6:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; COMMON-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP6]], i64 0
; COMMON-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]
; COMMON: [[PRED_STORE_IF6]]:
-; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8
+; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8
; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE7]]
; COMMON: [[PRED_STORE_CONTINUE7]]:
-; COMMON-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP10]], i64 1
-; COMMON-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]
-; COMMON: [[PRED_STORE_IF8]]:
-; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8
-; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE9]]
-; COMMON: [[PRED_STORE_CONTINUE9]]:
-; COMMON-NEXT: [[TMP22:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer
-; COMMON-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP22]], <2 x i1> [[BROADCAST_SPLAT3]], <2 x i1> zeroinitializer
-; COMMON-NEXT: [[TMP14:%.*]] = or <2 x i1> [[TMP6]], [[TMP13]]
-; COMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP13]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1)
-; COMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP14]], i64 0
+; COMMON-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP6]], i64 1
+; COMMON-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; COMMON: [[PRED_STORE_IF7]]:
+; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8
+; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; COMMON: [[PRED_STORE_CONTINUE8]]:
+; COMMON-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer
+; COMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP8]], i64 0
; COMMON-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]
; COMMON: [[PRED_STORE_IF10]]:
-; COMMON-NEXT: [[TMP16:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 0
-; COMMON-NEXT: store i64 [[TMP16]], ptr [[DST_2]], align 8
+; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8
; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE11]]
; COMMON: [[PRED_STORE_CONTINUE11]]:
-; COMMON-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP14]], i64 1
-; COMMON-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
-; COMMON: [[PRED_STORE_IF12]]:
-; COMMON-NEXT: [[TMP18:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 1
-; COMMON-NEXT: store i64 [[TMP18]], ptr [[DST_2]], align 8
-; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE13]]
-; COMMON: [[PRED_STORE_CONTINUE13]]:
-; COMMON-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP22]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer
-; COMMON-NEXT: [[TMP20:%.*]] = or <2 x i1> [[TMP14]], [[TMP19]]
-; COMMON-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP20]], i64 0
+; COMMON-NEXT: [[TMP27:%.*]] = extractelement <2 x i1> [[TMP8]], i64 1
+; COMMON-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
+; COMMON: [[PRED_STORE_IF15]]:
+; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8
+; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE16]]
+; COMMON: [[PRED_STORE_CONTINUE16]]:
+; COMMON-NEXT: [[TMP14:%.*]] = select <2 x i1> [[TMP6]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer
+; COMMON-NEXT: [[TMP12:%.*]] = select <2 x i1> [[TMP14]], <2 x i1> [[BROADCAST_SPLAT3]], <2 x i1> zeroinitializer
+; COMMON-NEXT: [[TMP13:%.*]] = or <2 x i1> [[TMP4]], [[TMP12]]
+; COMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP12]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1)
+; COMMON-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP13]], i64 0
; COMMON-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]]
; COMMON: [[PRED_STORE_IF14]]:
-; COMMON-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6:![0-9]+]]
-; COMMON-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
+; COMMON-NEXT: [[TMP28:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 0
+; COMMON-NEXT: store i64 [[TMP28]], ptr [[DST_2]], align 8
; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE15]]
; COMMON: [[PRED_STORE_CONTINUE15]]:
-; COMMON-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP20]], i64 1
-; COMMON-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17]]
+; COMMON-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP13]], i64 1
+; COMMON-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17:.*]]
; COMMON: [[PRED_STORE_IF16]]:
-; COMMON-NEXT: [[TMP26:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6]]
-; COMMON-NEXT: store i64 [[TMP26]], ptr [[DST]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; COMMON-NEXT: [[TMP30:%.*]] = extractelement <2 x i64> [[PREDPHI]], i64 1
+; COMMON-NEXT: store i64 [[TMP30]], ptr [[DST_2]], align 8
; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE17]]
; COMMON: [[PRED_STORE_CONTINUE17]]:
+; COMMON-NEXT: [[TMP18:%.*]] = select <2 x i1> [[TMP14]], <2 x i1> [[TMP2]], <2 x i1> zeroinitializer
+; COMMON-NEXT: [[TMP19:%.*]] = or <2 x i1> [[TMP13]], [[TMP18]]
+; COMMON-NEXT: [[TMP40:%.*]] = extractelement <2 x i1> [[TMP19]], i64 0
+; COMMON-NEXT: br i1 [[TMP40]], label %[[PRED_STORE_IF30:.*]], label %[[PRED_STORE_CONTINUE31:.*]]
+; COMMON: [[PRED_STORE_IF30]]:
+; COMMON-NEXT: [[TMP22:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6:![0-9]+]]
+; COMMON-NEXT: store i64 [[TMP22]], ptr [[DST]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
+; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE31]]
+; COMMON: [[PRED_STORE_CONTINUE31]]:
+; COMMON-NEXT: [[TMP43:%.*]] = extractelement <2 x i1> [[TMP19]], i64 1
+; COMMON-NEXT: br i1 [[TMP43]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33]]
+; COMMON: [[PRED_STORE_IF32]]:
+; COMMON-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META6]]
+; COMMON-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META9]], !noalias [[META6]]
+; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE33]]
+; COMMON: [[PRED_STORE_CONTINUE33]]:
; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; COMMON-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; COMMON-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
@@ -323,15 +323,20 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {
; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
-; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COST1-NEXT: [[TMP8:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST1-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16
-; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0
-; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
-; COST1-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]]
-; COST1-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[TMP2]], i64 2
-; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8
-; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COST1-NEXT: [[TMP10:%.*]] = alloca i8, i64 0, align 16
+; COST1-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i32 0
+; COST1-NEXT: [[TMP11:%.*]] = insertelement <2 x ptr> [[TMP5]], ptr [[TMP10]], i32 1
+; COST1-NEXT: [[TMP6:%.*]] = alloca i8, i64 0, align 16
+; COST1-NEXT: [[TMP7:%.*]] = alloca i8, i64 0, align 16
+; COST1-NEXT: [[TMP14:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP6]], i32 0
+; COST1-NEXT: [[TMP9:%.*]] = insertelement <2 x ptr> [[TMP14]], ptr [[TMP7]], i32 1
+; COST1-NEXT: [[TMP12:%.*]] = getelementptr ptr, ptr [[X]], i64 [[TMP8]]
+; COST1-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[TMP12]], i64 2
+; COST1-NEXT: store <2 x ptr> [[TMP11]], ptr [[TMP12]], align 8
+; COST1-NEXT: store <2 x ptr> [[TMP9]], ptr [[TMP3]], align 8
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP8]], 4
; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
@@ -352,10 +357,11 @@ define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16
-; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0
-; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; COST10-NEXT: [[TMP6:%.*]] = alloca i8, i64 0, align 16
+; COST10-NEXT: [[TMP4:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i32 0
+; COST10-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> [[TMP4]], ptr [[TMP6]], i32 1
; COST10-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]]
-; COST10-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8
+; COST10-NEXT: store <2 x ptr> [[TMP5]], ptr [[TMP2]], align 8
; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; COST10-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -383,26 +389,26 @@ for.end:
define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) {
; COMMON-LABEL: define void @loop_with_freeze_and_conditional_srem(
; COMMON-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) {
-; COMMON-NEXT: [[ENTRY:.*]]:
-; COMMON-NEXT: br label %[[LOOP:.*]]
-; COMMON: [[LOOP]]:
-; COMMON-NEXT: [[INDEX_NEXT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COMMON-NEXT: [[PRED_STORE_IF10:.*]]:
+; COMMON-NEXT: br label %[[LOOP_LATCH:.*]]
+; COMMON: [[LOOP_LATCH]]:
+; COMMON-NEXT: [[INDEX_NEXT:%.*]] = phi i64 [ 0, %[[PRED_STORE_IF10]] ], [ [[IV_NEXT:%.*]], %[[PRED_STORE_CONTINUE23:.*]] ]
; COMMON-NEXT: [[LOADED:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4
; COMMON-NEXT: [[FROZEN:%.*]] = freeze i32 [[LOADED]]
-; COMMON-NEXT: [[CMP:%.*]] = icmp eq i32 [[FROZEN]], 0
-; COMMON-NEXT: br i1 [[CMP]], label %[[IF_ZERO:.*]], label %[[IF_NONZERO:.*]]
-; COMMON: [[IF_ZERO]]:
+; COMMON-NEXT: [[TMP10:%.*]] = icmp eq i32 [[FROZEN]], 0
+; COMMON-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE13:.*]]
+; COMMON: [[EXIT]]:
; COMMON-NEXT: store i32 0, ptr [[KEYINFO]], align 4
-; COMMON-NEXT: br label %[[LOOP_LATCH]]
-; COMMON: [[IF_NONZERO]]:
-; COMMON-NEXT: [[TMP11:%.*]] = srem i32 1, [[DIVISOR]]
-; COMMON-NEXT: store i32 [[TMP11]], ptr [[DST]], align 4
-; COMMON-NEXT: br label %[[LOOP_LATCH]]
-; COMMON: [[LOOP_LATCH]]:
+; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE23]]
+; COMMON: [[PRED_STORE_CONTINUE13]]:
+; COMMON-NEXT: [[TMP14:%.*]] = srem i32 1, [[DIVISOR]]
+; COMMON-NEXT: store i32 [[TMP14]], ptr [[DST]], align 4
+; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE23]]
+; COMMON: [[PRED_STORE_CONTINUE23]]:
; COMMON-NEXT: [[IV_NEXT]] = add i64 [[INDEX_NEXT]], 1
-; COMMON-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
-; COMMON-NEXT: br i1 [[TMP16]], label %[[EXIT:.*]], label %[[LOOP]]
-; COMMON: [[EXIT]]:
+; COMMON-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32
+; COMMON-NEXT: br i1 [[TMP20]], label %[[SCALAR_PH:.*]], label %[[LOOP_LATCH]]
+; COMMON: [[SCALAR_PH]]:
; COMMON-NEXT: ret void
;
entry:
@@ -459,42 +465,42 @@ define void @interleave_group(ptr %dst) #1 {
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
+; COST1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[VEC_EPILOG_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
; COST1: [[VEC_EPILOG_VECTOR_BODY]]:
-; COST1-NEXT: [[INDEX1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/236602
More information about the llvm-branch-commits
mailing list