[llvm] [VPlan] Extend cse to eliminate redundant widened loads (PR #212543)
Ashutosh Nema via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 03:25:30 PDT 2026
https://github.com/nema-ashutosh updated https://github.com/llvm/llvm-project/pull/212543
>From cc9c85831a8a8a27e6a8a6a54faf2ee69458655b Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Wed, 5 Aug 2026 22:35:25 +0530
Subject: [PATCH 1/9] [VPlan] Precommit tests for widened-load CSE
---
.../AArch64/partial-reduce-fdot-product.ll | 6 +-
.../partial-reduce-sub-epilogue-vec.ll | 24 +-
...e-to-widen-memory-with-wide-ops-chained.ll | 6 +-
...nterleave-to-widen-memory-with-wide-ops.ll | 6 +-
...sform-narrow-interleave-to-widen-memory.ll | 6 +-
.../LoopVectorize/X86/cse-loads-gather.ll | 101 ++++++
.../LoopVectorize/X86/induction-costs.ll | 8 +-
...interleave-to-widen-memory-epilogue-vec.ll | 20 +-
.../LoopVectorize/cse-loads-extra.ll | 286 ++++++++++++++++
.../LoopVectorize/cse-loads-masked.ll | 186 ++++++++++
.../LoopVectorize/cse-loads-tailfold.ll | 49 +++
.../Transforms/LoopVectorize/cse-loads.ll | 324 ++++++++++++++++++
llvm/test/Transforms/LoopVectorize/cse.ll | 165 +++++++++
13 files changed, 1149 insertions(+), 38 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
index 65124a1d97bca..81d59df9918f4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
@@ -1022,9 +1022,9 @@ define float @fadd_fsub_chain_f16_f32(ptr %a, ptr %b, ptr %c) #0 {
; CHECK-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float>
; CHECK-NEXT: [[TMP14:%.*]] = fmul <vscale x 8 x float> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[PARTIAL_REDUCE5:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI1]], <vscale x 8 x float> [[TMP14]])
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 8 x half>, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[TMP17:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD7]] to <vscale x 8 x float>
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP17]], [[TMP12]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD3]] to <vscale x 8 x float>
+; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP8]], [[TMP12]]
; CHECK-NEXT: [[TMP22:%.*]] = fneg <vscale x 8 x float> [[TMP18]]
; CHECK-NEXT: [[PARTIAL_REDUCE9]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE5]], <vscale x 8 x float> [[TMP22]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index 9c064bcfeda5b..1e492a646ac3c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -22,11 +22,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -48,11 +48,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
-; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD4]] to <4 x i32>
; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
; CHECK-EPI-NEXT: [[TMP14:%.*]] = sext <4 x i8> [[WIDE_LOAD5]] to <4 x i32>
-; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
; CHECK-EPI-NEXT: [[TMP16]] = sub <4 x i32> [[VEC_PHI3]], [[TMP15]]
; CHECK-EPI-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX2]], 4
; CHECK-EPI-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT6]], 36
@@ -99,11 +99,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -124,11 +124,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP14]], [[TMP13]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI3]], <8 x i32> [[TMP15]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 60b13e460d8c9..e6d618013bb6b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -665,10 +665,10 @@ define void @narrow_with_select_and_invariant_cond(ptr noalias %dst, ptr noalias
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP10]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
-; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD1]], splat (i64 -1)
-; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[WIDE_LOAD]], <2 x i64> [[TMP4]]
+; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
+; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD2]], splat (i64 -1)
+; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[TMP4]]
; VF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP10]]
; VF2-NEXT: store <2 x i64> [[TMP5]], ptr [[TMP6]], align 4
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 29849f2c3d008..7a1598a704e06 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -1521,10 +1521,10 @@ define void @test_2xdouble_same_fcmp_predicate(ptr noalias %data) {
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[WIDE_LOAD]], zeroinitializer
-; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD1]]
+; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
+; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[WIDE_LOAD1]], zeroinitializer
+; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD2]]
; VF2-NEXT: store <2 x double> [[TMP3]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 5c9a3eac44fa0..c78b3f19dd05a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -612,12 +612,12 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF2: [[VECTOR_BODY]]:
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
+; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
; VF2-NEXT: [[TMP1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP2:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP2]], align 8
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD2]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
new file mode 100644
index 0000000000000..c918d16f56ca1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -mtriple=x86_64-- -mcpu=skylake-avx512 -S | FileCheck %s
+
+; Tests CSE of gather loads.
+
+; Two identical gathers with the same index vector are CSE'd into one.
+define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_gather(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %idx = load i64, ptr %gep.b, align 8
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %idx
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Gathers with different index vectors are not CSE'd.
+define void @dup_gather_diff_index(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_gather_diff_index(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP2]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER3]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %idx1 = load i64, ptr %gep.b, align 8
+ %gep.c = getelementptr inbounds i64, ptr %c, i64 %iv
+ %idx2 = load i64, ptr %gep.c, align 8
+ %gep.a1 = getelementptr inbounds i32, ptr %a, i64 %idx1
+ %gep.a2 = getelementptr inbounds i32, ptr %a, i64 %idx2
+ %x = load i32, ptr %gep.a1, align 4
+ %y = load i32, ptr %gep.a2, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index afd192246b7ef..056f3941b0842 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -287,10 +287,10 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
; CHECK-NEXT: [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP15]]
; CHECK-NEXT: [[NEXT_GEP23:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15:![0-9]+]]
-; CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-NEXT: [[WIDE_LOAD24:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15]]
-; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD24]] to <16 x i32>
-; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP19]], [[TMP21]]
+; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15]]
+; CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
+; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP21]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP22]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
; CHECK-NEXT: [[TMP24:%.*]] = lshr <16 x i32> [[TMP23]], splat (i32 1)
; CHECK-NEXT: [[TMP25:%.*]] = trunc <16 x i32> [[TMP24]] to <16 x i8>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
index bf14c06882195..810160a2c2f93 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
@@ -201,18 +201,18 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP3]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD7]], [[WIDE_LOAD12]]
; CHECK-NEXT: store <4 x double> [[TMP8]], ptr [[TMP4]], align 8
; CHECK-NEXT: store <4 x double> [[TMP9]], ptr [[TMP5]], align 8
; CHECK-NEXT: store <4 x double> [[TMP10]], ptr [[TMP6]], align 8
@@ -231,9 +231,9 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX9]], 2
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[P]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD10]], [[WIDE_LOAD11]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD11]], [[WIDE_LOAD13]]
; CHECK-NEXT: store <4 x double> [[TMP14]], ptr [[TMP13]], align 8
; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX9]], 1
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT12]], 99
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
new file mode 100644
index 0000000000000..88c1705e9fa22
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Additional load-CSE coverage.
+
+; Three identical loads are CSE'd into one.
+define void @triple_dup_load(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @triple_dup_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %z = load i32, ptr %p, align 4
+ %s1 = add i32 %x, %y
+ %s2 = add i32 %s1, %z
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s2, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Two independent groups of duplicate loads are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @two_dup_groups(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A duplicate load and the duplicate add depending on it are both CSE'd.
+define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @load_then_dup_add(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 7)
+; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP6]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %ax = add i32 %x, 7
+ %ay = add i32 %y, 7
+ %m = mul i32 %ax, %ay
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %m, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Two identical loads with different TBAA are CSE'd, keeping only common metadata.
+define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_diff_tbaa(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[INT_TBAA5:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4, !tbaa !0
+ %y = load i32, ptr %p, align 4, !tbaa !4
+ %sum = add i32 %x, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads separated by a replicate region are not CSE'd.
+define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_across_replicate_region(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY1]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br label %[[CONT:.*]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %cont
+then:
+ %idx = sext i32 %x to i64
+ %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
+ %ld = load i32, ptr %gep2, align 4
+ br label %cont
+cont:
+ %v = phi i32 [ %ld, %then ], [ 0, %loop ]
+ %y = load i32, ptr %p, align 4
+ %sum = add i32 %v, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"float", !2, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
new file mode 100644
index 0000000000000..269e6e9fbbdd1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -force-target-supports-masked-memory-ops -S | FileCheck %s
+
+; Tests CSE of masked widened loads and mixed masked/regular loads.
+
+; Two masked loads from the same address under the same condition are CSE'd.
+define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond) {
+; CHECK-LABEL: define void @two_masked_same_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: br label %[[LATCH:.*]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+if:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %s = add i32 %x, %y
+ br label %latch
+latch:
+ %m = phi i32 [ 0, %loop ], [ %s, %if ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %m, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Masked loads from the same address under different conditions are not CSE'd.
+define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2) {
+; CHECK-LABEL: define void @two_masked_diff_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C1:%.*]], ptr noalias [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[C1]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C2]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[WIDE_MASKED_LOAD2]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[PREDPHI]], [[PREDPHI3]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP7]], label %[[IF2:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[IF2]]:
+; CHECK-NEXT: br label %[[LATCH2:.*]]
+; CHECK: [[LATCH2]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
+ %v1 = load i32, ptr %gp1, align 4
+ %cmp1 = icmp ne i32 %v1, 0
+ br i1 %cmp1, label %if1, label %latch1
+if1:
+ %x = load i32, ptr %gep.a, align 4
+ br label %latch1
+latch1:
+ %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
+ %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
+ %v2 = load i32, ptr %gp2, align 4
+ %cmp2 = icmp ne i32 %v2, 0
+ br i1 %cmp2, label %if2, label %latch2
+if2:
+ %y = load i32, ptr %gep.a, align 4
+ br label %latch2
+latch2:
+ %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
+ %sum = add i32 %mx, %my
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A duplicated regular load and a duplicated masked load are each CSE'd independently.
+define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias %cond, ptr noalias %out) {
+; CHECK-LABEL: define void @masked_and_regular_mix(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD3]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP8]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: br label %[[LATCH:.*]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %r1 = load i32, ptr %gep.a, align 4
+ %r2 = load i32, ptr %gep.a, align 4
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+if:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %m1 = load i32, ptr %gep.b, align 4
+ %m2 = load i32, ptr %gep.b, align 4
+ %ms = add i32 %m1, %m2
+ br label %latch
+latch:
+ %mm = phi i32 [ 0, %loop ], [ %ms, %if ]
+ %rr = add i32 %r1, %r2
+ %sum = add i32 %rr, %mm
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
new file mode 100644
index 0000000000000..9dcc30b758042
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -tail-folding-policy=must-fold-tail -force-tail-folding-style=data \
+; RUN: -force-target-supports-masked-memory-ops -S | FileCheck %s
+
+; Under tail folding, two identical masked loads from the same address are CSE'd.
+define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_duplicate_masked_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[N]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP1]], ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
new file mode 100644
index 0000000000000..26b60236165e9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Tests CSE of duplicate widened loads.
+
+; Two identical loads from the same address are CSE'd into one.
+define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_duplicate_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+define void @no_cse_across_store(ptr %a) {
+; CHECK-LABEL: define void @no_cse_across_store(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 2)
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %sink = add i32 %y, 2
+ %gep.s = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %sink, ptr %gep.s, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads from different addresses are not CSE'd.
+define void @no_cse_different_address(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @no_cse_different_address(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %y = load i32, ptr %gep.b, align 4
+ %sum = add i32 %x, %y
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %sum, ptr %gep.c, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads of different scalar types are not CSE'd.
+define void @no_cse_different_type(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_different_type(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = zext <4 x i32> [[TMP14]] to <4 x i64>
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i64> [[TMP15]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i64, ptr %gep, align 8
+ %xe = zext i32 %x to i64
+ %sum = add i64 %xe, %y
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %sum, ptr %gep.b, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads from the same address with different alignment are not CSE'd.
+define void @no_cse_different_align(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_different_align(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Three identical loads from the same address collapse to a single load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_three_duplicate_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A store proven not to alias the loaded address (via scoped-noalias metadata)
+; does not prevent CSE of the second load.
+define void @cse_across_noalias_store(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @cse_across_noalias_store(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4, !alias.scope !3, !noalias !0
+ %y = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4, !alias.scope !3, !noalias !0
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain"}
+!3 = !{!4}
+!4 = distinct !{!4, !2, !"scope_b"}
diff --git a/llvm/test/Transforms/LoopVectorize/cse.ll b/llvm/test/Transforms/LoopVectorize/cse.ll
new file mode 100644
index 0000000000000..47ab7ee60dc4a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Tests CSE of duplicate non-memory recipes (GEPs, index arithmetic, casts).
+
+; Duplicate address GEPs for a load and a store are CSE'd into one.
+define void @dup_gep(ptr noalias %a) {
+; CHECK-LABEL: define void @dup_gep(
+; CHECK-SAME: ptr noalias [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %v = load i32, ptr %p, align 4
+ %q = getelementptr inbounds i32, ptr %a, i64 %iv
+ %inc = add i32 %v, 1
+ store i32 %inc, ptr %q, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Duplicate index arithmetic used for two arrays is CSE'd.
+define void @dup_index(ptr noalias %a, ptr noalias %b, i64 %o) {
+; CHECK-LABEL: define void @dup_index(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[O:%.*]]) {
+; CHECK-NEXT: [[VECTOR_SCEVCHECK:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[O]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %i1 = add i64 %iv, %o
+ %i2 = add i64 %iv, %o
+ %pa = getelementptr inbounds i32, ptr %a, i64 %i1
+ %pb = getelementptr inbounds i32, ptr %b, i64 %i2
+ %vb = load i32, ptr %pb, align 4
+ store i32 %vb, ptr %pa, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A duplicated loop-invariant cast is CSE'd.
+define void @dup_invariant_cast(ptr noalias %a, i16 %k) {
+; CHECK-LABEL: define void @dup_invariant_cast(
+; CHECK-SAME: ptr noalias [[A:%.*]], i16 [[K:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %z1 = zext i16 %k to i32
+ %z2 = zext i16 %k to i32
+ %sum = add i32 %z1, %z2
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %sum, ptr %p, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Casts of the same operand to different result types are not CSE'd.
+define void @no_cse_different_cast_type(ptr noalias %a, ptr noalias %b, i16 %k) {
+; CHECK-LABEL: define void @no_cse_different_cast_type(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i16 [[K:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i64>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP3]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %z32 = zext i16 %k to i32
+ %z64 = zext i16 %k to i64
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %z32, ptr %pa, align 4
+ %pb = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %z64, ptr %pb, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
>From c2b68c10ae5161ac4a098f2ee1620e7faf02b91f Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Wed, 5 Aug 2026 22:36:58 +0530
Subject: [PATCH 2/9] [VPlan] Extend cse to eliminate redundant widened loads
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 58 ++++++++++++++++++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 1 +
.../AArch64/partial-reduce-fdot-product.ll | 4 +-
.../partial-reduce-sub-epilogue-vec.ll | 24 +++-----
...e-to-widen-memory-with-wide-ops-chained.ll | 3 +-
...nterleave-to-widen-memory-with-wide-ops.ll | 9 +--
...sform-narrow-interleave-to-widen-memory.ll | 3 +-
.../LoopVectorize/X86/cse-loads-gather.ll | 3 +-
.../LoopVectorize/X86/induction-costs.ll | 14 ++---
...interleave-to-widen-memory-epilogue-vec.ll | 23 +++-----
...sform-narrow-interleave-to-widen-memory.ll | 3 +-
.../LoopVectorize/cse-loads-extra.ll | 25 +++-----
.../LoopVectorize/cse-loads-masked.ll | 9 +--
.../LoopVectorize/cse-loads-tailfold.ll | 3 +-
.../Transforms/LoopVectorize/cse-loads.ll | 12 ++--
15 files changed, 104 insertions(+), 90 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 74a4ba2d32ee7..33812ba2ad5b7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -280,6 +280,28 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
return true;
}
+/// Return true if no memory-writing recipe between \p From and \p To may alias
+/// \p MemLoc. Unlike canHoistOrSinkWithNoAliasCheck, this only scans the
+/// recipes strictly between \p From and \p To (which must be in the same
+/// block), as needed when common-subexpression-eliminating two loads: writes
+/// before \p From or after \p To cannot affect the reused value.
+static bool canCSEWithNoAliasCheck(const MemoryLocation &MemLoc,
+ VPRecipeBase *From, VPRecipeBase *To) {
+ if (From->getParent() != To->getParent())
+ return false;
+
+ for (VPRecipeBase &R :
+ make_range(std::next(From->getIterator()), To->getIterator())) {
+ if (!R.mayWriteToMemory())
+ continue;
+ auto Loc = vputils::getMemoryLocation(R);
+ if (!Loc ||
+ ScopedNoAliasAAResult::alias(*Loc, MemLoc) != AliasResult::NoAlias)
+ return false;
+ }
+ return true;
+}
+
/// Get the value type of the replicate load or store. \p IsLoad indicates
/// whether it is a load.
static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
@@ -2210,8 +2232,12 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // During CSE, we can only handle non-memory recipes, as memory can alias.
- return !Def->mayReadOrWriteMemory();
+ // Widened loads only read memory; the cse() driver guards their reuse with
+ // an aliasing check. Reject anything that may write, and any other recipe
+ // that reads memory.
+ if (Def->mayWriteToMemory())
+ return false;
+ return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
}
/// Hash the underlying data of \p Def.
@@ -2225,6 +2251,11 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, RFlags->getPredicate());
if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Def))
return hash_combine(Result, SIVSteps->getInductionOpcode());
+ // Alignment and the consecutive/masked flags are stored separately from
+ // the operands, so fold them in here.
+ if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
+ return hash_combine(Result, Load->getAlign().value(),
+ Load->isConsecutive(), Load->isMasked());
return Result;
}
@@ -2249,6 +2280,15 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
if (LSIV->getInductionOpcode() !=
cast<VPScalarIVStepsRecipe>(R)->getInductionOpcode())
return false;
+ // Alignment and the consecutive/masked flags are stored separately from
+ // the operands, so compare them explicitly.
+ if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L)) {
+ auto *RL = cast<VPWidenLoadRecipe>(R);
+ if (LL->getAlign() != RL->getAlign() ||
+ LL->isConsecutive() != RL->isConsecutive() ||
+ LL->isMasked() != RL->isMasked())
+ return false;
+ }
// Phi recipes can only be equal if they are in the same VPBB, as they
// implicitly depend on their predecessors.
if (isa<VPWidenPHIRecipe>(L) && L->getParent() != R->getParent())
@@ -2267,7 +2307,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
};
} // end anonymous namespace
-/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
+/// Perform a common-subexpression-elimination of single-def recipes on the \p
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
VPDominatorTree VPDT(Plan);
@@ -2284,6 +2324,18 @@ void VPlanTransforms::cse(VPlan &Plan) {
// V must dominate Def for a valid replacement.
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
+ // Reuse an earlier widened load only if no aliasing write lies between
+ // the two loads.
+ if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def)) {
+ auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
+ std::optional<MemoryLocation> Loc = vputils::getMemoryLocation(*Load);
+ if (!Loc || !canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
+ continue;
+ // Keep only metadata common to both loads on the survivor.
+ EarlierLoad->intersect(*Load);
+ Load->replaceAllUsesWith(EarlierLoad);
+ continue;
+ }
// Only keep flags present on both V and Def.
if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e6b267493d987..2970e0ae99b76 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -361,6 +361,7 @@ unsigned vputils::getOpcode(const VPValue *V) {
.Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
VPReplicateRecipe, VPWidenPHIRecipe>(
[](auto *I) { return I->getOpcode(); })
+ .Case<VPWidenLoadRecipe>([](auto *) { return Instruction::Load; })
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
[](auto *I) {
// For recipes that do not directly map to LLVM IR instructions,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
index 81d59df9918f4..220c9dbcd548b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
@@ -1022,9 +1022,7 @@ define float @fadd_fsub_chain_f16_f32(ptr %a, ptr %b, ptr %c) #0 {
; CHECK-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float>
; CHECK-NEXT: [[TMP14:%.*]] = fmul <vscale x 8 x float> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[PARTIAL_REDUCE5:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI1]], <vscale x 8 x float> [[TMP14]])
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD3]] to <vscale x 8 x float>
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP8]], [[TMP12]]
+; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP13]], [[TMP12]]
; CHECK-NEXT: [[TMP22:%.*]] = fneg <vscale x 8 x float> [[TMP18]]
; CHECK-NEXT: [[PARTIAL_REDUCE9]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE5]], <vscale x 8 x float> [[TMP22]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index 1e492a646ac3c..5a830113a2155 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -23,10 +23,8 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
-; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP6]]
; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -50,9 +48,7 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
; CHECK-EPI-NEXT: [[TMP14:%.*]] = sext <4 x i8> [[WIDE_LOAD5]] to <4 x i32>
-; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
-; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
-; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP14]]
; CHECK-EPI-NEXT: [[TMP16]] = sub <4 x i32> [[VEC_PHI3]], [[TMP15]]
; CHECK-EPI-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX2]], 4
; CHECK-EPI-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT6]], 36
@@ -62,11 +58,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX5:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-EPI: [[LOOP]]:
; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX7]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX5]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load i8, ptr [[SRC1_GEP]], align 4
; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = sext i8 [[SRC1_LOAD]] to i32
@@ -100,10 +96,8 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP6]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -125,10 +119,8 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP14]], [[TMP13]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP14]], [[TMP14]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI3]], <8 x i32> [[TMP15]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
@@ -139,11 +131,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX6]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load i8, ptr [[SRC1_GEP]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = sext i8 [[SRC1_LOAD]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index e6d618013bb6b..7f1b2c4f2cd55 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -666,8 +666,7 @@ define void @narrow_with_select_and_invariant_cond(ptr noalias %dst, ptr noalias
; VF2-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP10]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
-; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
-; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD2]], splat (i64 -1)
+; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD1]], splat (i64 -1)
; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[TMP4]]
; VF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP10]]
; VF2-NEXT: store <2 x i64> [[TMP5]], ptr [[TMP6]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 7a1598a704e06..a90e82b826775 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -576,8 +576,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP6:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
-; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF2-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD]]
; VF2-NEXT: [[TMP14:%.*]] = or disjoint i64 [[TMP6]], 1
; VF2-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP14]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP15]], align 8
@@ -607,8 +606,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP10:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP10]]
-; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD]]
; VF4-NEXT: [[TMP24:%.*]] = or disjoint i64 [[TMP10]], 1
; VF4-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP24]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP25]], align 8
@@ -1522,9 +1520,8 @@ define void @test_2xdouble_same_fcmp_predicate(ptr noalias %data) {
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[WIDE_LOAD1]], zeroinitializer
-; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD2]]
+; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD1]]
; VF2-NEXT: store <2 x double> [[TMP3]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index c78b3f19dd05a..34c730b32f092 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -613,11 +613,10 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
-; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
; VF2-NEXT: [[TMP1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP2:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD2]], ptr [[TMP2]], align 8
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
index c918d16f56ca1..9b6aa7e0850ed 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
@@ -18,8 +18,7 @@ define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 056f3941b0842..9029f4d833178 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -288,9 +288,7 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
; CHECK-NEXT: [[NEXT_GEP23:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15:![0-9]+]]
; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15]]
-; CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
-; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP21]], [[TMP19]]
+; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP21]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP22]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
; CHECK-NEXT: [[TMP24:%.*]] = lshr <16 x i32> [[TMP23]], splat (i32 1)
; CHECK-NEXT: [[TMP25:%.*]] = trunc <16 x i32> [[TMP24]] to <16 x i8>
@@ -335,15 +333,15 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ -12, %[[MIDDLE_BLOCK]] ], [ 100, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 2048, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL19:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL20:%.*]] = phi ptr [ [[IND_END5]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL18:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL19:%.*]] = phi ptr [ [[IND_END5]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[DEC:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[ADD38:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[BC_RESUME_VAL19]], %[[SCALAR_PH]] ], [ [[OUTPTR_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL20]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR36:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[BC_RESUME_VAL20]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR33:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[BC_RESUME_VAL18]], %[[SCALAR_PH]] ], [ [[OUTPTR_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL19]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR36:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[BC_RESUME_VAL19]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR33:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[INCDEC_PTR33]] = getelementptr i8, ptr [[PTR_IV_3]], i64 1
; CHECK-NEXT: [[TMP43:%.*]] = load i8, ptr [[PTR_IV_3]], align 1
; CHECK-NEXT: [[CONV34:%.*]] = zext i8 [[TMP43]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
index 810160a2c2f93..4fa4e1fe80826 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
@@ -205,14 +205,10 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
-; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
-; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
-; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
-; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD7]], [[WIDE_LOAD12]]
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD4]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD5]], [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD6]], [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD7]], [[WIDE_LOAD7]]
; CHECK-NEXT: store <4 x double> [[TMP8]], ptr [[TMP4]], align 8
; CHECK-NEXT: store <4 x double> [[TMP9]], ptr [[TMP5]], align 8
; CHECK-NEXT: store <4 x double> [[TMP10]], ptr [[TMP6]], align 8
@@ -232,8 +228,7 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX9]], 2
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[P]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD11]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD11]], [[WIDE_LOAD11]]
; CHECK-NEXT: store <4 x double> [[TMP14]], ptr [[TMP13]], align 8
; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX9]], 1
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT12]], 99
@@ -241,12 +236,12 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi i64 [ 396, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 384, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL14:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL8:%.*]] = phi i64 [ 396, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 384, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL9:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL14]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL8]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL9]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[V0:%.*]] = load double, ptr [[GEP0]], align 8
; CHECK-NEXT: [[R0:%.*]] = fadd double [[V0]], [[V0]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
index 8653d002e4cad..94d55191ddd74 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
@@ -472,8 +472,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[GEP_SRC_0]], align 8
; CHECK-NEXT: [[TMP5:%.*]] = shl nsw i64 [[IV]], 1
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP5]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[GEP_SRC_0]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[TMP5]], 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP7]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP8]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
index 88c1705e9fa22..8edb8ee72070d 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
@@ -15,10 +15,8 @@ define void @triple_dup_load(ptr noalias %a, ptr noalias %out) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -60,12 +58,10 @@ define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
@@ -111,10 +107,8 @@ define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
-; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 7)
-; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP6]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -155,14 +149,13 @@ define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[INT_TBAA5:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -246,7 +239,7 @@ define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, pt
; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[THEN]]:
; CHECK-NEXT: br label %[[CONT:.*]]
; CHECK: [[CONT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
index 269e6e9fbbdd1..b7be1727fd989 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
@@ -19,8 +19,7 @@ define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
@@ -134,16 +133,14 @@ define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD3]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[PREDPHI]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
index 9dcc30b758042..4529264346f5f 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -19,8 +19,7 @@ define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[N]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP1]], ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index 26b60236165e9..68a8e784b0c5e 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -15,8 +15,7 @@ define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -240,10 +239,8 @@ define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -288,8 +285,7 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
>From 0ddb1924798af971493e94ffaa0c8210cf05543a Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Thu, 6 Aug 2026 15:35:06 +0530
Subject: [PATCH 3/9] [VPlan] Address review comments for widened-load CSE
Adds getOpcode() to VPWidenLoadRecipe and a negative aliasing test.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 3 +-
.../Transforms/LoopVectorize/cse-loads.ll | 44 +++++++++++++++++++
4 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index fe1342e4309ab..678b157a4ced7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3828,6 +3828,9 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
+ /// Returns the opcode of the widened load.
+ unsigned getOpcode() const { return Instruction::Load; }
+
/// Generate a wide load or gather.
void execute(VPTransformState &State) override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 33812ba2ad5b7..1439ff057fce9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2232,7 +2232,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads only read memory; the cse() driver guards their reuse with
+ // Widened loads only read memory; the cse() routine guards their reuse with
// an aliasing check. Reject anything that may write, and any other recipe
// that reads memory.
if (Def->mayWriteToMemory())
@@ -2326,15 +2326,13 @@ void VPlanTransforms::cse(VPlan &Plan) {
continue;
// Reuse an earlier widened load only if no aliasing write lies between
// the two loads.
- if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def)) {
- auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
- std::optional<MemoryLocation> Loc = vputils::getMemoryLocation(*Load);
- if (!Loc || !canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
+ if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
+ auto *Load = cast<VPWidenLoadRecipe>(Def);
+ auto Loc = vputils::getMemoryLocation(*EarlierLoad);
+ if (!canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
continue;
// Keep only metadata common to both loads on the survivor.
EarlierLoad->intersect(*Load);
- Load->replaceAllUsesWith(EarlierLoad);
- continue;
}
// Only keep flags present on both V and Def.
if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 2970e0ae99b76..4da58a794bb38 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -359,9 +359,8 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
unsigned vputils::getOpcode(const VPValue *V) {
return TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
- VPReplicateRecipe, VPWidenPHIRecipe>(
+ VPReplicateRecipe, VPWidenPHIRecipe, VPWidenLoadRecipe>(
[](auto *I) { return I->getOpcode(); })
- .Case<VPWidenLoadRecipe>([](auto *) { return Instruction::Load; })
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
[](auto *I) {
// For recipes that do not directly map to LLVM IR instructions,
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index 68a8e784b0c5e..af9c67fd9e5d9 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -313,6 +313,50 @@ exit:
ret void
}
+; A store without alias metadata cannot be proven not to alias the loaded
+; address, so the second load is not CSE'd.
+define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_across_store_without_scopes(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4
+ %y = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
!0 = !{!1}
!1 = distinct !{!1, !2, !"scope_a"}
!2 = distinct !{!2, !"domain"}
>From fddf0d624876d139ef0f8cc16133018488663113 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 11 Aug 2026 16:24:59 +0530
Subject: [PATCH 4/9] [VPlan] Allow widened-load CSE across differing
alignments
Reuse widened loads only with equal-or-better alignment.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 35 +-
.../LoopVectorize/X86/cse-loads-gather.ll | 4 +
.../LoopVectorize/cse-loads-extra.ll | 279 ----------
.../LoopVectorize/cse-loads-masked.ll | 41 +-
.../LoopVectorize/cse-loads-tailfold.ll | 2 +
.../Transforms/LoopVectorize/cse-loads.ll | 492 ++++++++++++++++--
llvm/test/Transforms/LoopVectorize/cse.ll | 165 ------
7 files changed, 499 insertions(+), 519 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/cse.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1439ff057fce9..2d94a969adfcf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2232,9 +2232,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads only read memory; the cse() routine guards their reuse with
- // an aliasing check. Reject anything that may write, and any other recipe
- // that reads memory.
+ // Widened loads are handled, as cse() guards their reuse with an aliasing
+ // check. Any other memory access is rejected.
if (Def->mayWriteToMemory())
return false;
return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
@@ -2251,11 +2250,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, RFlags->getPredicate());
if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Def))
return hash_combine(Result, SIVSteps->getInductionOpcode());
- // Alignment and the consecutive/masked flags are stored separately from
- // the operands, so fold them in here.
+ // Fold in the separately stored consecutive/masked flags. Alignment is left
+ // out of the key and checked by cse().
if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
- return hash_combine(Result, Load->getAlign().value(),
- Load->isConsecutive(), Load->isMasked());
+ return hash_combine(Result, Load->isConsecutive(), Load->isMasked());
return Result;
}
@@ -2280,12 +2278,11 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
if (LSIV->getInductionOpcode() !=
cast<VPScalarIVStepsRecipe>(R)->getInductionOpcode())
return false;
- // Alignment and the consecutive/masked flags are stored separately from
- // the operands, so compare them explicitly.
+ // Compare the separately stored consecutive/masked flags. Alignment is left
+ // out and checked by cse().
if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L)) {
auto *RL = cast<VPWidenLoadRecipe>(R);
- if (LL->getAlign() != RL->getAlign() ||
- LL->isConsecutive() != RL->isConsecutive() ||
+ if (LL->isConsecutive() != RL->isConsecutive() ||
LL->isMasked() != RL->isMasked())
return false;
}
@@ -2324,13 +2321,21 @@ void VPlanTransforms::cse(VPlan &Plan) {
// V must dominate Def for a valid replacement.
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
- // Reuse an earlier widened load only if no aliasing write lies between
- // the two loads.
if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
auto *Load = cast<VPWidenLoadRecipe>(Def);
- auto Loc = vputils::getMemoryLocation(*EarlierLoad);
- if (!canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
+ // Reuse EarlierLoad only if it is at least as aligned as Load and no
+ // aliasing write lies between the two loads.
+ bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign();
+ if (CanReuse) {
+ auto Loc = vputils::getMemoryLocation(*EarlierLoad);
+ CanReuse = canCSEWithNoAliasCheck(*Loc, EarlierLoad, Def);
+ }
+ if (!CanReuse) {
+ // Record Load as the candidate for subsequent loads, as it may be
+ // reusable where EarlierLoad is not.
+ CSEMap[Def] = Def;
continue;
+ }
// Keep only metadata common to both loads on the survivor.
EarlierLoad->intersect(*Load);
}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
index 9b6aa7e0850ed..832b5709bf865 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
@@ -31,6 +31,7 @@ define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
@@ -44,6 +45,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -79,6 +81,7 @@ define void @dup_gather_diff_index(ptr noalias %a, ptr noalias %b, ptr noalias %
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
@@ -95,6 +98,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
deleted file mode 100644
index 8edb8ee72070d..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
+++ /dev/null
@@ -1,279 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
-
-; Additional load-CSE coverage.
-
-; Three identical loads are CSE'd into one.
-define void @triple_dup_load(ptr noalias %a, ptr noalias %out) {
-; CHECK-LABEL: define void @triple_dup_load(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4
- %y = load i32, ptr %p, align 4
- %z = load i32, ptr %p, align 4
- %s1 = add i32 %x, %y
- %s2 = add i32 %s1, %z
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %s2, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Two independent groups of duplicate loads are each CSE'd.
-define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
-; CHECK-LABEL: define void @two_dup_groups(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %pa = getelementptr inbounds i32, ptr %a, i64 %iv
- %a1 = load i32, ptr %pa, align 4
- %a2 = load i32, ptr %pa, align 4
- %pb = getelementptr inbounds i32, ptr %b, i64 %iv
- %b1 = load i32, ptr %pb, align 4
- %b2 = load i32, ptr %pb, align 4
- %sa = add i32 %a1, %a2
- %sb = add i32 %b1, %b2
- %s = add i32 %sa, %sb
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %s, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; A duplicate load and the duplicate add depending on it are both CSE'd.
-define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
-; CHECK-LABEL: define void @load_then_dup_add(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
-; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4
- %y = load i32, ptr %p, align 4
- %ax = add i32 %x, 7
- %ay = add i32 %y, 7
- %m = mul i32 %ax, %ay
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %m, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Two identical loads with different TBAA are CSE'd, keeping only common metadata.
-define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
-; CHECK-LABEL: define void @dup_load_diff_tbaa(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4, !tbaa !0
- %y = load i32, ptr %p, align 4, !tbaa !4
- %sum = add i32 %x, %y
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %sum, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Loads separated by a replicate region are not CSE'd.
-define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
-; CHECK-LABEL: define void @dup_load_across_replicate_region(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[VECTOR_PH:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
-; CHECK: [[VECTOR_BODY1]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY1]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
-; CHECK: [[PRED_LOAD_IF1]]:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD7]]
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: br label %[[CONT:.*]]
-; CHECK: [[CONT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4
- %c = icmp sgt i32 %x, 0
- br i1 %c, label %then, label %cont
-then:
- %idx = sext i32 %x to i64
- %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
- %ld = load i32, ptr %gep2, align 4
- br label %cont
-cont:
- %v = phi i32 [ %ld, %then ], [ 0, %loop ]
- %y = load i32, ptr %p, align 4
- %sum = add i32 %v, %y
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %sum, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-!0 = !{!1, !1, i64 0}
-!1 = !{!"int", !2, i64 0}
-!2 = !{!"omnipotent char", !3, i64 0}
-!3 = !{!"Simple C/C++ TBAA"}
-!4 = !{!5, !5, i64 0}
-!5 = !{!"float", !2, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
index b7be1727fd989..f09de9cf72073 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
@@ -25,26 +25,29 @@ define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP5]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: br label %[[LATCH:.*]]
-; CHECK: [[LATCH]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
%gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
%c = load i32, ptr %gep.c, align 4
%cmp = icmp ne i32 %c, 0
br i1 %cmp, label %if, label %latch
+
if:
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
%x = load i32, ptr %gep.a, align 4
%y = load i32, ptr %gep.a, align 4
%s = add i32 %x, %y
br label %latch
+
latch:
%m = phi i32 [ 0, %loop ], [ %s, %if ]
%gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
@@ -52,6 +55,7 @@ latch:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -82,14 +86,15 @@ define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP7]], label %[[IF2:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[IF2]]:
-; CHECK-NEXT: br label %[[LATCH2:.*]]
-; CHECK: [[LATCH2]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -97,18 +102,22 @@ loop:
%v1 = load i32, ptr %gp1, align 4
%cmp1 = icmp ne i32 %v1, 0
br i1 %cmp1, label %if1, label %latch1
+
if1:
%x = load i32, ptr %gep.a, align 4
br label %latch1
+
latch1:
%mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
%gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
%v2 = load i32, ptr %gp2, align 4
%cmp2 = icmp ne i32 %v2, 0
br i1 %cmp2, label %if2, label %latch2
+
if2:
%y = load i32, ptr %gep.a, align 4
br label %latch2
+
latch2:
%my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
%sum = add i32 %mx, %my
@@ -117,11 +126,13 @@ latch2:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-; A duplicated regular load and a duplicated masked load are each CSE'd independently.
+; A duplicated regular load and a duplicated masked load are each CSE'd
+; independently.
define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias %cond, ptr noalias %out) {
; CHECK-LABEL: define void @masked_and_regular_mix(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[OUT:%.*]]) {
@@ -146,14 +157,15 @@ define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP8]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: br label %[[LATCH:.*]]
-; CHECK: [[LATCH]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -163,12 +175,14 @@ loop:
%c = load i32, ptr %gep.c, align 4
%cmp = icmp ne i32 %c, 0
br i1 %cmp, label %if, label %latch
+
if:
%gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
%m1 = load i32, ptr %gep.b, align 4
%m2 = load i32, ptr %gep.b, align 4
%ms = add i32 %m1, %m2
br label %latch
+
latch:
%mm = phi i32 [ 0, %loop ], [ %ms, %if ]
%rr = add i32 %r1, %r2
@@ -178,6 +192,7 @@ latch:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
index 4529264346f5f..abc34bd18910c 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -32,6 +32,7 @@ define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -43,6 +44,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index af9c67fd9e5d9..151df87401fc7 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -20,14 +20,15 @@ define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -39,6 +40,148 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three identical loads from the same address collapse to a single load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_three_duplicate_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two independent groups of duplicate loads are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @two_dup_groups(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A duplicate load and the duplicate add depending on it are both CSE'd.
+define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @load_then_dup_add(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
+; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %ax = add i32 %x, 7
+ %ay = add i32 %y, 7
+ %m = mul i32 %ax, %ay
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %m, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -62,14 +205,15 @@ define void @no_cse_across_store(ptr %a) {
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -83,6 +227,56 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The store clobbers the first load, but the two loads after it are duplicates
+; of each other and are still CSE'd.
+define void @cse_duplicate_loads_after_store(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_duplicate_loads_after_store(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %z = load i32, ptr %gep, align 4
+ %sum = add i32 %y, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -106,14 +300,15 @@ define void @no_cse_different_address(ptr noalias %a, ptr noalias %b, ptr noalia
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -126,6 +321,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -162,14 +358,15 @@ define void @no_cse_different_type(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep = getelementptr inbounds i64, ptr %a, i64 %iv
@@ -182,13 +379,58 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-; Loads from the same address with different alignment are not CSE'd.
-define void @no_cse_different_align(ptr noalias %a, ptr noalias %b) {
-; CHECK-LABEL: define void @no_cse_different_align(
+; The earlier load has the stronger alignment, so the later load reuses it and
+; no alignment information is lost.
+define void @cse_stronger_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_stronger_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i32, ptr %gep, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The earlier load is weaker aligned, so reusing it would drop the stronger
+; alignment of the second load and CSE is skipped.
+define void @no_cse_weaker_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_weaker_align_first(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
@@ -204,14 +446,15 @@ define void @no_cse_different_align(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -223,14 +466,16 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-; Three identical loads from the same address collapse to a single load.
-define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
-; CHECK-LABEL: define void @cse_three_duplicate_loads(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; Two identical loads with different TBAA are CSE'd, keeping only common
+; metadata.
+define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_diff_tbaa(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -240,12 +485,11 @@ define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -253,19 +497,19 @@ define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %gep.a, align 4
- %y = load i32, ptr %gep.a, align 4
- %z = load i32, ptr %gep.a, align 4
- %t = add i32 %x, %y
- %sum = add i32 %t, %z
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- store i32 %sum, ptr %gep.b, align 4
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4, !tbaa !5
+ %y = load i32, ptr %p, align 4, !tbaa !9
+ %sum = add i32 %x, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -282,14 +526,14 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META16:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -297,6 +541,7 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -309,6 +554,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -325,15 +571,15 @@ define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b)
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13]], !noalias [[META16]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13]], !noalias [[META16]]
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -341,6 +587,7 @@ define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b)
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -353,6 +600,151 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Loads separated by a replicate region are not CSE'd.
+define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_across_replicate_region(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %cont
+
+then:
+ %idx = sext i32 %x to i64
+ %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
+ %ld = load i32, ptr %gep2, align 4
+ br label %cont
+
+cont:
+ %v = phi i32 [ %ld, %then ], [ 0, %loop ]
+ %y = load i32, ptr %p, align 4
+ %sum = add i32 %v, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The first load cannot be reused by the stronger aligned second load, but the
+; second one becomes the candidate for the third load, which reuses it.
+define void @cse_after_weaker_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_after_weaker_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %z = load i32, ptr %gep, align 8
+ %s1 = add i32 %x, %y
+ %sum = add i32 %s1, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -362,3 +754,9 @@ exit:
!2 = distinct !{!2, !"domain"}
!3 = !{!4}
!4 = distinct !{!4, !2, !"scope_b"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"float", !7, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/cse.ll b/llvm/test/Transforms/LoopVectorize/cse.ll
deleted file mode 100644
index 47ab7ee60dc4a..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/cse.ll
+++ /dev/null
@@ -1,165 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
-
-; Tests CSE of duplicate non-memory recipes (GEPs, index arithmetic, casts).
-
-; Duplicate address GEPs for a load and a store are CSE'd into one.
-define void @dup_gep(ptr noalias %a) {
-; CHECK-LABEL: define void @dup_gep(
-; CHECK-SAME: ptr noalias [[A:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %v = load i32, ptr %p, align 4
- %q = getelementptr inbounds i32, ptr %a, i64 %iv
- %inc = add i32 %v, 1
- store i32 %inc, ptr %q, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Duplicate index arithmetic used for two arrays is CSE'd.
-define void @dup_index(ptr noalias %a, ptr noalias %b, i64 %o) {
-; CHECK-LABEL: define void @dup_index(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[O:%.*]]) {
-; CHECK-NEXT: [[VECTOR_SCEVCHECK:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[O]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %i1 = add i64 %iv, %o
- %i2 = add i64 %iv, %o
- %pa = getelementptr inbounds i32, ptr %a, i64 %i1
- %pb = getelementptr inbounds i32, ptr %b, i64 %i2
- %vb = load i32, ptr %pb, align 4
- store i32 %vb, ptr %pa, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; A duplicated loop-invariant cast is CSE'd.
-define void @dup_invariant_cast(ptr noalias %a, i16 %k) {
-; CHECK-LABEL: define void @dup_invariant_cast(
-; CHECK-SAME: ptr noalias [[A:%.*]], i16 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[TMP0]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %z1 = zext i16 %k to i32
- %z2 = zext i16 %k to i32
- %sum = add i32 %z1, %z2
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- store i32 %sum, ptr %p, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Casts of the same operand to different result types are not CSE'd.
-define void @no_cse_different_cast_type(ptr noalias %a, ptr noalias %b, i16 %k) {
-; CHECK-LABEL: define void @no_cse_different_cast_type(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i16 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i64>
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP3]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %z32 = zext i16 %k to i32
- %z64 = zext i16 %k to i64
- %pa = getelementptr inbounds i32, ptr %a, i64 %iv
- store i32 %z32, ptr %pa, align 4
- %pb = getelementptr inbounds i64, ptr %b, i64 %iv
- store i64 %z64, ptr %pb, align 8
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
>From 578d543d726378764c7e9fa3ca5048fcb3f3fbd5 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 11 Aug 2026 17:34:54 +0530
Subject: [PATCH 5/9] [VPlan] Replace widened-load CSE alias scan with a memory
epoch
Reuse widened loads only within the same memory epoch, incremented on
every memory-writing recipe. This removes canCSEWithNoAliasCheck, but any
write now invalidates reuse, including across non-aliasing stores.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 +++++++------------
.../Transforms/LoopVectorize/cse-loads.ll | 12 ++--
2 files changed, 28 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2d94a969adfcf..292e298763e95 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -280,28 +280,6 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
return true;
}
-/// Return true if no memory-writing recipe between \p From and \p To may alias
-/// \p MemLoc. Unlike canHoistOrSinkWithNoAliasCheck, this only scans the
-/// recipes strictly between \p From and \p To (which must be in the same
-/// block), as needed when common-subexpression-eliminating two loads: writes
-/// before \p From or after \p To cannot affect the reused value.
-static bool canCSEWithNoAliasCheck(const MemoryLocation &MemLoc,
- VPRecipeBase *From, VPRecipeBase *To) {
- if (From->getParent() != To->getParent())
- return false;
-
- for (VPRecipeBase &R :
- make_range(std::next(From->getIterator()), To->getIterator())) {
- if (!R.mayWriteToMemory())
- continue;
- auto Loc = vputils::getMemoryLocation(R);
- if (!Loc ||
- ScopedNoAliasAAResult::alias(*Loc, MemLoc) != AliasResult::NoAlias)
- return false;
- }
- return true;
-}
-
/// Get the value type of the replicate load or store. \p IsLoad indicates
/// whether it is a load.
static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
@@ -2232,8 +2210,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads are handled, as cse() guards their reuse with an aliasing
- // check. Any other memory access is rejected.
+ // Widened loads are handled, as cse() guards their reuse with a memory
+ // epoch check. Any other memory access is rejected.
if (Def->mayWriteToMemory())
return false;
return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
@@ -2308,32 +2286,40 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
VPDominatorTree VPDT(Plan);
- DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
+ // Maps a recipe to the equivalent recipe recorded earlier, together with the
+ // memory epoch it was recorded at.
+ DenseMap<VPSingleDefRecipe *, std::pair<VPSingleDefRecipe *, unsigned>,
+ VPCSEDenseMapInfo>
+ CSEMap;
+ // Incremented for every recipe that may write to memory.
+ unsigned MemEpoch = 0;
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : *VPBB) {
+ if (R.mayWriteToMemory())
+ ++MemEpoch;
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
continue;
- if (VPSingleDefRecipe *V = CSEMap.lookup(Def)) {
+ auto It = CSEMap.find(Def);
+ if (It != CSEMap.end()) {
+ auto [V, Epoch] = It->second;
// V must dominate Def for a valid replacement.
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
auto *Load = cast<VPWidenLoadRecipe>(Def);
- // Reuse EarlierLoad only if it is at least as aligned as Load and no
- // aliasing write lies between the two loads.
- bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign();
- if (CanReuse) {
- auto Loc = vputils::getMemoryLocation(*EarlierLoad);
- CanReuse = canCSEWithNoAliasCheck(*Loc, EarlierLoad, Def);
- }
+ // Reuse EarlierLoad only if it is at least as aligned as Load and in
+ // the same block, with an unchanged epoch meaning no write lies
+ // between the two loads.
+ bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign() &&
+ EarlierLoad->getParent() == VPBB && Epoch == MemEpoch;
if (!CanReuse) {
// Record Load as the candidate for subsequent loads, as it may be
// reusable where EarlierLoad is not.
- CSEMap[Def] = Def;
+ CSEMap[Def] = {Def, MemEpoch};
continue;
}
// Keep only metadata common to both loads on the survivor.
@@ -2345,7 +2331,7 @@ void VPlanTransforms::cse(VPlan &Plan) {
Def->replaceAllUsesWith(V);
continue;
}
- CSEMap[Def] = Def;
+ CSEMap[Def] = {Def, MemEpoch};
}
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index 151df87401fc7..b19bf3b59957d 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -514,10 +514,11 @@ exit:
ret void
}
-; A store proven not to alias the loaded address (via scoped-noalias metadata)
-; does not prevent CSE of the second load.
-define void @cse_across_noalias_store(ptr %a, ptr %b) {
-; CHECK-LABEL: define void @cse_across_noalias_store(
+; A store between two loads prevents CSE of the second load.
+; TODO: The store is proven not to alias the loaded address via scoped-noalias
+; metadata, so the second load could be CSE'd here.
+define void @no_cse_across_noalias_store(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @no_cse_across_noalias_store(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
@@ -529,7 +530,8 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META16:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13]], !noalias [[META16]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
>From 6d11e8aa10fee0821293b07b7ebf791cd47e1275 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 25 Aug 2026 18:06:06 +0530
Subject: [PATCH 6/9] Address review comments for widened-load CSE
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 85 +++++++++----------
1 file changed, 38 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 292e298763e95..f426dda29a363 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2210,8 +2210,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads are handled, as cse() guards their reuse with a memory
- // epoch check. Any other memory access is rejected.
+ // Widened loads are handled, as cse() only reuses them within a block with
+ // no intervening memory write. Any other memory access is rejected.
if (Def->mayWriteToMemory())
return false;
return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
@@ -2228,10 +2228,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, RFlags->getPredicate());
if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Def))
return hash_combine(Result, SIVSteps->getInductionOpcode());
- // Fold in the separately stored consecutive/masked flags. Alignment is left
- // out of the key and checked by cse().
+ // Fold in the separately stored consecutive flag. Alignment is left out and
+ // handled by cse.
if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
- return hash_combine(Result, Load->isConsecutive(), Load->isMasked());
+ return hash_combine(Result, Load->isConsecutive());
return Result;
}
@@ -2256,14 +2256,11 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
if (LSIV->getInductionOpcode() !=
cast<VPScalarIVStepsRecipe>(R)->getInductionOpcode())
return false;
- // Compare the separately stored consecutive/masked flags. Alignment is left
- // out and checked by cse().
- if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L)) {
- auto *RL = cast<VPWidenLoadRecipe>(R);
- if (LL->isConsecutive() != RL->isConsecutive() ||
- LL->isMasked() != RL->isMasked())
+ // Compare the separately stored consecutive flag. Alignment is left out and
+ // handled by cse.
+ if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L))
+ if (LL->isConsecutive() != cast<VPWidenLoadRecipe>(R)->isConsecutive())
return false;
- }
// Phi recipes can only be equal if they are in the same VPBB, as they
// implicitly depend on their predecessors.
if (isa<VPWidenPHIRecipe>(L) && L->getParent() != R->getParent())
@@ -2282,56 +2279,50 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
};
} // end anonymous namespace
-/// Perform a common-subexpression-elimination of single-def recipes on the \p
+/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
VPDominatorTree VPDT(Plan);
- // Maps a recipe to the equivalent recipe recorded earlier, together with the
- // memory epoch it was recorded at.
- DenseMap<VPSingleDefRecipe *, std::pair<VPSingleDefRecipe *, unsigned>,
- VPCSEDenseMapInfo>
- CSEMap;
- // Incremented for every recipe that may write to memory.
- unsigned MemEpoch = 0;
+ DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
+ // CSE map for widened loads.
+ DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo>
+ LoadCSEMap;
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ LoadCSEMap.clear();
for (VPRecipeBase &R : *VPBB) {
if (R.mayWriteToMemory())
- ++MemEpoch;
+ LoadCSEMap.clear();
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
continue;
- auto It = CSEMap.find(Def);
- if (It != CSEMap.end()) {
- auto [V, Epoch] = It->second;
- // V must dominate Def for a valid replacement.
- if (!VPDT.dominates(V->getParent(), VPBB))
+ bool IsLoad = isa<VPWidenLoadRecipe>(Def);
+ auto [It, Inserted] =
+ (IsLoad ? LoadCSEMap : CSEMap).try_emplace(Def, Def);
+ if (Inserted)
+ continue;
+ VPSingleDefRecipe *V = It->second;
+ // V must dominate Def for a valid replacement.
+ if (!VPDT.dominates(V->getParent(), VPBB))
+ continue;
+ if (IsLoad) {
+ auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
+ auto *Load = cast<VPWidenLoadRecipe>(Def);
+ if (EarlierLoad->getAlign() < Load->getAlign()) {
+ // Record Load as the candidate for subsequent loads, as it may be
+ // reusable where EarlierLoad is not.
+ It->second = Load;
continue;
- if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
- auto *Load = cast<VPWidenLoadRecipe>(Def);
- // Reuse EarlierLoad only if it is at least as aligned as Load and in
- // the same block, with an unchanged epoch meaning no write lies
- // between the two loads.
- bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign() &&
- EarlierLoad->getParent() == VPBB && Epoch == MemEpoch;
- if (!CanReuse) {
- // Record Load as the candidate for subsequent loads, as it may be
- // reusable where EarlierLoad is not.
- CSEMap[Def] = {Def, MemEpoch};
- continue;
- }
- // Keep only metadata common to both loads on the survivor.
- EarlierLoad->intersect(*Load);
}
- // Only keep flags present on both V and Def.
- if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
- RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def));
- Def->replaceAllUsesWith(V);
- continue;
+ // Keep only metadata common to both loads on the survivor.
+ EarlierLoad->intersect(*Load);
}
- CSEMap[Def] = {Def, MemEpoch};
+ // Only keep flags present on both V and Def.
+ if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
+ RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def));
+ Def->replaceAllUsesWith(V);
}
}
}
>From 39ca9c2e799b9c1cfe31222e261c529aeda1bbac Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 25 Aug 2026 19:51:35 +0530
Subject: [PATCH 7/9] Support EVL load variant in cse
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 23 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +-
.../LoopVectorize/RISCV/cse-loads-evl.ll | 531 ++++++++++++++++++
4 files changed, 549 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 678b157a4ced7..70f78a418af0a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3881,6 +3881,9 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadEVLRecipe final
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadEVLSC)
+ /// Returns the opcode of the widened load.
+ unsigned getOpcode() const { return Instruction::Load; }
+
/// Return the EVL operand.
VPValue *getEVL() const { return getOperand(1); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index f426dda29a363..80f4f461763f9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2210,11 +2210,13 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads are handled, as cse() only reuses them within a block with
- // no intervening memory write. Any other memory access is rejected.
+ // Widened loads (including the EVL variant) are handled, as cse() only
+ // reuses them within a block with no intervening memory write. Any other
+ // memory access is rejected.
if (Def->mayWriteToMemory())
return false;
- return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
+ return !Def->mayReadFromMemory() ||
+ isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Def);
}
/// Hash the underlying data of \p Def.
@@ -2230,7 +2232,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, SIVSteps->getInductionOpcode());
// Fold in the separately stored consecutive flag. Alignment is left out and
// handled by cse.
- if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
+ if (auto *Load = dyn_cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(Def)))
return hash_combine(Result, Load->isConsecutive());
return Result;
}
@@ -2258,8 +2260,9 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return false;
// Compare the separately stored consecutive flag. Alignment is left out and
// handled by cse.
- if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L))
- if (LL->isConsecutive() != cast<VPWidenLoadRecipe>(R)->isConsecutive())
+ if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(L)))
+ if (LL->isConsecutive() !=
+ cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(R))->isConsecutive())
return false;
// Phi recipes can only be equal if they are in the same VPBB, as they
// implicitly depend on their predecessors.
@@ -2298,7 +2301,7 @@ void VPlanTransforms::cse(VPlan &Plan) {
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
continue;
- bool IsLoad = isa<VPWidenLoadRecipe>(Def);
+ bool IsLoad = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Def);
auto [It, Inserted] =
(IsLoad ? LoadCSEMap : CSEMap).try_emplace(Def, Def);
if (Inserted)
@@ -2308,12 +2311,12 @@ void VPlanTransforms::cse(VPlan &Plan) {
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
if (IsLoad) {
- auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
- auto *Load = cast<VPWidenLoadRecipe>(Def);
+ auto *EarlierLoad = cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(V));
+ auto *Load = cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(Def));
if (EarlierLoad->getAlign() < Load->getAlign()) {
// Record Load as the candidate for subsequent loads, as it may be
// reusable where EarlierLoad is not.
- It->second = Load;
+ It->second = Def;
continue;
}
// Keep only metadata common to both loads on the survivor.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 4da58a794bb38..69f49bc5ae691 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -359,8 +359,8 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
unsigned vputils::getOpcode(const VPValue *V) {
return TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
- VPReplicateRecipe, VPWidenPHIRecipe, VPWidenLoadRecipe>(
- [](auto *I) { return I->getOpcode(); })
+ VPReplicateRecipe, VPWidenPHIRecipe, VPWidenLoadRecipe,
+ VPWidenLoadEVLRecipe>([](auto *I) { return I->getOpcode(); })
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
[](auto *I) {
// For recipes that do not directly map to LLVM IR instructions,
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll
new file mode 100644
index 0000000000000..6ffe8bc3aa387
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll
@@ -0,0 +1,531 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v \
+; RUN: -tail-folding-policy=prefer-fold-tail -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -S | FileCheck %s
+
+; CSE of VPWidenLoadEVLRecipe: duplicate consecutive loads become one vp.load.
+
+define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_duplicate_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+define void @no_cse_across_store(ptr %a, i64 %n) {
+; CHECK-LABEL: define void @no_cse_across_store(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD1]], splat (i32 2)
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %sink = add i32 %y, 2
+ store i32 %sink, ptr %gep, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two masked loads from the same address under the same condition are CSE'd.
+define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: define void @two_masked_same_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD1]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP6]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+
+if:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %s = add i32 %x, %y
+ br label %latch
+
+latch:
+ %m = phi i32 [ 0, %loop ], [ %s, %if ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %m, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Masked loads from the same address under different conditions are not CSE'd.
+define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2, i64 %n) {
+; CHECK-LABEL: define void @two_masked_diff_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C1:%.*]], ptr noalias [[C2:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[C1]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> [[TMP3]], i32 [[TMP0]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i32> [[VP_OP_LOAD1]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[C2]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[VP_OP_LOAD3]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[PREDPHI]], [[PREDPHI4]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP6]], ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
+ %v1 = load i32, ptr %gp1, align 4
+ %cmp1 = icmp ne i32 %v1, 0
+ br i1 %cmp1, label %if1, label %latch1
+
+if1:
+ %x = load i32, ptr %gep.a, align 4
+ br label %latch1
+
+latch1:
+ %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
+ %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
+ %v2 = load i32, ptr %gp2, align 4
+ %cmp2 = icmp ne i32 %v2, 0
+ br i1 %cmp2, label %if2, label %latch2
+
+if2:
+ %y = load i32, ptr %gep.a, align 4
+ br label %latch2
+
+latch2:
+ %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
+ %sum = add i32 %mx, %my
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three identical loads collapse to a single vp.load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_three_duplicate_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[TMP2]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two independent duplicate-load groups are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out, i64 %n) {
+; CHECK-LABEL: define void @two_dup_groups(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD1]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Stronger alignment first: later load reuses it, alignment 8 is kept.
+define void @cse_stronger_align_first(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_stronger_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 8 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i32, ptr %gep, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Weaker alignment first: CSE is skipped so the later load keeps align 8.
+define void @no_cse_weaker_align_first(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @no_cse_weaker_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 8 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Any intervening store clears LoadCSEMap, even if it cannot alias the load.
+define void @no_cse_across_noalias_store(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @no_cse_across_noalias_store(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_OP_LOAD]], ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Duplicate gathers (non-consecutive EVL loads) are CSE'd into one vp.gather.
+define void @dup_gather(ptr noalias %a, ptr noalias %idx, ptr noalias %out, i64 %n) {
+; CHECK-LABEL: define void @dup_gather(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[IDX:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[IDX]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <vscale x 4 x i64> [[VP_OP_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.idx = getelementptr inbounds i64, ptr %idx, i64 %iv
+ %i = load i64, ptr %gep.idx, align 8
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %i
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 1f18a88f3a091721627f159cc531b24384a8b762 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Wed, 26 Aug 2026 16:35:13 +0530
Subject: [PATCH 8/9] Refresh cse-loads CHECKs after rebase
---
llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll | 2 +-
llvm/test/Transforms/LoopVectorize/cse-loads.ll | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
index abc34bd18910c..a8d1f73b7bd2e 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -11,7 +11,7 @@ define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N_RND_UP]], 3
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index b19bf3b59957d..f19afabc2d522 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -347,10 +347,10 @@ define void @no_cse_different_type(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i32 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 2
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
; CHECK-NEXT: [[TMP15:%.*]] = zext <4 x i32> [[TMP14]] to <4 x i64>
; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i64> [[TMP15]], [[WIDE_LOAD]]
>From c9d8dcdecde89f7b7100e7960beb12a398a6e329 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Thu, 27 Aug 2026 15:33:19 +0530
Subject: [PATCH 9/9] [VPlan] Wrap cse in RUN_VPLAN_PASS and add VPlan dump
tests
---
.../VPlan/RISCV/cse-loads-evl.ll | 543 ++++++++++++
.../VPlan/X86/cse-loads-gather.ll | 108 +++
.../LoopVectorize/VPlan/cse-loads-masked.ll | 203 +++++
.../LoopVectorize/VPlan/cse-loads-tailfold.ll | 53 ++
.../LoopVectorize/VPlan/cse-loads.ll | 790 ++++++++++++++++++
5 files changed, 1697 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/RISCV/cse-loads-evl.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/X86/cse-loads-gather.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-masked.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-tailfold.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/cse-loads.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/cse-loads-evl.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/cse-loads-evl.ll
new file mode 100644
index 0000000000000..705293083950f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/cse-loads-evl.ll
@@ -0,0 +1,543 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v \
+; RUN: -tail-folding-policy=prefer-fold-tail -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -disable-output \
+; RUN: -vplan-print-after="printFinalVPlan$" %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; CSE of VPWidenLoadEVLRecipe: duplicate consecutive loads become one vp.load.
+
+define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'cse_duplicate_load'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep.a>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+define void @no_cse_across_store(ptr %a, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_across_store'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%inc> = add ir<%x>, ir<1>
+; CHECK-NEXT: WIDEN vp.store ir<%gep>, ir<%inc>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%y> = vp.load ir<%gep>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sink> = add ir<%y>, ir<2>
+; CHECK-NEXT: WIDEN vp.store ir<%gep>, ir<%sink>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %sink = add i32 %y, 2
+ store i32 %sink, ptr %gep, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two masked loads from the same address under the same condition are CSE'd.
+define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'two_masked_same_mask'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep.c> = getelementptr inbounds ir<%cond>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%c> = vp.load ir<%gep.c>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ne ir<%c>, ir<0>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep.a>, vp<%evl>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%s> = add ir<%x>, ir<%x>
+; CHECK-NEXT: EMIT vp<%predphi> = select ir<%cmp>, ir<%s>, ir<0>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, vp<%predphi>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+
+if:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %s = add i32 %x, %y
+ br label %latch
+
+latch:
+ %m = phi i32 [ 0, %loop ], [ %s, %if ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %m, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Masked loads from the same address under different conditions are not CSE'd.
+define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'two_masked_diff_mask'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gp1> = getelementptr inbounds ir<%c1>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%v1> = vp.load ir<%gp1>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%cmp1> = icmp ne ir<%v1>, ir<0>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep.a>, vp<%evl>, ir<%cmp1>
+; CHECK-NEXT: EMIT vp<%predphi> = select ir<%cmp1>, ir<%x>, ir<0>
+; CHECK-NEXT: CLONE ir<%gp2> = getelementptr inbounds ir<%c2>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%v2> = vp.load ir<%gp2>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%cmp2> = icmp ne ir<%v2>, ir<0>
+; CHECK-NEXT: WIDEN ir<%y> = vp.load ir<%gep.a>, vp<%evl>, ir<%cmp2>
+; CHECK-NEXT: EMIT vp<%predphi>.1 = select ir<%cmp2>, ir<%y>, ir<0>
+; CHECK-NEXT: WIDEN ir<%sum> = add vp<%predphi>, vp<%predphi>.1
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
+ %v1 = load i32, ptr %gp1, align 4
+ %cmp1 = icmp ne i32 %v1, 0
+ br i1 %cmp1, label %if1, label %latch1
+
+if1:
+ %x = load i32, ptr %gep.a, align 4
+ br label %latch1
+
+latch1:
+ %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
+ %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
+ %v2 = load i32, ptr %gp2, align 4
+ %cmp2 = icmp ne i32 %v2, 0
+ br i1 %cmp2, label %if2, label %latch2
+
+if2:
+ %y = load i32, ptr %gep.a, align 4
+ br label %latch2
+
+latch2:
+ %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
+ %sum = add i32 %mx, %my
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three identical loads collapse to a single vp.load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'cse_three_duplicate_loads'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep.a>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%t> = add ir<%x>, ir<%x>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%t>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two independent duplicate-load groups are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'two_dup_groups'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%pa> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%a1> = vp.load ir<%pa>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%pb> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%b1> = vp.load ir<%pb>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sa> = add ir<%a1>, ir<%a1>
+; CHECK-NEXT: WIDEN ir<%sb> = add ir<%b1>, ir<%b1>
+; CHECK-NEXT: WIDEN ir<%s> = add ir<%sa>, ir<%sb>
+; CHECK-NEXT: CLONE ir<%po> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%po>, ir<%s>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Stronger alignment first: later load reuses it, alignment 8 is kept.
+define void @cse_stronger_align_first(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'cse_stronger_align_first'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i32, ptr %gep, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Weaker alignment first: CSE is skipped so the later load keeps align 8.
+define void @no_cse_weaker_align_first(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_weaker_align_first'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%y> = vp.load ir<%gep>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Any intervening store clears LoadCSEMap, even if it cannot alias the load.
+define void @no_cse_across_noalias_store(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_across_noalias_store'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep.a>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%x>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%y> = vp.load ir<%gep.a>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.b>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Duplicate gathers (non-consecutive EVL loads) are CSE'd into one vp.gather.
+define void @dup_gather(ptr noalias %a, ptr noalias %idx, ptr noalias %out, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'dup_gather'
+; CHECK: VPlan 'Final VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%current.iteration.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ ir<%n>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: CLONE ir<%gep.idx> = getelementptr inbounds ir<%idx>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%i> = vp.load ir<%gep.idx>, vp<%evl>
+; CHECK-NEXT: WIDEN-GEP ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%i>
+; CHECK-NEXT: WIDEN ir<%x> = vp.load ir<%gep.a>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.o> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN vp.store ir<%gep.o>, ir<%sum>, vp<%evl>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = zext vp<%evl> to i64
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<[[VP2]]>, vp<%index>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%avl.next>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.idx = getelementptr inbounds i64, ptr %idx, i64 %iv
+ %i = load i64, ptr %gep.idx, align 8
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %i
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/cse-loads-gather.ll
new file mode 100644
index 0000000000000..1560305ba90ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/cse-loads-gather.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -mtriple=x86_64-- -mcpu=skylake-avx512 -disable-output \
+; RUN: -vplan-print-after="printFinalVPlan$" %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Tests CSE of gather loads.
+
+; Two identical gathers with the same index vector are CSE'd into one.
+define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'dup_gather'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%idx> = load ir<%gep.b>
+; CHECK-NEXT: WIDEN-GEP ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%idx>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.o> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.o>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %idx = load i64, ptr %gep.b, align 8
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %idx
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Gathers with different index vectors are not CSE'd.
+define void @dup_gather_diff_index(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'dup_gather_diff_index'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%idx1> = load ir<%gep.b>
+; CHECK-NEXT: CLONE ir<%gep.c> = getelementptr inbounds ir<%c>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%idx2> = load ir<%gep.c>
+; CHECK-NEXT: WIDEN-GEP ir<%gep.a1> = getelementptr inbounds ir<%a>, ir<%idx1>
+; CHECK-NEXT: WIDEN-GEP ir<%gep.a2> = getelementptr inbounds ir<%a>, ir<%idx2>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a1>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep.a2>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.o> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.o>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %idx1 = load i64, ptr %gep.b, align 8
+ %gep.c = getelementptr inbounds i64, ptr %c, i64 %iv
+ %idx2 = load i64, ptr %gep.c, align 8
+ %gep.a1 = getelementptr inbounds i32, ptr %a, i64 %idx1
+ %gep.a2 = getelementptr inbounds i32, ptr %a, i64 %idx2
+ %x = load i32, ptr %gep.a1, align 4
+ %y = load i32, ptr %gep.a2, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-masked.ll
new file mode 100644
index 0000000000000..fbb21329dbaa5
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-masked.ll
@@ -0,0 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -force-target-supports-masked-memory-ops -disable-output \
+; RUN: -vplan-print-after="printFinalVPlan$" %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Tests CSE of masked widened loads and mixed masked/regular loads.
+
+; Two masked loads from the same address under the same condition are CSE'd.
+define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond) {
+; CHECK-LABEL: VPlan for loop in 'two_masked_same_mask'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.c> = getelementptr inbounds ir<%cond>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%c> = load ir<%gep.c>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ne ir<%c>, ir<0>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%s> = add ir<%x>, ir<%x>
+; CHECK-NEXT: EMIT vp<%predphi> = select ir<%cmp>, ir<%s>, ir<0>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, vp<%predphi>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+
+if:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %s = add i32 %x, %y
+ br label %latch
+
+latch:
+ %m = phi i32 [ 0, %loop ], [ %s, %if ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %m, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Masked loads from the same address under different conditions are not CSE'd.
+define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2) {
+; CHECK-LABEL: VPlan for loop in 'two_masked_diff_mask'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gp1> = getelementptr inbounds ir<%c1>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%v1> = load ir<%gp1>
+; CHECK-NEXT: WIDEN ir<%cmp1> = icmp ne ir<%v1>, ir<0>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>, ir<%cmp1>
+; CHECK-NEXT: EMIT vp<%predphi> = select ir<%cmp1>, ir<%x>, ir<0>
+; CHECK-NEXT: CLONE ir<%gp2> = getelementptr inbounds ir<%c2>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%v2> = load ir<%gp2>
+; CHECK-NEXT: WIDEN ir<%cmp2> = icmp ne ir<%v2>, ir<0>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep.a>, ir<%cmp2>
+; CHECK-NEXT: EMIT vp<%predphi>.1 = select ir<%cmp2>, ir<%y>, ir<0>
+; CHECK-NEXT: WIDEN ir<%sum> = add vp<%predphi>, vp<%predphi>.1
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
+ %v1 = load i32, ptr %gp1, align 4
+ %cmp1 = icmp ne i32 %v1, 0
+ br i1 %cmp1, label %if1, label %latch1
+
+if1:
+ %x = load i32, ptr %gep.a, align 4
+ br label %latch1
+
+latch1:
+ %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
+ %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
+ %v2 = load i32, ptr %gp2, align 4
+ %cmp2 = icmp ne i32 %v2, 0
+ br i1 %cmp2, label %if2, label %latch2
+
+if2:
+ %y = load i32, ptr %gep.a, align 4
+ br label %latch2
+
+latch2:
+ %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
+ %sum = add i32 %mx, %my
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A duplicated regular load and a duplicated masked load are each CSE'd
+; independently.
+define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias %cond, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'masked_and_regular_mix'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%r1> = load ir<%gep.a>
+; CHECK-NEXT: CLONE ir<%gep.c> = getelementptr inbounds ir<%cond>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%c> = load ir<%gep.c>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ne ir<%c>, ir<0>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%m1> = load ir<%gep.b>, ir<%cmp>
+; CHECK-NEXT: WIDEN ir<%ms> = add ir<%m1>, ir<%m1>
+; CHECK-NEXT: EMIT vp<%predphi> = select ir<%cmp>, ir<%ms>, ir<0>
+; CHECK-NEXT: WIDEN ir<%rr> = add ir<%r1>, ir<%r1>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%rr>, vp<%predphi>
+; CHECK-NEXT: CLONE ir<%gep.o> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.o>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %r1 = load i32, ptr %gep.a, align 4
+ %r2 = load i32, ptr %gep.a, align 4
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+
+if:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %m1 = load i32, ptr %gep.b, align 4
+ %m2 = load i32, ptr %gep.b, align 4
+ %ms = add i32 %m1, %m2
+ br label %latch
+
+latch:
+ %mm = phi i32 [ 0, %loop ], [ %ms, %if ]
+ %rr = add i32 %r1, %r2
+ %sum = add i32 %rr, %mm
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-tailfold.ll
new file mode 100644
index 0000000000000..ac97db6b4947a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads-tailfold.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -tail-folding-policy=must-fold-tail -force-tail-folding-style=data \
+; RUN: -force-target-supports-masked-memory-ops -disable-output \
+; RUN: -vplan-print-after="printFinalVPlan$" %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Under tail folding, two identical masked loads from the same address are CSE'd.
+define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'cse_duplicate_masked_load'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<%n.rnd.up> = add ir<%n>, ir<3>
+; CHECK-NEXT: EMIT vp<[[VP2:%[0-9]+]]> = and vp<%n.rnd.up>, ir<3>
+; CHECK-NEXT: EMIT vp<%n.vec> = sub vp<%n.rnd.up>, vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: EMIT vp<%active.lane.mask> = active lane mask vp<%index>, ir<%n>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>, vp<%active.lane.mask>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>, vp<%active.lane.mask>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = icmp eq vp<%index.next>, vp<%n.vec>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads.ll
new file mode 100644
index 0000000000000..0d6c82e1f749c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/cse-loads.ll
@@ -0,0 +1,790 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block" --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -disable-output -vplan-print-after="printFinalVPlan$" %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Tests CSE of duplicate widened loads.
+
+; Two identical loads from the same address are CSE'd into one.
+define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'cse_duplicate_load'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three identical loads from the same address collapse to a single load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'cse_three_duplicate_loads'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>
+; CHECK-NEXT: WIDEN ir<%t> = add ir<%x>, ir<%x>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%t>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two independent groups of duplicate loads are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'two_dup_groups'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%pa> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%a1> = load ir<%pa>
+; CHECK-NEXT: CLONE ir<%pb> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%b1> = load ir<%pb>
+; CHECK-NEXT: WIDEN ir<%sa> = add ir<%a1>, ir<%a1>
+; CHECK-NEXT: WIDEN ir<%sb> = add ir<%b1>, ir<%b1>
+; CHECK-NEXT: WIDEN ir<%s> = add ir<%sa>, ir<%sb>
+; CHECK-NEXT: CLONE ir<%po> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%po>, ir<%s>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A duplicate load and the duplicate add depending on it are both CSE'd.
+define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'load_then_dup_add'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%p> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%p>
+; CHECK-NEXT: WIDEN ir<%ax> = add ir<%x>, ir<7>
+; CHECK-NEXT: WIDEN ir<%m> = mul ir<%ax>, ir<%ax>
+; CHECK-NEXT: CLONE ir<%po> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%po>, ir<%m>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %ax = add i32 %x, 7
+ %ay = add i32 %y, 7
+ %m = mul i32 %ax, %ay
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %m, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+define void @no_cse_across_store(ptr %a) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_across_store'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%inc> = add ir<%x>, ir<1>
+; CHECK-NEXT: WIDEN store ir<%gep>, ir<%inc>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%sink> = add ir<%y>, ir<2>
+; CHECK-NEXT: WIDEN store ir<%gep>, ir<%sink>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %sink = add i32 %y, 2
+ %gep.s = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %sink, ptr %gep.s, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The store clobbers the first load, but the two loads after it are duplicates
+; of each other and are still CSE'd.
+define void @cse_duplicate_loads_after_store(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'cse_duplicate_loads_after_store'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%inc> = add ir<%x>, ir<1>
+; CHECK-NEXT: WIDEN store ir<%gep>, ir<%inc>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%y>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %z = load i32, ptr %gep, align 4
+ %sum = add i32 %y, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Loads from different addresses are not CSE'd.
+define void @no_cse_different_address(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_different_address'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep.b>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.c> = getelementptr inbounds ir<%c>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.c>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %y = load i32, ptr %gep.b, align 4
+ %sum = add i32 %x, %y
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %sum, ptr %gep.c, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Loads of different scalar types are not CSE'd.
+define void @no_cse_different_type(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_different_type'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: vp<[[VP1:%[0-9]+]]> = SCALAR-STEPS vp<%index>, ir<1>, ir<4>, ir<1>
+; CHECK-NEXT: vp<[[VP2:%[0-9]+]]> = SCALAR-STEPS vp<%index>, ir<1>, ir<4>, ir<2>
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = SCALAR-STEPS vp<%index>, ir<1>, ir<4>, ir<3>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep>.1 = getelementptr inbounds ir<%a>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%gep>.2 = getelementptr inbounds ir<%a>, vp<[[VP2]]>
+; CHECK-NEXT: CLONE ir<%gep>.3 = getelementptr inbounds ir<%a>, vp<[[VP3]]>
+; CHECK-NEXT: CLONE ir<%x> = load ir<%gep>
+; CHECK-NEXT: CLONE ir<%x>.1 = load ir<%gep>.1
+; CHECK-NEXT: CLONE ir<%x>.2 = load ir<%gep>.2
+; CHECK-NEXT: CLONE ir<%x>.3 = load ir<%gep>.3
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = buildvector ir<%x>, ir<%x>.1, ir<%x>.2, ir<%x>.3
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep>
+; CHECK-NEXT: WIDEN-CAST ir<%xe> = zext vp<[[VP4]]> to i64
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%xe>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP5]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i64, ptr %gep, align 8
+ %xe = zext i32 %x to i64
+ %sum = add i64 %xe, %y
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %sum, ptr %gep.b, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The earlier load has the stronger alignment, so the later load reuses it and
+; no alignment information is lost.
+define void @cse_stronger_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'cse_stronger_align_first'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i32, ptr %gep, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The earlier load is weaker aligned, so reusing it would drop the stronger
+; alignment of the second load and CSE is skipped.
+define void @no_cse_weaker_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_weaker_align_first'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two identical loads with different TBAA are CSE'd, keeping only common
+; metadata.
+define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'dup_load_diff_tbaa'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%p> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%p>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%x>
+; CHECK-NEXT: CLONE ir<%po> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%po>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4, !tbaa !5
+ %y = load i32, ptr %p, align 4, !tbaa !9
+ %sum = add i32 %x, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+; TODO: The store is proven not to alias the loaded address via scoped-noalias
+; metadata, so the second load could be CSE'd here.
+define void @no_cse_across_noalias_store(ptr %a, ptr %b) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_across_noalias_store'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a> (!alias.scope !13, !noalias !16)
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%x> (!alias.scope !16, !noalias !13)
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep.a> (!alias.scope !13, !noalias !16)
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum> (!alias.scope !16, !noalias !13)
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4, !alias.scope !3, !noalias !0
+ %y = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4, !alias.scope !3, !noalias !0
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A store without alias metadata cannot be proven not to alias the loaded
+; address, so the second load is not CSE'd.
+define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'no_cse_across_store_without_scopes'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep.a> (!alias.scope !13, !noalias !16)
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%x>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep.a> (!alias.scope !13, !noalias !16)
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%x>, ir<%y>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4
+ %y = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Loads separated by a replicate region are not CSE'd.
+define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: VPlan for loop in 'dup_load_across_replicate_region'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, pred.load.continue ]
+; CHECK-NEXT: CLONE ir<%p> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%p>
+; CHECK-NEXT: WIDEN ir<%c> = icmp sgt ir<%x>, ir<0>
+; CHECK-NEXT: WIDEN-CAST ir<%idx> = sext ir<%x> to i64
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = extractelement ir<%c>, ir<0>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = extractelement ir<%idx>, ir<0>
+; CHECK-NEXT: CLONE ir<%gep2> = getelementptr inbounds ir<%b>, vp<[[VP3]]>
+; CHECK-NEXT: CLONE ir<%ld> = load ir<%gep2>
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = insertelement ir<poison>, ir<%ld>, ir<0>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<poison>, vector.body ], [ vp<[[VP4]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = extractelement ir<%c>, ir<1>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = extractelement ir<%idx>, ir<1>
+; CHECK-NEXT: CLONE ir<%gep2>.1 = getelementptr inbounds ir<%b>, vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%ld>.1 = load ir<%gep2>.1
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = insertelement vp<[[VP5]]>, ir<%ld>.1, ir<1>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP10:%[0-9]+]]> = phi [ vp<[[VP5]]>, pred.load.continue ], [ vp<[[VP9]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extractelement ir<%c>, ir<2>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP11]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extractelement ir<%idx>, ir<2>
+; CHECK-NEXT: CLONE ir<%gep2>.2 = getelementptr inbounds ir<%b>, vp<[[VP13]]>
+; CHECK-NEXT: CLONE ir<%ld>.2 = load ir<%gep2>.2
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = insertelement vp<[[VP10]]>, ir<%ld>.2, ir<2>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP15:%[0-9]+]]> = phi [ vp<[[VP10]]>, pred.load.continue ], [ vp<[[VP14]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extractelement ir<%c>, ir<3>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP16]]>
+; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.if:
+; CHECK-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = extractelement ir<%idx>, ir<3>
+; CHECK-NEXT: CLONE ir<%gep2>.3 = getelementptr inbounds ir<%b>, vp<[[VP18]]>
+; CHECK-NEXT: CLONE ir<%ld>.3 = load ir<%gep2>.3
+; CHECK-NEXT: EMIT vp<[[VP19:%[0-9]+]]> = insertelement vp<[[VP15]]>, ir<%ld>.3, ir<3>
+; CHECK-NEXT: Successor(s): pred.load.continue
+; CHECK-EMPTY:
+; CHECK-NEXT: pred.load.continue:
+; CHECK-NEXT: WIDEN-PHI vp<[[VP20:%[0-9]+]]> = phi [ vp<[[VP15]]>, pred.load.continue ], [ vp<[[VP19]]>, pred.load.if ]
+; CHECK-NEXT: EMIT vp<%predphi> = select ir<%c>, vp<[[VP20]]>, ir<0>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%p>
+; CHECK-NEXT: WIDEN ir<%sum> = add vp<%predphi>, ir<%y>
+; CHECK-NEXT: CLONE ir<%po> = getelementptr inbounds ir<%out>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%po>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP21:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP21]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %cont
+
+then:
+ %idx = sext i32 %x to i64
+ %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
+ %ld = load i32, ptr %gep2, align 4
+ br label %cont
+
+cont:
+ %v = phi i32 [ %ld, %then ], [ 0, %loop ]
+ %y = load i32, ptr %p, align 4
+ %sum = add i32 %v, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The first load cannot be reused by the stronger aligned second load, but the
+; second one becomes the candidate for the third load, which reuses it.
+define void @cse_after_weaker_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: VPlan for loop in 'cse_after_weaker_align_first'
+; CHECK: VPlan 'Final VPlan for VF={4},UF={1}' {
+; CHECK-NEXT: Live-in ir<1024> = vector-trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector.body
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%a>, vp<%index>
+; CHECK-NEXT: WIDEN ir<%x> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%y> = load ir<%gep>
+; CHECK-NEXT: WIDEN ir<%s1> = add ir<%x>, ir<%y>
+; CHECK-NEXT: WIDEN ir<%sum> = add ir<%s1>, ir<%y>
+; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr inbounds ir<%b>, vp<%index>
+; CHECK-NEXT: WIDEN store ir<%gep.b>, ir<%sum>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<4>
+; CHECK-NEXT: EMIT vp<[[VP1:%[0-9]+]]> = icmp eq vp<%index.next>, ir<1024>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP1]]>
+; CHECK-NEXT: Successor(s): middle.block, vector.body
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %z = load i32, ptr %gep, align 8
+ %s1 = add i32 %x, %y
+ %sum = add i32 %s1, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain"}
+!3 = !{!4}
+!4 = distinct !{!4, !2, !"scope_b"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"float", !7, i64 0}
More information about the llvm-commits
mailing list