[llvm] [VPlan] Extend cse to eliminate redundant widened loads (PR #212543)
Ashutosh Nema via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 07:22:18 PDT 2026
https://github.com/nema-ashutosh updated https://github.com/llvm/llvm-project/pull/212543
>From ec6c8348289f5da624e561ea783e339c459716c3 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Wed, 5 Aug 2026 22:35:25 +0530
Subject: [PATCH 1/7] [VPlan] Precommit tests for widened-load CSE
---
.../AArch64/partial-reduce-fdot-product.ll | 6 +-
.../partial-reduce-sub-epilogue-vec.ll | 24 +-
...e-to-widen-memory-with-wide-ops-chained.ll | 6 +-
...nterleave-to-widen-memory-with-wide-ops.ll | 6 +-
...sform-narrow-interleave-to-widen-memory.ll | 6 +-
.../LoopVectorize/X86/cse-loads-gather.ll | 101 ++++++
.../LoopVectorize/X86/induction-costs.ll | 8 +-
...interleave-to-widen-memory-epilogue-vec.ll | 20 +-
.../LoopVectorize/cse-loads-extra.ll | 286 ++++++++++++++++
.../LoopVectorize/cse-loads-masked.ll | 186 ++++++++++
.../LoopVectorize/cse-loads-tailfold.ll | 49 +++
.../Transforms/LoopVectorize/cse-loads.ll | 324 ++++++++++++++++++
llvm/test/Transforms/LoopVectorize/cse.ll | 165 +++++++++
13 files changed, 1149 insertions(+), 38 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/cse.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
index 59e5b3c08eeb9..204716477e262 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
@@ -1035,9 +1035,9 @@ define float @fadd_fsub_chain_f16_f32(ptr %a, ptr %b, ptr %c) #0 {
; CHECK-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float>
; CHECK-NEXT: [[TMP14:%.*]] = fmul <vscale x 8 x float> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[PARTIAL_REDUCE5:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI1]], <vscale x 8 x float> [[TMP14]])
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <vscale x 8 x half>, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[TMP17:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD7]] to <vscale x 8 x float>
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP17]], [[TMP12]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP8:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD3]] to <vscale x 8 x float>
+; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP8]], [[TMP12]]
; CHECK-NEXT: [[TMP22:%.*]] = fneg <vscale x 8 x float> [[TMP18]]
; CHECK-NEXT: [[PARTIAL_REDUCE9]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE5]], <vscale x 8 x float> [[TMP22]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index c6a7709c7379d..eaf5ba0f105fb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -23,11 +23,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -49,11 +49,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP11]], %[[VEC_EPILOG_PH]] ], [ [[TMP16:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
-; CHECK-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
-; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD4]] to <4 x i32>
; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
; CHECK-EPI-NEXT: [[TMP14:%.*]] = sext <4 x i8> [[WIDE_LOAD5]] to <4 x i32>
-; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
; CHECK-EPI-NEXT: [[TMP16]] = sub <4 x i32> [[VEC_PHI3]], [[TMP15]]
; CHECK-EPI-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX2]], 4
; CHECK-EPI-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT6]], 36
@@ -101,11 +101,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP5]], [[TMP6]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -126,11 +126,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX2:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD4]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP14]], [[TMP13]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI3]], <8 x i32> [[TMP15]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 469b801961cd9..215b8ea4fc90f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -665,10 +665,10 @@ define void @narrow_with_select_and_invariant_cond(ptr noalias %dst, ptr noalias
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP10]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
-; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD1]], splat (i64 -1)
-; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[WIDE_LOAD]], <2 x i64> [[TMP4]]
+; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
+; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD2]], splat (i64 -1)
+; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[TMP4]]
; VF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP10]]
; VF2-NEXT: store <2 x i64> [[TMP5]], ptr [[TMP6]], align 4
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 29849f2c3d008..7a1598a704e06 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -1521,10 +1521,10 @@ define void @test_2xdouble_same_fcmp_predicate(ptr noalias %data) {
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[WIDE_LOAD]], zeroinitializer
-; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD1]]
+; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
+; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[WIDE_LOAD1]], zeroinitializer
+; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD2]]
; VF2-NEXT: store <2 x double> [[TMP3]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 5c9a3eac44fa0..c78b3f19dd05a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -612,12 +612,12 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF2: [[VECTOR_BODY]]:
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
-; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
+; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
; VF2-NEXT: [[TMP1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD]], ptr [[TMP1]], align 8
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP2:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP2]], align 8
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD2]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
new file mode 100644
index 0000000000000..c918d16f56ca1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -mtriple=x86_64-- -mcpu=skylake-avx512 -S | FileCheck %s
+
+; Tests CSE of gather loads.
+
+; Two identical gathers with the same index vector are CSE'd into one.
+define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_gather(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %idx = load i64, ptr %gep.b, align 8
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %idx
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Gathers with different index vectors are not CSE'd.
+define void @dup_gather_diff_index(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_gather_diff_index(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], ptr noalias [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT: [[WIDE_GEP2:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP2]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER3]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ %idx1 = load i64, ptr %gep.b, align 8
+ %gep.c = getelementptr inbounds i64, ptr %c, i64 %iv
+ %idx2 = load i64, ptr %gep.c, align 8
+ %gep.a1 = getelementptr inbounds i32, ptr %a, i64 %idx1
+ %gep.a2 = getelementptr inbounds i32, ptr %a, i64 %idx2
+ %x = load i32, ptr %gep.a1, align 4
+ %y = load i32, ptr %gep.a2, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 165d81b285a19..d5f096d96945f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -287,10 +287,10 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
; CHECK-NEXT: [[NEXT_GEP21:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP15]]
; CHECK-NEXT: [[NEXT_GEP23:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15:![0-9]+]]
-; CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-NEXT: [[WIDE_LOAD24:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15]]
-; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD24]] to <16 x i32>
-; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP19]], [[TMP21]]
+; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15]]
+; CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
+; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP21]], [[TMP19]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP22]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
; CHECK-NEXT: [[TMP24:%.*]] = lshr <16 x i32> [[TMP23]], splat (i32 1)
; CHECK-NEXT: [[TMP25:%.*]] = trunc <16 x i32> [[TMP24]] to <16 x i8>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
index c599c7e3e17e2..df5cff2d21465 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
@@ -201,18 +201,18 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[P]], i64 [[TMP3]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
-; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD7]], [[WIDE_LOAD12]]
; CHECK-NEXT: store <4 x double> [[TMP8]], ptr [[TMP4]], align 8
; CHECK-NEXT: store <4 x double> [[TMP9]], ptr [[TMP5]], align 8
; CHECK-NEXT: store <4 x double> [[TMP10]], ptr [[TMP6]], align 8
@@ -231,9 +231,9 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[INDEX9:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX9]], 2
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[P]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD10]], [[WIDE_LOAD11]]
+; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD11]], [[WIDE_LOAD13]]
; CHECK-NEXT: store <4 x double> [[TMP14]], ptr [[TMP13]], align 8
; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX9]], 1
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT12]], 99
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
new file mode 100644
index 0000000000000..88c1705e9fa22
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Additional load-CSE coverage.
+
+; Three identical loads are CSE'd into one.
+define void @triple_dup_load(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @triple_dup_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %z = load i32, ptr %p, align 4
+ %s1 = add i32 %x, %y
+ %s2 = add i32 %s1, %z
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s2, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Two independent groups of duplicate loads are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @two_dup_groups(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A duplicate load and the duplicate add depending on it are both CSE'd.
+define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @load_then_dup_add(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 7)
+; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP6]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %ax = add i32 %x, 7
+ %ay = add i32 %y, 7
+ %m = mul i32 %ax, %ay
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %m, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Two identical loads with different TBAA are CSE'd, keeping only common metadata.
+define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_diff_tbaa(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[INT_TBAA5:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4, !tbaa !0
+ %y = load i32, ptr %p, align 4, !tbaa !4
+ %sum = add i32 %x, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads separated by a replicate region are not CSE'd.
+define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_across_replicate_region(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[VECTOR_PH:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
+; CHECK: [[VECTOR_BODY1]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY1]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: br label %[[CONT:.*]]
+; CHECK: [[CONT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %cont
+then:
+ %idx = sext i32 %x to i64
+ %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
+ %ld = load i32, ptr %gep2, align 4
+ br label %cont
+cont:
+ %v = phi i32 [ %ld, %then ], [ 0, %loop ]
+ %y = load i32, ptr %p, align 4
+ %sum = add i32 %v, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"float", !2, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
new file mode 100644
index 0000000000000..269e6e9fbbdd1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -force-target-supports-masked-memory-ops -S | FileCheck %s
+
+; Tests CSE of masked widened loads and mixed masked/regular loads.
+
+; Two masked loads from the same address under the same condition are CSE'd.
+define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond) {
+; CHECK-LABEL: define void @two_masked_same_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP5]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: br label %[[LATCH:.*]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+if:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %s = add i32 %x, %y
+ br label %latch
+latch:
+ %m = phi i32 [ 0, %loop ], [ %s, %if ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %m, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Masked loads from the same address under different conditions are not CSE'd.
+define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2) {
+; CHECK-LABEL: define void @two_masked_diff_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C1:%.*]], ptr noalias [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[C1]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C2]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[WIDE_MASKED_LOAD2]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[PREDPHI]], [[PREDPHI3]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP7]], label %[[IF2:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[IF2]]:
+; CHECK-NEXT: br label %[[LATCH2:.*]]
+; CHECK: [[LATCH2]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
+ %v1 = load i32, ptr %gp1, align 4
+ %cmp1 = icmp ne i32 %v1, 0
+ br i1 %cmp1, label %if1, label %latch1
+if1:
+ %x = load i32, ptr %gep.a, align 4
+ br label %latch1
+latch1:
+ %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
+ %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
+ %v2 = load i32, ptr %gp2, align 4
+ %cmp2 = icmp ne i32 %v2, 0
+ br i1 %cmp2, label %if2, label %latch2
+if2:
+ %y = load i32, ptr %gep.a, align 4
+ br label %latch2
+latch2:
+ %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
+ %sum = add i32 %mx, %my
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A duplicated regular load and a duplicated masked load are each CSE'd independently.
+define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias %cond, ptr noalias %out) {
+; CHECK-LABEL: define void @masked_and_regular_mix(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD3]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[PREDPHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP8]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[IF]]:
+; CHECK-NEXT: br label %[[LATCH:.*]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %r1 = load i32, ptr %gep.a, align 4
+ %r2 = load i32, ptr %gep.a, align 4
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+if:
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %m1 = load i32, ptr %gep.b, align 4
+ %m2 = load i32, ptr %gep.b, align 4
+ %ms = add i32 %m1, %m2
+ br label %latch
+latch:
+ %mm = phi i32 [ 0, %loop ], [ %ms, %if ]
+ %rr = add i32 %r1, %r2
+ %sum = add i32 %rr, %mm
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
new file mode 100644
index 0000000000000..9dcc30b758042
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -tail-folding-policy=must-fold-tail -force-tail-folding-style=data \
+; RUN: -force-target-supports-masked-memory-ops -S | FileCheck %s
+
+; Under tail folding, two identical masked loads from the same address are CSE'd.
+define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_duplicate_masked_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[N]])
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP1]], ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
new file mode 100644
index 0000000000000..26b60236165e9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -0,0 +1,324 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Tests CSE of duplicate widened loads.
+
+; Two identical loads from the same address are CSE'd into one.
+define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_duplicate_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+define void @no_cse_across_store(ptr %a) {
+; CHECK-LABEL: define void @no_cse_across_store(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 2)
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %sink = add i32 %y, 2
+ %gep.s = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %sink, ptr %gep.s, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads from different addresses are not CSE'd.
+define void @no_cse_different_address(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @no_cse_different_address(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %y = load i32, ptr %gep.b, align 4
+ %sum = add i32 %x, %y
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %sum, ptr %gep.c, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads of different scalar types are not CSE'd.
+define void @no_cse_different_type(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_different_type(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = zext <4 x i32> [[TMP14]] to <4 x i64>
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i64> [[TMP15]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i64, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i64, ptr %gep, align 8
+ %xe = zext i32 %x to i64
+ %sum = add i64 %xe, %y
+ %gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %sum, ptr %gep.b, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Loads from the same address with different alignment are not CSE'd.
+define void @no_cse_different_align(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_different_align(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Three identical loads from the same address collapse to a single load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_three_duplicate_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A store proven not to alias the loaded address (via scoped-noalias metadata)
+; does not prevent CSE of the second load.
+define void @cse_across_noalias_store(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @cse_across_noalias_store(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4, !alias.scope !3, !noalias !0
+ %y = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4, !alias.scope !3, !noalias !0
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"scope_a"}
+!2 = distinct !{!2, !"domain"}
+!3 = !{!4}
+!4 = distinct !{!4, !2, !"scope_b"}
diff --git a/llvm/test/Transforms/LoopVectorize/cse.ll b/llvm/test/Transforms/LoopVectorize/cse.ll
new file mode 100644
index 0000000000000..47ab7ee60dc4a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/cse.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+; Tests CSE of duplicate non-memory recipes (GEPs, index arithmetic, casts).
+
+; Duplicate address GEPs for a load and a store are CSE'd into one.
+define void @dup_gep(ptr noalias %a) {
+; CHECK-LABEL: define void @dup_gep(
+; CHECK-SAME: ptr noalias [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %v = load i32, ptr %p, align 4
+ %q = getelementptr inbounds i32, ptr %a, i64 %iv
+ %inc = add i32 %v, 1
+ store i32 %inc, ptr %q, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Duplicate index arithmetic used for two arrays is CSE'd.
+define void @dup_index(ptr noalias %a, ptr noalias %b, i64 %o) {
+; CHECK-LABEL: define void @dup_index(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[O:%.*]]) {
+; CHECK-NEXT: [[VECTOR_SCEVCHECK:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[O]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %i1 = add i64 %iv, %o
+ %i2 = add i64 %iv, %o
+ %pa = getelementptr inbounds i32, ptr %a, i64 %i1
+ %pb = getelementptr inbounds i32, ptr %b, i64 %i2
+ %vb = load i32, ptr %pb, align 4
+ store i32 %vb, ptr %pa, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; A duplicated loop-invariant cast is CSE'd.
+define void @dup_invariant_cast(ptr noalias %a, i16 %k) {
+; CHECK-LABEL: define void @dup_invariant_cast(
+; CHECK-SAME: ptr noalias [[A:%.*]], i16 [[K:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %z1 = zext i16 %k to i32
+ %z2 = zext i16 %k to i32
+ %sum = add i32 %z1, %z2
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %sum, ptr %p, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Casts of the same operand to different result types are not CSE'd.
+define void @no_cse_different_cast_type(ptr noalias %a, ptr noalias %b, i16 %k) {
+; CHECK-LABEL: define void @no_cse_different_cast_type(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i16 [[K:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i64>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP3]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %z32 = zext i16 %k to i32
+ %z64 = zext i16 %k to i64
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ store i32 %z32, ptr %pa, align 4
+ %pb = getelementptr inbounds i64, ptr %b, i64 %iv
+ store i64 %z64, ptr %pb, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
>From 0f92a607079f30148ccb1303820b9ed470cb2f81 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Wed, 5 Aug 2026 22:36:58 +0530
Subject: [PATCH 2/7] [VPlan] Extend cse to eliminate redundant widened loads
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 58 ++++++++++++++++++-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 1 +
.../AArch64/partial-reduce-fdot-product.ll | 4 +-
.../partial-reduce-sub-epilogue-vec.ll | 24 +++-----
...e-to-widen-memory-with-wide-ops-chained.ll | 3 +-
...nterleave-to-widen-memory-with-wide-ops.ll | 9 +--
...sform-narrow-interleave-to-widen-memory.ll | 3 +-
.../LoopVectorize/X86/cse-loads-gather.ll | 3 +-
.../LoopVectorize/X86/induction-costs.ll | 14 ++---
...interleave-to-widen-memory-epilogue-vec.ll | 23 +++-----
...sform-narrow-interleave-to-widen-memory.ll | 3 +-
.../LoopVectorize/cse-loads-extra.ll | 25 +++-----
.../LoopVectorize/cse-loads-masked.ll | 9 +--
.../LoopVectorize/cse-loads-tailfold.ll | 3 +-
.../Transforms/LoopVectorize/cse-loads.ll | 12 ++--
15 files changed, 104 insertions(+), 90 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9990fd142df02..517e47ce3fc4b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -274,6 +274,28 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
return true;
}
+/// Return true if no memory-writing recipe between \p From and \p To may alias
+/// \p MemLoc. Unlike canHoistOrSinkWithNoAliasCheck, this only scans the
+/// recipes strictly between \p From and \p To (which must be in the same
+/// block), as needed when common-subexpression-eliminating two loads: writes
+/// before \p From or after \p To cannot affect the reused value.
+static bool canCSEWithNoAliasCheck(const MemoryLocation &MemLoc,
+ VPRecipeBase *From, VPRecipeBase *To) {
+ if (From->getParent() != To->getParent())
+ return false;
+
+ for (VPRecipeBase &R :
+ make_range(std::next(From->getIterator()), To->getIterator())) {
+ if (!R.mayWriteToMemory())
+ continue;
+ auto Loc = vputils::getMemoryLocation(R);
+ if (!Loc ||
+ ScopedNoAliasAAResult::alias(*Loc, MemLoc) != AliasResult::NoAlias)
+ return false;
+ }
+ return true;
+}
+
/// Get the value type of the replicate load or store. \p IsLoad indicates
/// whether it is a load.
static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
@@ -2251,8 +2273,12 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // During CSE, we can only handle non-memory recipes, as memory can alias.
- return !Def->mayReadOrWriteMemory();
+ // Widened loads only read memory; the cse() driver guards their reuse with
+ // an aliasing check. Reject anything that may write, and any other recipe
+ // that reads memory.
+ if (Def->mayWriteToMemory())
+ return false;
+ return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
}
/// Hash the underlying data of \p Def.
@@ -2266,6 +2292,11 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, RFlags->getPredicate());
if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Def))
return hash_combine(Result, SIVSteps->getInductionOpcode());
+ // Alignment and the consecutive/masked flags are stored separately from
+ // the operands, so fold them in here.
+ if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
+ return hash_combine(Result, Load->getAlign().value(),
+ Load->isConsecutive(), Load->isMasked());
return Result;
}
@@ -2290,6 +2321,15 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
if (LSIV->getInductionOpcode() !=
cast<VPScalarIVStepsRecipe>(R)->getInductionOpcode())
return false;
+ // Alignment and the consecutive/masked flags are stored separately from
+ // the operands, so compare them explicitly.
+ if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L)) {
+ auto *RL = cast<VPWidenLoadRecipe>(R);
+ if (LL->getAlign() != RL->getAlign() ||
+ LL->isConsecutive() != RL->isConsecutive() ||
+ LL->isMasked() != RL->isMasked())
+ return false;
+ }
// Phi recipes can only be equal if they are in the same VPBB, as they
// implicitly depend on their predecessors.
if (isa<VPWidenPHIRecipe>(L) && L->getParent() != R->getParent())
@@ -2308,7 +2348,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
};
} // end anonymous namespace
-/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
+/// Perform a common-subexpression-elimination of single-def recipes on the \p
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
VPDominatorTree VPDT(Plan);
@@ -2325,6 +2365,18 @@ void VPlanTransforms::cse(VPlan &Plan) {
// V must dominate Def for a valid replacement.
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
+ // Reuse an earlier widened load only if no aliasing write lies between
+ // the two loads.
+ if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def)) {
+ auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
+ std::optional<MemoryLocation> Loc = vputils::getMemoryLocation(*Load);
+ if (!Loc || !canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
+ continue;
+ // Keep only metadata common to both loads on the survivor.
+ EarlierLoad->intersect(*Load);
+ Load->replaceAllUsesWith(EarlierLoad);
+ continue;
+ }
// Only keep flags present on both V and Def.
if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e8023468e98ef..79cfe20e171ac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -368,6 +368,7 @@ unsigned vputils::getOpcode(const VPValue *V) {
.Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
VPReplicateRecipe, VPWidenPHIRecipe>(
[](auto *I) { return I->getOpcode(); })
+ .Case<VPWidenLoadRecipe>([](auto *) { return Instruction::Load; })
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
[](auto *I) {
// For recipes that do not directly map to LLVM IR instructions,
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
index 204716477e262..f9e0e65bf3907 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-fdot-product.ll
@@ -1035,9 +1035,7 @@ define float @fadd_fsub_chain_f16_f32(ptr %a, ptr %b, ptr %c) #0 {
; CHECK-NEXT: [[TMP13:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD2]] to <vscale x 8 x float>
; CHECK-NEXT: [[TMP14:%.*]] = fmul <vscale x 8 x float> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[PARTIAL_REDUCE5:%.*]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[VEC_PHI1]], <vscale x 8 x float> [[TMP14]])
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = fpext <vscale x 8 x half> [[WIDE_LOAD3]] to <vscale x 8 x float>
-; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP8]], [[TMP12]]
+; CHECK-NEXT: [[TMP18:%.*]] = fmul <vscale x 8 x float> [[TMP13]], [[TMP12]]
; CHECK-NEXT: [[TMP22:%.*]] = fneg <vscale x 8 x float> [[TMP18]]
; CHECK-NEXT: [[PARTIAL_REDUCE9]] = call reassoc contract <vscale x 4 x float> @llvm.vector.partial.reduce.fadd.nxv4f32.nxv8f32(<vscale x 4 x float> [[PARTIAL_REDUCE5]], <vscale x 8 x float> [[TMP22]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
index eaf5ba0f105fb..58b7424597fe4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub-epilogue-vec.ll
@@ -24,10 +24,8 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
; CHECK-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
-; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP6]]
; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -51,9 +49,7 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
; CHECK-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
; CHECK-EPI-NEXT: [[TMP14:%.*]] = sext <4 x i8> [[WIDE_LOAD5]] to <4 x i32>
-; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP12]], align 4
-; CHECK-EPI-NEXT: [[TMP13:%.*]] = sext <4 x i8> [[WIDE_LOAD6]] to <4 x i32>
-; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP13]]
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = mul <4 x i32> [[TMP14]], [[TMP14]]
; CHECK-EPI-NEXT: [[TMP16]] = sub <4 x i32> [[VEC_PHI3]], [[TMP15]]
; CHECK-EPI-NEXT: [[INDEX_NEXT6]] = add nuw i32 [[INDEX2]], 4
; CHECK-EPI-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT6]], 36
@@ -63,11 +59,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 36, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-EPI-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX5:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
; CHECK-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-EPI: [[LOOP]]:
; CHECK-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX7]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX5]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
; CHECK-EPI-NEXT: [[SRC1_LOAD:%.*]] = load i8, ptr [[SRC1_GEP]], align 4
; CHECK-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = sext i8 [[SRC1_LOAD]] to i32
@@ -102,10 +98,8 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = mul <vscale x 16 x i32> [[TMP6]], [[TMP6]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -127,10 +121,8 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[INDEX2]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD5:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
-; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP12]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = sext <8 x i8> [[WIDE_LOAD5]] to <8 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = sext <8 x i8> [[WIDE_LOAD6]] to <8 x i32>
-; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP14]], [[TMP13]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = mul <8 x i32> [[TMP14]], [[TMP14]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE6]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI3]], <8 x i32> [[TMP15]])
; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT7]] = add nuw i32 [[INDEX2]], 8
; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT7]], 32
@@ -141,11 +133,11 @@ define i32 @sub_reduction(i32 %startval, ptr %src1, ptr %src2) #0 {
; CHECK-PARTIAL-RED-EPI-NEXT: br i1 false, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX8:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i32 [ [[TMP18]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[STARTVAL]], %[[ITER_CHECK]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[LOOP:.*]]
; CHECK-PARTIAL-RED-EPI: [[LOOP]]:
; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX8]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX6]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_GEP:%.*]] = getelementptr i8, ptr [[SRC1]], i32 [[IV]]
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD:%.*]] = load i8, ptr [[SRC1_GEP]], align 4
; CHECK-PARTIAL-RED-EPI-NEXT: [[SRC1_LOAD_EXT:%.*]] = sext i8 [[SRC1_LOAD]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 215b8ea4fc90f..fa3b15f481996 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -666,8 +666,7 @@ define void @narrow_with_select_and_invariant_cond(ptr noalias %dst, ptr noalias
; VF2-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP10]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
-; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 4
-; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD2]], splat (i64 -1)
+; VF2-NEXT: [[TMP4:%.*]] = xor <2 x i64> [[WIDE_LOAD1]], splat (i64 -1)
; VF2-NEXT: [[TMP5:%.*]] = select <2 x i1> [[TMP1]], <2 x i64> [[WIDE_LOAD1]], <2 x i64> [[TMP4]]
; VF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP10]]
; VF2-NEXT: store <2 x i64> [[TMP5]], ptr [[TMP6]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
index 7a1598a704e06..a90e82b826775 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll
@@ -576,8 +576,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP6:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
-; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF2-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD]]
; VF2-NEXT: [[TMP14:%.*]] = or disjoint i64 [[TMP6]], 1
; VF2-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP14]]
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP15]], align 8
@@ -607,8 +606,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
; VF4-NEXT: [[TMP10:%.*]] = shl nsw i64 [[INDEX]], 1
; VF4-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP10]]
-; VF4-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; VF4-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD]]
; VF4-NEXT: [[TMP24:%.*]] = or disjoint i64 [[TMP10]], 1
; VF4-NEXT: [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP24]]
; VF4-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP25]], align 8
@@ -1522,9 +1520,8 @@ define void @test_2xdouble_same_fcmp_predicate(ptr noalias %data) {
; VF2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
-; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x double> [[WIDE_LOAD1]], zeroinitializer
-; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD2]]
+; VF2-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> [[WIDE_LOAD1]]
; VF2-NEXT: store <2 x double> [[TMP3]], ptr [[TMP1]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index c78b3f19dd05a..34c730b32f092 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -613,11 +613,10 @@ define void @multiple_store_groups_storing_same_load_group(ptr noalias %A, ptr n
; VF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF2-NEXT: [[TMP0:%.*]] = getelementptr { double, double }, ptr [[A]], i64 [[INDEX]]
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
-; VF2-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x double>, ptr [[TMP0]], align 8
; VF2-NEXT: [[TMP1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP1]], align 8
; VF2-NEXT: [[TMP2:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
-; VF2-NEXT: store <2 x double> [[WIDE_LOAD2]], ptr [[TMP2]], align 8
+; VF2-NEXT: store <2 x double> [[WIDE_LOAD1]], ptr [[TMP2]], align 8
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
; VF2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
; VF2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
index c918d16f56ca1..9b6aa7e0850ed 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
@@ -18,8 +18,7 @@ define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <4 x i64> [[WIDE_LOAD]]
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index d5f096d96945f..e069fc409f08f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -288,9 +288,7 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
; CHECK-NEXT: [[NEXT_GEP23:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15:![0-9]+]]
; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-NEXT: [[WIDE_LOAD18:%.*]] = load <16 x i8>, ptr [[NEXT_GEP23]], align 1, !alias.scope [[META15]]
-; CHECK-NEXT: [[TMP19:%.*]] = zext <16 x i8> [[WIDE_LOAD18]] to <16 x i32>
-; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP21]], [[TMP19]]
+; CHECK-NEXT: [[TMP22]] = add <16 x i32> [[TMP21]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x i32> [[VECTOR_RECUR]], <16 x i32> [[TMP22]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
; CHECK-NEXT: [[TMP24:%.*]] = lshr <16 x i32> [[TMP23]], splat (i32 1)
; CHECK-NEXT: [[TMP25:%.*]] = trunc <16 x i32> [[TMP24]] to <16 x i8>
@@ -335,15 +333,15 @@ define void @multiple_pointer_ivs_with_scalar_uses_only(ptr %A, ptr %B) #0 {
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ -12, %[[MIDDLE_BLOCK]] ], [ 100, %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 2048, %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL19:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL20:%.*]] = phi ptr [ [[IND_END5]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL18:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL19:%.*]] = phi ptr [ [[IND_END5]], %[[MIDDLE_BLOCK]] ], [ [[B]], %[[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[DEC:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[ADD38:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[BC_RESUME_VAL19]], %[[SCALAR_PH]] ], [ [[OUTPTR_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL20]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR36:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[BC_RESUME_VAL20]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR33:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[BC_RESUME_VAL18]], %[[SCALAR_PH]] ], [ [[OUTPTR_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL19]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR36:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[BC_RESUME_VAL19]], %[[SCALAR_PH]] ], [ [[INCDEC_PTR33:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[INCDEC_PTR33]] = getelementptr i8, ptr [[PTR_IV_3]], i64 1
; CHECK-NEXT: [[TMP43:%.*]] = load i8, ptr [[PTR_IV_3]], align 1
; CHECK-NEXT: [[CONV34:%.*]] = zext i8 [[TMP43]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
index df5cff2d21465..f6838b9b82cc3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory-epilogue-vec.ll
@@ -205,14 +205,10 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
-; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x double>, ptr [[TMP4]], align 8
-; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x double>, ptr [[TMP5]], align 8
-; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
-; CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x double>, ptr [[TMP7]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
-; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD5]], [[WIDE_LOAD9]]
-; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD6]], [[WIDE_LOAD10]]
-; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD7]], [[WIDE_LOAD12]]
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <4 x double> [[WIDE_LOAD4]], [[WIDE_LOAD4]]
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[WIDE_LOAD5]], [[WIDE_LOAD5]]
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_LOAD6]], [[WIDE_LOAD6]]
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_LOAD7]], [[WIDE_LOAD7]]
; CHECK-NEXT: store <4 x double> [[TMP8]], ptr [[TMP4]], align 8
; CHECK-NEXT: store <4 x double> [[TMP9]], ptr [[TMP5]], align 8
; CHECK-NEXT: store <4 x double> [[TMP10]], ptr [[TMP6]], align 8
@@ -232,8 +228,7 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX9]], 2
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[P]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[WIDE_LOAD13:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD11]], [[WIDE_LOAD13]]
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <4 x double> [[WIDE_LOAD11]], [[WIDE_LOAD11]]
; CHECK-NEXT: store <4 x double> [[TMP14]], ptr [[TMP13]], align 8
; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX9]], 1
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT12]], 99
@@ -241,12 +236,12 @@ define void @test_non_unit_step_resume_values(ptr %p) #1 {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi i64 [ 396, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 384, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL14:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL8:%.*]] = phi i64 [ 396, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 384, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL9:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 3, %[[VEC_EPILOG_ITER_CHECK]] ], [ 99, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ]
-; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL14]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL8]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT: [[CNT:%.*]] = phi i64 [ [[CNT_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL9]], %[[VEC_EPILOG_SCALAR_PH]] ]
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV]]
; CHECK-NEXT: [[V0:%.*]] = load double, ptr [[GEP0]], align 8
; CHECK-NEXT: [[R0:%.*]] = fadd double [[V0]], [[V0]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
index 3b1219be49ab6..9a967931c0bb5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
@@ -472,8 +472,7 @@ define void @test_2xi64_different_loads_feeding_fmul(ptr noalias %data, ptr noal
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[GEP_SRC_0]], align 8
; CHECK-NEXT: [[TMP5:%.*]] = shl nsw i64 [[IV]], 1
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP5]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[GEP_SRC_0]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[TMP5]], 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP7]]
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP8]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
index 88c1705e9fa22..8edb8ee72070d 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
@@ -15,10 +15,8 @@ define void @triple_dup_load(ptr noalias %a, ptr noalias %out) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -60,12 +58,10 @@ define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
@@ -111,10 +107,8 @@ define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
-; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[WIDE_LOAD1]], splat (i32 7)
-; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP6]]
+; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -155,14 +149,13 @@ define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[INT_TBAA5:![0-9]+]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !tbaa [[FLOAT_TBAA9:![0-9]+]]
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
@@ -246,7 +239,7 @@ define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, pt
; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[THEN]]:
; CHECK-NEXT: br label %[[CONT:.*]]
; CHECK: [[CONT]]:
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
index 269e6e9fbbdd1..b7be1727fd989 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
@@ -19,8 +19,7 @@ define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c
; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP2]], <4 x i1> [[TMP1]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
@@ -134,16 +133,14 @@ define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP2]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD3]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP4]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP5]], [[PREDPHI]]
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
index 9dcc30b758042..4529264346f5f 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -19,8 +19,7 @@ define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[N]])
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
-; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP0]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP1]], ptr align 4 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]])
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index 26b60236165e9..68a8e784b0c5e 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -15,8 +15,7 @@ define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -240,10 +239,8 @@ define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -288,8 +285,7 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
>From d418d1cb27e11c7967bd325f2d419aef82535111 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Thu, 6 Aug 2026 15:35:06 +0530
Subject: [PATCH 3/7] [VPlan] Address review comments for widened-load CSE
Adds getOpcode() to VPWidenLoadRecipe and a negative aliasing test.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 ++
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 3 +-
.../Transforms/LoopVectorize/cse-loads.ll | 44 +++++++++++++++++++
4 files changed, 53 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4cb214ac0a257..f15310a6de382 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3808,6 +3808,9 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
+ /// Returns the opcode of the widened load.
+ unsigned getOpcode() const { return Instruction::Load; }
+
/// Generate a wide load or gather.
void execute(VPTransformState &State) override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 517e47ce3fc4b..e9096f9aa4ab9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2273,7 +2273,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads only read memory; the cse() driver guards their reuse with
+ // Widened loads only read memory; the cse() routine guards their reuse with
// an aliasing check. Reject anything that may write, and any other recipe
// that reads memory.
if (Def->mayWriteToMemory())
@@ -2367,15 +2367,13 @@ void VPlanTransforms::cse(VPlan &Plan) {
continue;
// Reuse an earlier widened load only if no aliasing write lies between
// the two loads.
- if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def)) {
- auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
- std::optional<MemoryLocation> Loc = vputils::getMemoryLocation(*Load);
- if (!Loc || !canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
+ if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
+ auto *Load = cast<VPWidenLoadRecipe>(Def);
+ auto Loc = vputils::getMemoryLocation(*EarlierLoad);
+ if (!canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
continue;
// Keep only metadata common to both loads on the survivor.
EarlierLoad->intersect(*Load);
- Load->replaceAllUsesWith(EarlierLoad);
- continue;
}
// Only keep flags present on both V and Def.
if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 79cfe20e171ac..ea905cbf24797 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -366,9 +366,8 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
unsigned vputils::getOpcode(const VPValue *V) {
return TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
- VPReplicateRecipe, VPWidenPHIRecipe>(
+ VPReplicateRecipe, VPWidenPHIRecipe, VPWidenLoadRecipe>(
[](auto *I) { return I->getOpcode(); })
- .Case<VPWidenLoadRecipe>([](auto *) { return Instruction::Load; })
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
[](auto *I) {
// For recipes that do not directly map to LLVM IR instructions,
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index 68a8e784b0c5e..af9c67fd9e5d9 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -313,6 +313,50 @@ exit:
ret void
}
+; A store without alias metadata cannot be proven not to alias the loaded
+; address, so the second load is not CSE'd.
+define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_across_store_without_scopes(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4
+ %y = load i32, ptr %gep.a, align 4, !alias.scope !0, !noalias !3
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
!0 = !{!1}
!1 = distinct !{!1, !2, !"scope_a"}
!2 = distinct !{!2, !"domain"}
>From 150af299bd452c8cb3d8deb312bd7e494571a6ca Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 11 Aug 2026 16:24:59 +0530
Subject: [PATCH 4/7] [VPlan] Allow widened-load CSE across differing
alignments
Reuse widened loads only with equal-or-better alignment.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 35 +-
.../LoopVectorize/X86/cse-loads-gather.ll | 4 +
.../LoopVectorize/cse-loads-extra.ll | 279 ----------
.../LoopVectorize/cse-loads-masked.ll | 41 +-
.../LoopVectorize/cse-loads-tailfold.ll | 2 +
.../Transforms/LoopVectorize/cse-loads.ll | 492 ++++++++++++++++--
llvm/test/Transforms/LoopVectorize/cse.ll | 165 ------
7 files changed, 499 insertions(+), 519 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/cse.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e9096f9aa4ab9..6ff2c605e0c09 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2273,9 +2273,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads only read memory; the cse() routine guards their reuse with
- // an aliasing check. Reject anything that may write, and any other recipe
- // that reads memory.
+ // Widened loads are handled, as cse() guards their reuse with an aliasing
+ // check. Any other memory access is rejected.
if (Def->mayWriteToMemory())
return false;
return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
@@ -2292,11 +2291,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, RFlags->getPredicate());
if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Def))
return hash_combine(Result, SIVSteps->getInductionOpcode());
- // Alignment and the consecutive/masked flags are stored separately from
- // the operands, so fold them in here.
+ // Fold in the separately stored consecutive/masked flags. Alignment is left
+ // out of the key and checked by cse().
if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
- return hash_combine(Result, Load->getAlign().value(),
- Load->isConsecutive(), Load->isMasked());
+ return hash_combine(Result, Load->isConsecutive(), Load->isMasked());
return Result;
}
@@ -2321,12 +2319,11 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
if (LSIV->getInductionOpcode() !=
cast<VPScalarIVStepsRecipe>(R)->getInductionOpcode())
return false;
- // Alignment and the consecutive/masked flags are stored separately from
- // the operands, so compare them explicitly.
+ // Compare the separately stored consecutive/masked flags. Alignment is left
+ // out and checked by cse().
if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L)) {
auto *RL = cast<VPWidenLoadRecipe>(R);
- if (LL->getAlign() != RL->getAlign() ||
- LL->isConsecutive() != RL->isConsecutive() ||
+ if (LL->isConsecutive() != RL->isConsecutive() ||
LL->isMasked() != RL->isMasked())
return false;
}
@@ -2365,13 +2362,21 @@ void VPlanTransforms::cse(VPlan &Plan) {
// V must dominate Def for a valid replacement.
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
- // Reuse an earlier widened load only if no aliasing write lies between
- // the two loads.
if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
auto *Load = cast<VPWidenLoadRecipe>(Def);
- auto Loc = vputils::getMemoryLocation(*EarlierLoad);
- if (!canCSEWithNoAliasCheck(*Loc, EarlierLoad, Load))
+ // Reuse EarlierLoad only if it is at least as aligned as Load and no
+ // aliasing write lies between the two loads.
+ bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign();
+ if (CanReuse) {
+ auto Loc = vputils::getMemoryLocation(*EarlierLoad);
+ CanReuse = canCSEWithNoAliasCheck(*Loc, EarlierLoad, Def);
+ }
+ if (!CanReuse) {
+ // Record Load as the candidate for subsequent loads, as it may be
+ // reusable where EarlierLoad is not.
+ CSEMap[Def] = Def;
continue;
+ }
// Keep only metadata common to both loads on the survivor.
EarlierLoad->intersect(*Load);
}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
index 9b6aa7e0850ed..832b5709bf865 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cse-loads-gather.ll
@@ -31,6 +31,7 @@ define void @dup_gather(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
@@ -44,6 +45,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -79,6 +81,7 @@ define void @dup_gather_diff_index(ptr noalias %a, ptr noalias %b, ptr noalias %
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.b = getelementptr inbounds i64, ptr %b, i64 %iv
@@ -95,6 +98,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
deleted file mode 100644
index 8edb8ee72070d..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-extra.ll
+++ /dev/null
@@ -1,279 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
-
-; Additional load-CSE coverage.
-
-; Three identical loads are CSE'd into one.
-define void @triple_dup_load(ptr noalias %a, ptr noalias %out) {
-; CHECK-LABEL: define void @triple_dup_load(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4
- %y = load i32, ptr %p, align 4
- %z = load i32, ptr %p, align 4
- %s1 = add i32 %x, %y
- %s2 = add i32 %s1, %z
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %s2, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Two independent groups of duplicate loads are each CSE'd.
-define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
-; CHECK-LABEL: define void @two_dup_groups(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
-; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %pa = getelementptr inbounds i32, ptr %a, i64 %iv
- %a1 = load i32, ptr %pa, align 4
- %a2 = load i32, ptr %pa, align 4
- %pb = getelementptr inbounds i32, ptr %b, i64 %iv
- %b1 = load i32, ptr %pb, align 4
- %b2 = load i32, ptr %pb, align 4
- %sa = add i32 %a1, %a2
- %sb = add i32 %b1, %b2
- %s = add i32 %sa, %sb
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %s, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; A duplicate load and the duplicate add depending on it are both CSE'd.
-define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
-; CHECK-LABEL: define void @load_then_dup_add(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
-; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> [[TMP2]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4
- %y = load i32, ptr %p, align 4
- %ax = add i32 %x, 7
- %ay = add i32 %y, 7
- %m = mul i32 %ax, %ay
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %m, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Two identical loads with different TBAA are CSE'd, keeping only common metadata.
-define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
-; CHECK-LABEL: define void @dup_load_diff_tbaa(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4, !tbaa !0
- %y = load i32, ptr %p, align 4, !tbaa !4
- %sum = add i32 %x, %y
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %sum, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Loads separated by a replicate region are not CSE'd.
-define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
-; CHECK-LABEL: define void @dup_load_across_replicate_region(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
-; CHECK-NEXT: [[VECTOR_PH:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY1:.*]]
-; CHECK: [[VECTOR_BODY1]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY1]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
-; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
-; CHECK: [[PRED_LOAD_IF1]]:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
-; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
-; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
-; CHECK: [[PRED_LOAD_IF3]]:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
-; CHECK: [[PRED_LOAD_CONTINUE4]]:
-; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
-; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_IF5]]:
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
-; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
-; CHECK: [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer
-; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD7]]
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP29]], label %[[THEN:.*]], label %[[VECTOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: br label %[[CONT:.*]]
-; CHECK: [[CONT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %p, align 4
- %c = icmp sgt i32 %x, 0
- br i1 %c, label %then, label %cont
-then:
- %idx = sext i32 %x to i64
- %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
- %ld = load i32, ptr %gep2, align 4
- br label %cont
-cont:
- %v = phi i32 [ %ld, %then ], [ 0, %loop ]
- %y = load i32, ptr %p, align 4
- %sum = add i32 %v, %y
- %po = getelementptr inbounds i32, ptr %out, i64 %iv
- store i32 %sum, ptr %po, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-!0 = !{!1, !1, i64 0}
-!1 = !{!"int", !2, i64 0}
-!2 = !{!"omnipotent char", !3, i64 0}
-!3 = !{!"Simple C/C++ TBAA"}
-!4 = !{!5, !5, i64 0}
-!5 = !{!"float", !2, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
index b7be1727fd989..f09de9cf72073 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-masked.ll
@@ -25,26 +25,29 @@ define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c
; CHECK-NEXT: store <4 x i32> [[PREDPHI]], ptr [[TMP4]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP5]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: br label %[[LATCH:.*]]
-; CHECK: [[LATCH]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
%gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
%c = load i32, ptr %gep.c, align 4
%cmp = icmp ne i32 %c, 0
br i1 %cmp, label %if, label %latch
+
if:
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
%x = load i32, ptr %gep.a, align 4
%y = load i32, ptr %gep.a, align 4
%s = add i32 %x, %y
br label %latch
+
latch:
%m = phi i32 [ 0, %loop ], [ %s, %if ]
%gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
@@ -52,6 +55,7 @@ latch:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -82,14 +86,15 @@ define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP7]], label %[[IF2:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[IF2]]:
-; CHECK-NEXT: br label %[[LATCH2:.*]]
-; CHECK: [[LATCH2]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -97,18 +102,22 @@ loop:
%v1 = load i32, ptr %gp1, align 4
%cmp1 = icmp ne i32 %v1, 0
br i1 %cmp1, label %if1, label %latch1
+
if1:
%x = load i32, ptr %gep.a, align 4
br label %latch1
+
latch1:
%mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
%gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
%v2 = load i32, ptr %gp2, align 4
%cmp2 = icmp ne i32 %v2, 0
br i1 %cmp2, label %if2, label %latch2
+
if2:
%y = load i32, ptr %gep.a, align 4
br label %latch2
+
latch2:
%my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
%sum = add i32 %mx, %my
@@ -117,11 +126,13 @@ latch2:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-; A duplicated regular load and a duplicated masked load are each CSE'd independently.
+; A duplicated regular load and a duplicated masked load are each CSE'd
+; independently.
define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias %cond, ptr noalias %out) {
; CHECK-LABEL: define void @masked_and_regular_mix(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], ptr noalias [[OUT:%.*]]) {
@@ -146,14 +157,15 @@ define void @masked_and_regular_mix(ptr noalias %a, ptr noalias %b, ptr noalias
; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP8]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[IF]]:
-; CHECK-NEXT: br label %[[LATCH:.*]]
-; CHECK: [[LATCH]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -163,12 +175,14 @@ loop:
%c = load i32, ptr %gep.c, align 4
%cmp = icmp ne i32 %c, 0
br i1 %cmp, label %if, label %latch
+
if:
%gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
%m1 = load i32, ptr %gep.b, align 4
%m2 = load i32, ptr %gep.b, align 4
%ms = add i32 %m1, %m2
br label %latch
+
latch:
%mm = phi i32 [ 0, %loop ], [ %ms, %if ]
%rr = add i32 %r1, %r2
@@ -178,6 +192,7 @@ latch:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
index 4529264346f5f..abc34bd18910c 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads-tailfold.ll
@@ -32,6 +32,7 @@ define void @cse_duplicate_masked_load(ptr noalias %a, ptr noalias %b, i64 %n) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -43,6 +44,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, %n
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index af9c67fd9e5d9..151df87401fc7 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -20,14 +20,15 @@ define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -39,6 +40,148 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three identical loads from the same address collapse to a single load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_three_duplicate_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two independent groups of duplicate loads are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @two_dup_groups(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A duplicate load and the duplicate add depending on it are both CSE'd.
+define void @load_then_dup_add(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @load_then_dup_add(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 7)
+; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %y = load i32, ptr %p, align 4
+ %ax = add i32 %x, 7
+ %ay = add i32 %y, 7
+ %m = mul i32 %ax, %ay
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %m, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -62,14 +205,15 @@ define void @no_cse_across_store(ptr %a) {
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP0]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -83,6 +227,56 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The store clobbers the first load, but the two loads after it are duplicates
+; of each other and are still CSE'd.
+define void @cse_duplicate_loads_after_store(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_duplicate_loads_after_store(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %z = load i32, ptr %gep, align 4
+ %sum = add i32 %y, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -106,14 +300,15 @@ define void @no_cse_different_address(ptr noalias %a, ptr noalias %b, ptr noalia
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -126,6 +321,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -162,14 +358,15 @@ define void @no_cse_different_type(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep = getelementptr inbounds i64, ptr %a, i64 %iv
@@ -182,13 +379,58 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-; Loads from the same address with different alignment are not CSE'd.
-define void @no_cse_different_align(ptr noalias %a, ptr noalias %b) {
-; CHECK-LABEL: define void @no_cse_different_align(
+; The earlier load has the stronger alignment, so the later load reuses it and
+; no alignment information is lost.
+define void @cse_stronger_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_stronger_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i32, ptr %gep, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The earlier load is weaker aligned, so reusing it would drop the stronger
+; alignment of the second load and CSE is skipped.
+define void @no_cse_weaker_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @no_cse_weaker_align_first(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
@@ -204,14 +446,15 @@ define void @no_cse_different_align(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -223,14 +466,16 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
-; Three identical loads from the same address collapse to a single load.
-define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
-; CHECK-LABEL: define void @cse_three_duplicate_loads(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; Two identical loads with different TBAA are CSE'd, keeping only common
+; metadata.
+define void @dup_load_diff_tbaa(ptr noalias %a, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_diff_tbaa(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[OUT:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
@@ -240,12 +485,11 @@ define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -253,19 +497,19 @@ define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
- %x = load i32, ptr %gep.a, align 4
- %y = load i32, ptr %gep.a, align 4
- %z = load i32, ptr %gep.a, align 4
- %t = add i32 %x, %y
- %sum = add i32 %t, %z
- %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
- store i32 %sum, ptr %gep.b, align 4
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4, !tbaa !5
+ %y = load i32, ptr %p, align 4, !tbaa !9
+ %sum = add i32 %x, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -282,14 +526,14 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8:![0-9]+]], !noalias [[META11:![0-9]+]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META16:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
-; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META11]], !noalias [[META8]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -297,6 +541,7 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -309,6 +554,7 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -325,15 +571,15 @@ define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b)
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13]], !noalias [[META16]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META8]], !noalias [[META11]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13]], !noalias [[META16]]
; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
@@ -341,6 +587,7 @@ define void @no_cse_across_store_without_scopes(ptr noalias %a, ptr noalias %b)
;
entry:
br label %loop
+
loop:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
%gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
@@ -353,6 +600,151 @@ loop:
%iv.next = add i64 %iv, 1
%ec = icmp eq i64 %iv.next, 1024
br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Loads separated by a replicate region are not CSE'd.
+define void @dup_load_across_replicate_region(ptr noalias %a, ptr noalias %b, ptr noalias %out) {
+; CHECK-LABEL: define void @dup_load_across_replicate_region(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-NEXT: br i1 [[TMP15]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK: [[PRED_LOAD_IF3]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK: [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-NEXT: br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_IF5]]:
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK: [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD7]]
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP27]], ptr [[TMP28]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %cont ]
+ %p = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %p, align 4
+ %c = icmp sgt i32 %x, 0
+ br i1 %c, label %then, label %cont
+
+then:
+ %idx = sext i32 %x to i64
+ %gep2 = getelementptr inbounds i32, ptr %b, i64 %idx
+ %ld = load i32, ptr %gep2, align 4
+ br label %cont
+
+cont:
+ %v = phi i32 [ %ld, %then ], [ 0, %loop ]
+ %y = load i32, ptr %p, align 4
+ %sum = add i32 %v, %y
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The first load cannot be reused by the stronger aligned second load, but the
+; second one becomes the candidate for the third load, which reuses it.
+define void @cse_after_weaker_align_first(ptr noalias %a, ptr noalias %b) {
+; CHECK-LABEL: define void @cse_after_weaker_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %z = load i32, ptr %gep, align 8
+ %s1 = add i32 %x, %y
+ %sum = add i32 %s1, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
exit:
ret void
}
@@ -362,3 +754,9 @@ exit:
!2 = distinct !{!2, !"domain"}
!3 = !{!4}
!4 = distinct !{!4, !2, !"scope_b"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"float", !7, i64 0}
diff --git a/llvm/test/Transforms/LoopVectorize/cse.ll b/llvm/test/Transforms/LoopVectorize/cse.ll
deleted file mode 100644
index 47ab7ee60dc4a..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/cse.ll
+++ /dev/null
@@ -1,165 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
-
-; Tests CSE of duplicate non-memory recipes (GEPs, index arithmetic, casts).
-
-; Duplicate address GEPs for a load and a store are CSE'd into one.
-define void @dup_gep(ptr noalias %a) {
-; CHECK-LABEL: define void @dup_gep(
-; CHECK-SAME: ptr noalias [[A:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP0]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP2]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- %v = load i32, ptr %p, align 4
- %q = getelementptr inbounds i32, ptr %a, i64 %iv
- %inc = add i32 %v, 1
- store i32 %inc, ptr %q, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Duplicate index arithmetic used for two arrays is CSE'd.
-define void @dup_index(ptr noalias %a, ptr noalias %b, i64 %o) {
-; CHECK-LABEL: define void @dup_index(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[O:%.*]]) {
-; CHECK-NEXT: [[VECTOR_SCEVCHECK:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[O]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %i1 = add i64 %iv, %o
- %i2 = add i64 %iv, %o
- %pa = getelementptr inbounds i32, ptr %a, i64 %i1
- %pb = getelementptr inbounds i32, ptr %b, i64 %i2
- %vb = load i32, ptr %pb, align 4
- store i32 %vb, ptr %pa, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; A duplicated loop-invariant cast is CSE'd.
-define void @dup_invariant_cast(ptr noalias %a, i16 %k) {
-; CHECK-LABEL: define void @dup_invariant_cast(
-; CHECK-SAME: ptr noalias [[A:%.*]], i16 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[TMP0]]
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[TMP2]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %z1 = zext i16 %k to i32
- %z2 = zext i16 %k to i32
- %sum = add i32 %z1, %z2
- %p = getelementptr inbounds i32, ptr %a, i64 %iv
- store i32 %sum, ptr %p, align 4
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
-
-; Casts of the same operand to different result types are not CSE'd.
-define void @no_cse_different_cast_type(ptr noalias %a, ptr noalias %b, i16 %k) {
-; CHECK-LABEL: define void @no_cse_different_cast_type(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i16 [[K:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[K]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i32>
-; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i16> [[BROADCAST_SPLAT]] to <4 x i64>
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[TMP2]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP3]], align 8
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %z32 = zext i16 %k to i32
- %z64 = zext i16 %k to i64
- %pa = getelementptr inbounds i32, ptr %a, i64 %iv
- store i32 %z32, ptr %pa, align 4
- %pb = getelementptr inbounds i64, ptr %b, i64 %iv
- store i64 %z64, ptr %pb, align 8
- %iv.next = add i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 1024
- br i1 %ec, label %exit, label %loop
-exit:
- ret void
-}
>From 3fc6ed63a002ae6e79140803083ad401abe6823b Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 11 Aug 2026 17:34:54 +0530
Subject: [PATCH 5/7] [VPlan] Replace widened-load CSE alias scan with a memory
epoch
Reuse widened loads only within the same memory epoch, incremented on
every memory-writing recipe. This removes canCSEWithNoAliasCheck, but any
write now invalidates reuse, including across non-aliasing stores.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 +++++++------------
.../Transforms/LoopVectorize/cse-loads.ll | 12 ++--
2 files changed, 28 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6ff2c605e0c09..2eddadf5290b6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -274,28 +274,6 @@ canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
return true;
}
-/// Return true if no memory-writing recipe between \p From and \p To may alias
-/// \p MemLoc. Unlike canHoistOrSinkWithNoAliasCheck, this only scans the
-/// recipes strictly between \p From and \p To (which must be in the same
-/// block), as needed when common-subexpression-eliminating two loads: writes
-/// before \p From or after \p To cannot affect the reused value.
-static bool canCSEWithNoAliasCheck(const MemoryLocation &MemLoc,
- VPRecipeBase *From, VPRecipeBase *To) {
- if (From->getParent() != To->getParent())
- return false;
-
- for (VPRecipeBase &R :
- make_range(std::next(From->getIterator()), To->getIterator())) {
- if (!R.mayWriteToMemory())
- continue;
- auto Loc = vputils::getMemoryLocation(R);
- if (!Loc ||
- ScopedNoAliasAAResult::alias(*Loc, MemLoc) != AliasResult::NoAlias)
- return false;
- }
- return true;
-}
-
/// Get the value type of the replicate load or store. \p IsLoad indicates
/// whether it is a load.
static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
@@ -2273,8 +2251,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads are handled, as cse() guards their reuse with an aliasing
- // check. Any other memory access is rejected.
+ // Widened loads are handled, as cse() guards their reuse with a memory
+ // epoch check. Any other memory access is rejected.
if (Def->mayWriteToMemory())
return false;
return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
@@ -2349,32 +2327,40 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
VPDominatorTree VPDT(Plan);
- DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
+ // Maps a recipe to the equivalent recipe recorded earlier, together with the
+ // memory epoch it was recorded at.
+ DenseMap<VPSingleDefRecipe *, std::pair<VPSingleDefRecipe *, unsigned>,
+ VPCSEDenseMapInfo>
+ CSEMap;
+ // Incremented for every recipe that may write to memory.
+ unsigned MemEpoch = 0;
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
for (VPRecipeBase &R : *VPBB) {
+ if (R.mayWriteToMemory())
+ ++MemEpoch;
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
continue;
- if (VPSingleDefRecipe *V = CSEMap.lookup(Def)) {
+ auto It = CSEMap.find(Def);
+ if (It != CSEMap.end()) {
+ auto [V, Epoch] = It->second;
// V must dominate Def for a valid replacement.
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
auto *Load = cast<VPWidenLoadRecipe>(Def);
- // Reuse EarlierLoad only if it is at least as aligned as Load and no
- // aliasing write lies between the two loads.
- bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign();
- if (CanReuse) {
- auto Loc = vputils::getMemoryLocation(*EarlierLoad);
- CanReuse = canCSEWithNoAliasCheck(*Loc, EarlierLoad, Def);
- }
+ // Reuse EarlierLoad only if it is at least as aligned as Load and in
+ // the same block, with an unchanged epoch meaning no write lies
+ // between the two loads.
+ bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign() &&
+ EarlierLoad->getParent() == VPBB && Epoch == MemEpoch;
if (!CanReuse) {
// Record Load as the candidate for subsequent loads, as it may be
// reusable where EarlierLoad is not.
- CSEMap[Def] = Def;
+ CSEMap[Def] = {Def, MemEpoch};
continue;
}
// Keep only metadata common to both loads on the survivor.
@@ -2386,7 +2372,7 @@ void VPlanTransforms::cse(VPlan &Plan) {
Def->replaceAllUsesWith(V);
continue;
}
- CSEMap[Def] = Def;
+ CSEMap[Def] = {Def, MemEpoch};
}
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/cse-loads.ll b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
index 151df87401fc7..b19bf3b59957d 100644
--- a/llvm/test/Transforms/LoopVectorize/cse-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/cse-loads.ll
@@ -514,10 +514,11 @@ exit:
ret void
}
-; A store proven not to alias the loaded address (via scoped-noalias metadata)
-; does not prevent CSE of the second load.
-define void @cse_across_noalias_store(ptr %a, ptr %b) {
-; CHECK-LABEL: define void @cse_across_noalias_store(
+; A store between two loads prevents CSE of the second load.
+; TODO: The store is proven not to alias the loaded address via scoped-noalias
+; metadata, so the second load could be CSE'd here.
+define void @no_cse_across_noalias_store(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @no_cse_across_noalias_store(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
@@ -529,7 +530,8 @@ define void @cse_across_noalias_store(ptr %a, ptr %b) {
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13:![0-9]+]], !noalias [[META16:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
-; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope [[META13]], !noalias [[META16]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4, !alias.scope [[META16]], !noalias [[META13]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
>From 7f0a05a88c262e51a94892663e215b65ef5f558e Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 25 Aug 2026 18:06:06 +0530
Subject: [PATCH 6/7] Address review comments for widened-load CSE
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 85 +++++++++----------
1 file changed, 38 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2eddadf5290b6..6df959b05d704 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2251,8 +2251,8 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads are handled, as cse() guards their reuse with a memory
- // epoch check. Any other memory access is rejected.
+ // Widened loads are handled, as cse() only reuses them within a block with
+ // no intervening memory write. Any other memory access is rejected.
if (Def->mayWriteToMemory())
return false;
return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
@@ -2269,10 +2269,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, RFlags->getPredicate());
if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Def))
return hash_combine(Result, SIVSteps->getInductionOpcode());
- // Fold in the separately stored consecutive/masked flags. Alignment is left
- // out of the key and checked by cse().
+ // Fold in the separately stored consecutive flag. Alignment is left out and
+ // handled by cse.
if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
- return hash_combine(Result, Load->isConsecutive(), Load->isMasked());
+ return hash_combine(Result, Load->isConsecutive());
return Result;
}
@@ -2297,14 +2297,11 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
if (LSIV->getInductionOpcode() !=
cast<VPScalarIVStepsRecipe>(R)->getInductionOpcode())
return false;
- // Compare the separately stored consecutive/masked flags. Alignment is left
- // out and checked by cse().
- if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L)) {
- auto *RL = cast<VPWidenLoadRecipe>(R);
- if (LL->isConsecutive() != RL->isConsecutive() ||
- LL->isMasked() != RL->isMasked())
+ // Compare the separately stored consecutive flag. Alignment is left out and
+ // handled by cse.
+ if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L))
+ if (LL->isConsecutive() != cast<VPWidenLoadRecipe>(R)->isConsecutive())
return false;
- }
// Phi recipes can only be equal if they are in the same VPBB, as they
// implicitly depend on their predecessors.
if (isa<VPWidenPHIRecipe>(L) && L->getParent() != R->getParent())
@@ -2323,56 +2320,50 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
};
} // end anonymous namespace
-/// Perform a common-subexpression-elimination of single-def recipes on the \p
+/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
VPDominatorTree VPDT(Plan);
- // Maps a recipe to the equivalent recipe recorded earlier, together with the
- // memory epoch it was recorded at.
- DenseMap<VPSingleDefRecipe *, std::pair<VPSingleDefRecipe *, unsigned>,
- VPCSEDenseMapInfo>
- CSEMap;
- // Incremented for every recipe that may write to memory.
- unsigned MemEpoch = 0;
+ DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
+ // CSE map for widened loads.
+ DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo>
+ LoadCSEMap;
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
Plan.getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ LoadCSEMap.clear();
for (VPRecipeBase &R : *VPBB) {
if (R.mayWriteToMemory())
- ++MemEpoch;
+ LoadCSEMap.clear();
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
continue;
- auto It = CSEMap.find(Def);
- if (It != CSEMap.end()) {
- auto [V, Epoch] = It->second;
- // V must dominate Def for a valid replacement.
- if (!VPDT.dominates(V->getParent(), VPBB))
+ bool IsLoad = isa<VPWidenLoadRecipe>(Def);
+ auto [It, Inserted] =
+ (IsLoad ? LoadCSEMap : CSEMap).try_emplace(Def, Def);
+ if (Inserted)
+ continue;
+ VPSingleDefRecipe *V = It->second;
+ // V must dominate Def for a valid replacement.
+ if (!VPDT.dominates(V->getParent(), VPBB))
+ continue;
+ if (IsLoad) {
+ auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
+ auto *Load = cast<VPWidenLoadRecipe>(Def);
+ if (EarlierLoad->getAlign() < Load->getAlign()) {
+ // Record Load as the candidate for subsequent loads, as it may be
+ // reusable where EarlierLoad is not.
+ It->second = Load;
continue;
- if (auto *EarlierLoad = dyn_cast<VPWidenLoadRecipe>(V)) {
- auto *Load = cast<VPWidenLoadRecipe>(Def);
- // Reuse EarlierLoad only if it is at least as aligned as Load and in
- // the same block, with an unchanged epoch meaning no write lies
- // between the two loads.
- bool CanReuse = EarlierLoad->getAlign() >= Load->getAlign() &&
- EarlierLoad->getParent() == VPBB && Epoch == MemEpoch;
- if (!CanReuse) {
- // Record Load as the candidate for subsequent loads, as it may be
- // reusable where EarlierLoad is not.
- CSEMap[Def] = {Def, MemEpoch};
- continue;
- }
- // Keep only metadata common to both loads on the survivor.
- EarlierLoad->intersect(*Load);
}
- // Only keep flags present on both V and Def.
- if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
- RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def));
- Def->replaceAllUsesWith(V);
- continue;
+ // Keep only metadata common to both loads on the survivor.
+ EarlierLoad->intersect(*Load);
}
- CSEMap[Def] = {Def, MemEpoch};
+ // Only keep flags present on both V and Def.
+ if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(V))
+ RFlags->intersectFlags(*cast<VPRecipeWithIRFlags>(Def));
+ Def->replaceAllUsesWith(V);
}
}
}
>From eae71ce9f18eeca0e251a1e0c29823f6ed608cdf Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 25 Aug 2026 19:51:35 +0530
Subject: [PATCH 7/7] Support EVL load variant in cse
---
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 23 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +-
.../LoopVectorize/RISCV/cse-loads-evl.ll | 531 ++++++++++++++++++
4 files changed, 549 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f15310a6de382..8f45fe98fafa4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3861,6 +3861,9 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadEVLRecipe final
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadEVLSC)
+ /// Returns the opcode of the widened load.
+ unsigned getOpcode() const { return Instruction::Load; }
+
/// Return the EVL operand.
VPValue *getEVL() const { return getOperand(1); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6df959b05d704..4c57da1172fef 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2251,11 +2251,13 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
- // Widened loads are handled, as cse() only reuses them within a block with
- // no intervening memory write. Any other memory access is rejected.
+ // Widened loads (including the EVL variant) are handled, as cse() only
+ // reuses them within a block with no intervening memory write. Any other
+ // memory access is rejected.
if (Def->mayWriteToMemory())
return false;
- return !Def->mayReadFromMemory() || isa<VPWidenLoadRecipe>(Def);
+ return !Def->mayReadFromMemory() ||
+ isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Def);
}
/// Hash the underlying data of \p Def.
@@ -2271,7 +2273,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return hash_combine(Result, SIVSteps->getInductionOpcode());
// Fold in the separately stored consecutive flag. Alignment is left out and
// handled by cse.
- if (auto *Load = dyn_cast<VPWidenLoadRecipe>(Def))
+ if (auto *Load = dyn_cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(Def)))
return hash_combine(Result, Load->isConsecutive());
return Result;
}
@@ -2299,8 +2301,9 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return false;
// Compare the separately stored consecutive flag. Alignment is left out and
// handled by cse.
- if (auto *LL = dyn_cast<VPWidenLoadRecipe>(L))
- if (LL->isConsecutive() != cast<VPWidenLoadRecipe>(R)->isConsecutive())
+ if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(L)))
+ if (LL->isConsecutive() !=
+ cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(R))->isConsecutive())
return false;
// Phi recipes can only be equal if they are in the same VPBB, as they
// implicitly depend on their predecessors.
@@ -2339,7 +2342,7 @@ void VPlanTransforms::cse(VPlan &Plan) {
auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
continue;
- bool IsLoad = isa<VPWidenLoadRecipe>(Def);
+ bool IsLoad = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Def);
auto [It, Inserted] =
(IsLoad ? LoadCSEMap : CSEMap).try_emplace(Def, Def);
if (Inserted)
@@ -2349,12 +2352,12 @@ void VPlanTransforms::cse(VPlan &Plan) {
if (!VPDT.dominates(V->getParent(), VPBB))
continue;
if (IsLoad) {
- auto *EarlierLoad = cast<VPWidenLoadRecipe>(V);
- auto *Load = cast<VPWidenLoadRecipe>(Def);
+ auto *EarlierLoad = cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(V));
+ auto *Load = cast<VPWidenMemoryRecipe>(cast<VPRecipeBase>(Def));
if (EarlierLoad->getAlign() < Load->getAlign()) {
// Record Load as the candidate for subsequent loads, as it may be
// reusable where EarlierLoad is not.
- It->second = Load;
+ It->second = Def;
continue;
}
// Keep only metadata common to both loads on the survivor.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index ea905cbf24797..7be8161cb78a6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -366,8 +366,8 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
unsigned vputils::getOpcode(const VPValue *V) {
return TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
- VPReplicateRecipe, VPWidenPHIRecipe, VPWidenLoadRecipe>(
- [](auto *I) { return I->getOpcode(); })
+ VPReplicateRecipe, VPWidenPHIRecipe, VPWidenLoadRecipe,
+ VPWidenLoadEVLRecipe>([](auto *I) { return I->getOpcode(); })
.Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
[](auto *I) {
// For recipes that do not directly map to LLVM IR instructions,
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll
new file mode 100644
index 0000000000000..6ffe8bc3aa387
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/cse-loads-evl.ll
@@ -0,0 +1,531 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt < %s -passes=loop-vectorize -mtriple=riscv64 -mattr=+v \
+; RUN: -tail-folding-policy=prefer-fold-tail -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -scalable-vectorization=on -S | FileCheck %s
+
+; CSE of VPWidenLoadEVLRecipe: duplicate consecutive loads become one vp.load.
+
+define void @cse_duplicate_load(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_duplicate_load(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; A store between two loads prevents CSE of the second load.
+define void @no_cse_across_store(ptr %a, i64 %n) {
+; CHECK-LABEL: define void @no_cse_across_store(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 1)
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD1]], splat (i32 2)
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %inc = add i32 %x, 1
+ store i32 %inc, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 4
+ %sink = add i32 %y, 2
+ store i32 %sink, ptr %gep, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two masked loads from the same address under the same condition are CSE'd.
+define void @two_masked_same_mask(ptr noalias %a, ptr noalias %b, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: define void @two_masked_same_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> [[TMP2]], i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD1]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP2]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI]], ptr align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP6:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP6]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %gep.c = getelementptr inbounds i32, ptr %cond, i64 %iv
+ %c = load i32, ptr %gep.c, align 4
+ %cmp = icmp ne i32 %c, 0
+ br i1 %cmp, label %if, label %latch
+
+if:
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %s = add i32 %x, %y
+ br label %latch
+
+latch:
+ %m = phi i32 [ 0, %loop ], [ %s, %if ]
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %m, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Masked loads from the same address under different conditions are not CSE'd.
+define void @two_masked_diff_mask(ptr noalias %a, ptr noalias %b, ptr noalias %c1, ptr noalias %c2, i64 %n) {
+; CHECK-LABEL: define void @two_masked_diff_mask(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C1:%.*]], ptr noalias [[C2:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[C1]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> [[TMP3]], i32 [[TMP0]])
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP3]], <vscale x 4 x i32> [[VP_OP_LOAD1]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[C2]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 4 x i32> [[VP_OP_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[VP_OP_LOAD3:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> [[TMP5]], i32 [[TMP0]])
+; CHECK-NEXT: [[PREDPHI4:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x i32> [[VP_OP_LOAD3]], <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <vscale x 4 x i32> [[PREDPHI]], [[PREDPHI4]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP6]], ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch2 ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %gp1 = getelementptr inbounds i32, ptr %c1, i64 %iv
+ %v1 = load i32, ptr %gp1, align 4
+ %cmp1 = icmp ne i32 %v1, 0
+ br i1 %cmp1, label %if1, label %latch1
+
+if1:
+ %x = load i32, ptr %gep.a, align 4
+ br label %latch1
+
+latch1:
+ %mx = phi i32 [ 0, %loop ], [ %x, %if1 ]
+ %gp2 = getelementptr inbounds i32, ptr %c2, i64 %iv
+ %v2 = load i32, ptr %gp2, align 4
+ %cmp2 = icmp ne i32 %v2, 0
+ br i1 %cmp2, label %if2, label %latch2
+
+if2:
+ %y = load i32, ptr %gep.a, align 4
+ br label %latch2
+
+latch2:
+ %my = phi i32 [ 0, %latch1 ], [ %y, %if2 ]
+ %sum = add i32 %mx, %my
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Three identical loads collapse to a single vp.load.
+define void @cse_three_duplicate_loads(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_three_duplicate_loads(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[TMP2]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %z = load i32, ptr %gep.a, align 4
+ %t = add i32 %x, %y
+ %sum = add i32 %t, %z
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Two independent duplicate-load groups are each CSE'd.
+define void @two_dup_groups(ptr noalias %a, ptr noalias %b, ptr noalias %out, i64 %n) {
+; CHECK-LABEL: define void @two_dup_groups(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD1]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <vscale x 4 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %pa = getelementptr inbounds i32, ptr %a, i64 %iv
+ %a1 = load i32, ptr %pa, align 4
+ %a2 = load i32, ptr %pa, align 4
+ %pb = getelementptr inbounds i32, ptr %b, i64 %iv
+ %b1 = load i32, ptr %pb, align 4
+ %b2 = load i32, ptr %pb, align 4
+ %sa = add i32 %a1, %a2
+ %sb = add i32 %b1, %b2
+ %s = add i32 %sa, %sb
+ %po = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %s, ptr %po, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Stronger alignment first: later load reuses it, alignment 8 is kept.
+define void @cse_stronger_align_first(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @cse_stronger_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 8 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 8
+ %y = load i32, ptr %gep, align 4
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Weaker alignment first: CSE is skipped so the later load keeps align 8.
+define void @no_cse_weaker_align_first(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @no_cse_weaker_align_first(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 8 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep, align 4
+ %y = load i32, ptr %gep, align 8
+ %sum = add i32 %x, %y
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Any intervening store clears LoadCSEMap, even if it cannot alias the load.
+define void @no_cse_across_noalias_store(ptr noalias %a, ptr noalias %b, i64 %n) {
+; CHECK-LABEL: define void @no_cse_across_noalias_store(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_OP_LOAD]], ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP3:%.*]] = add <vscale x 4 x i32> [[VP_OP_LOAD]], [[VP_OP_LOAD1]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP3]], ptr align 4 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+ %x = load i32, ptr %gep.a, align 4
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ store i32 %x, ptr %gep.b, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ store i32 %sum, ptr %gep.b, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Duplicate gathers (non-consecutive EVL loads) are CSE'd into one vp.gather.
+define void @dup_gather(ptr noalias %a, ptr noalias %idx, ptr noalias %out, i64 %n) {
+; CHECK-LABEL: define void @dup_gather(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[IDX:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[IDX]], i64 [[INDEX]]
+; CHECK-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP1]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <vscale x 4 x i64> [[VP_OP_LOAD]]
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr align 4 [[TMP3]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %gep.idx = getelementptr inbounds i64, ptr %idx, i64 %iv
+ %i = load i64, ptr %gep.idx, align 8
+ %gep.a = getelementptr inbounds i32, ptr %a, i64 %i
+ %x = load i32, ptr %gep.a, align 4
+ %y = load i32, ptr %gep.a, align 4
+ %sum = add i32 %x, %y
+ %gep.o = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %sum, ptr %gep.o, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list