[llvm-branch-commits] [llvm] 907f6ab - [InstCombine] Fix miscompile when folding a select into a masked load (#216730)
Douglas Yung via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 20 13:50:30 PDT 2026
Author: Chennes
Date: 2026-08-20T20:50:18Z
New Revision: 907f6abbd8011635913900eb76c5fb757dbed825
URL: https://github.com/llvm/llvm-project/commit/907f6abbd8011635913900eb76c5fb757dbed825
DIFF: https://github.com/llvm/llvm-project/commit/907f6abbd8011635913900eb76c5fb757dbed825.diff
LOG: [InstCombine] Fix miscompile when folding a select into a masked load (#216730)
`visitSelectInst` folds:
select(mask, masked.load(ptr, mask, PT), FV)
into:
masked.load(ptr, mask, FV)
The replacement load was previously created at the select, effectively
moving the memory access past any intervening instructions. If one of
them writes the loaded memory, the replacement load reads the updated
value instead of the original one. This was also observed downstream in
[ispc/ispc#3891](https://github.com/ispc/ispc/issues/3891).
The fold was added in `eb8589987267`. The issue is labelled
`regression:22`, so it affects LLVM 22.1 as well as current trunk.
Create the replacement load at the original load's position and require
`FV` to be available there. Otherwise, leave the select unchanged.
Requiring `FV` to be available at the original load means the fold no
longer fires when `FV` is computed between the load and the select. No
existing `llvm/test/Transforms` checks change as a result of this
restriction.
Tests cover an intervening aliasing store and the case where `FV` is
unavailable at the original load. They also guard against carrying over
call-site attributes such as `range` and `noundef` when those attributes
no longer apply.
Fixes #215453
(cherry picked from commit fec2cbad15395d9d971e5a81e660e5ab604da4e8)
Added:
Modified:
llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
llvm/test/Transforms/InstCombine/select-masked_load.ll
llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index f107c15304d9f..98b2b7dd2c01f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -5306,12 +5306,18 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
Value *MaskedLoadPtr;
if (match(TrueVal, m_OneUse(m_MaskedLoad(m_Value(MaskedLoadPtr),
- m_Specific(CondVal), m_Value()))))
- return replaceInstUsesWith(
- SI, Builder.CreateMaskedLoad(
- TrueVal->getType(), MaskedLoadPtr,
- cast<IntrinsicInst>(TrueVal)->getParamAlign(0).valueOrOne(),
- CondVal, FalseVal));
+ m_Specific(CondVal), m_Value())))) {
+ auto *LoadInst = cast<IntrinsicInst>(TrueVal);
+ // Keep the load at its original position to avoid crossing writes. The new
+ // passthrough must therefore be available there.
+ if (DT.dominates(FalseVal, LoadInst)) {
+ Builder.SetInsertPoint(LoadInst);
+ Instruction *In = Builder.CreateMaskedLoad(
+ TrueVal->getType(), MaskedLoadPtr,
+ LoadInst->getParamAlign(0).valueOrOne(), CondVal, FalseVal);
+ return replaceInstUsesWith(SI, In);
+ }
+ }
// Canonicalize sign function ashr pattern: select (icmp slt X, 1), ashr X,
// bitwidth-1, 1 -> scmp(X, 0)
diff --git a/llvm/test/Transforms/InstCombine/select-masked_load.ll b/llvm/test/Transforms/InstCombine/select-masked_load.ll
index f7584c41ce641..a196c56bcac5e 100644
--- a/llvm/test/Transforms/InstCombine/select-masked_load.ll
+++ b/llvm/test/Transforms/InstCombine/select-masked_load.ll
@@ -149,6 +149,47 @@ define <vscale x 4 x float> @fold_sel_into_masked_load_scalable_one_use_check(pt
ret <vscale x 4 x float> %sel
}
+; Keep the folded load before an intervening aliasing store.
+define <4 x float> @fold_sel_into_masked_load_aliasing_store(ptr %ptr, <4 x i1> %mask, <4 x float> %passthrough) {
+; CHECK-LABEL: @fold_sel_into_masked_load_aliasing_store(
+; CHECK-NEXT: [[SEL:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK:%.*]], <4 x float> [[PASSTHROUGH:%.*]])
+; CHECK-NEXT: store <4 x float> [[PASSTHROUGH]], ptr [[PTR]], align 16
+; CHECK-NEXT: ret <4 x float> [[SEL]]
+;
+ %load = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x float> zeroinitializer)
+ store <4 x float> %passthrough, ptr %ptr, align 16
+ %sel = select <4 x i1> %mask, <4 x float> %load, <4 x float> %passthrough
+ ret <4 x float> %sel
+}
+
+; Do not fold when the new passthrough is unavailable at the old load.
+define <4 x float> @neg_fold_sel_into_masked_load_passthrough_after_load(ptr %ptr, <4 x i1> %mask, <4 x float> %a) {
+; CHECK-LABEL: @neg_fold_sel_into_masked_load_passthrough_after_load(
+; CHECK-NEXT: [[LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK:%.*]], <4 x float> zeroinitializer)
+; CHECK-NEXT: [[PASSTHROUGH:%.*]] = fadd <4 x float> [[A:%.*]], [[A]]
+; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[MASK]], <4 x float> [[LOAD]], <4 x float> [[PASSTHROUGH]]
+; CHECK-NEXT: ret <4 x float> [[SEL]]
+;
+ %load = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x float> zeroinitializer)
+ %passthrough = fadd <4 x float> %a, %a
+ %sel = select <4 x i1> %mask, <4 x float> %load, <4 x float> %passthrough
+ ret <4 x float> %sel
+}
+
+; Do not copy result or passthrough attributes (range/noundef) to the new load.
+; Use the current intrinsic form because auto-upgrading the legacy form drops
+; these attributes before InstCombine.
+define <8 x i16> @fold_sel_into_masked_load_drop_attrs(ptr %ptr, <8 x i1> %mask, <8 x i16> %passthrough) {
+; CHECK-LABEL: @fold_sel_into_masked_load_drop_attrs(
+; CHECK-NEXT: [[SEL:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[PTR:%.*]], <8 x i1> [[MASK:%.*]], <8 x i16> [[PASSTHROUGH:%.*]])
+; CHECK-NEXT: ret <8 x i16> [[SEL]]
+;
+ %load = call range(i16 0, 2) <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 %ptr, <8 x i1> %mask, <8 x i16> noundef zeroinitializer)
+ %sel = select <8 x i1> %mask, <8 x i16> %load, <8 x i16> %passthrough
+ ret <8 x i16> %sel
+}
+
declare <8 x float> @llvm.masked.load.v8f32.p0(ptr, i32 immarg, <8 x i1>, <8 x float>)
declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)
declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32 immarg, <4 x i1>, <4 x float>)
+declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, <8 x i1>, <8 x i16>)
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
index 4a1ce5a8a3f4c..5fa77a014c442 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-pred.ll
@@ -163,14 +163,14 @@ define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ult <4 x i16> [[VEC_IND]], splat (i16 257)
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP43:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP32]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP46:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
; CHECK-NEXT: [[TMP40:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[VEC_IND1]], <4 x i32> splat (i32 1)
; CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = mul i32 [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP32]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
; CHECK-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP43]])
; CHECK-NEXT: [[TMP45:%.*]] = mul i32 [[TMP42]], [[TMP44]]
-; CHECK-NEXT: [[TMP46:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
; CHECK-NEXT: [[TMP47:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP46]])
; CHECK-NEXT: [[TMP48]] = mul i32 [[TMP45]], [[TMP47]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP31]], 4
@@ -276,11 +276,11 @@ define i32 @reduction_mul(ptr noalias nocapture %A, ptr noalias nocapture %B) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ult <4 x i16> [[VEC_IND]], splat (i16 257)
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
; CHECK-NEXT: [[TMP40:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP32]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP43:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
; CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = mul i32 [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 1))
; CHECK-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP43]])
; CHECK-NEXT: [[TMP45]] = mul i32 [[TMP42]], [[TMP44]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP31]], 4
@@ -326,11 +326,11 @@ define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ult <4 x i16> [[VEC_IND]], splat (i16 257)
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
; CHECK-NEXT: [[TMP40:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP32]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 -1))
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP43:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 -1))
; CHECK-NEXT: [[TMP41:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = and i32 [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x i32> splat (i32 -1))
; CHECK-NEXT: [[TMP44:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP43]])
; CHECK-NEXT: [[TMP45]] = and i32 [[TMP42]], [[TMP44]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP31]], 4
@@ -524,11 +524,11 @@ define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ult <4 x i16> [[VEC_IND]], splat (i16 257)
; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds [4 x i8], ptr [[A:%.*]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
; CHECK-NEXT: [[TMP40:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP32]], <4 x i1> [[TMP0]], <4 x float> splat (float 1.000000e+00))
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds [4 x i8], ptr [[B:%.*]], i64 [[TMP31]]
+; CHECK-NEXT: [[TMP43:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x float> splat (float 1.000000e+00))
; CHECK-NEXT: [[TMP41:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = fmul fast float [[VEC_PHI]], [[TMP41]]
-; CHECK-NEXT: [[TMP43:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[TMP35]], <4 x i1> [[TMP0]], <4 x float> splat (float 1.000000e+00))
; CHECK-NEXT: [[TMP44:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP43]])
; CHECK-NEXT: [[TMP45]] = fmul fast float [[TMP42]], [[TMP44]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP31]], 4
More information about the llvm-branch-commits
mailing list