[llvm] [InstCombine] Fix vector_reduce_mul(sext <n x i1>) for odd n. (PR #199401)
Justin Lebar via llvm-commits
llvm-commits at lists.llvm.org
Sun May 24 03:01:09 PDT 2026
https://github.com/jlebar updated https://github.com/llvm/llvm-project/pull/199401
>From 8bf3a14eba897d0a71a22d43c79cd93513e06f71 Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Thu, 21 May 2026 21:49:46 -0700
Subject: [PATCH 1/2] [InstCombine] Fix vector_reduce_mul(sext <n x i1>).
Before this patch, instcombine folded
vector_reduce_mul(sext (<n x i1> val))
to
zext(vector_reduce_and(<n x i1> val)).
But this is incorrect when n is odd: The result of the reduction is -1,
not 1.
This bug was found by a large run of Opus 4.7 looking for bugs in LLVM.
---
.../InstCombine/InstCombineCalls.cpp | 45 +++++++++++++------
.../InstCombine/reduction-mul-sext-zext-i1.ll | 25 +++++++++++
2 files changed, 56 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 360326f47594d..b567d5a0b665b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -4111,14 +4111,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
}
case Intrinsic::vector_reduce_mul: {
if (IID == Intrinsic::vector_reduce_mul) {
- // Multiplicative reduction over the vector with (potentially-extended)
- // i1 element type is actually a (potentially zero-extended)
- // logical `and` reduction over the original non-extended value:
- // vector_reduce_mul(?ext(<n x i1>))
- // -->
- // zext(vector_reduce_and(<n x i1>))
Value *Arg = II->getArgOperand(0);
- Value *Vect;
if (Value *NewOp =
simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {
@@ -4126,13 +4119,37 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
return II;
}
- if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
- if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))
- if (VTy->getElementType() == Builder.getInt1Ty()) {
- Value *Res = Builder.CreateAndReduce(Vect);
- Res = Builder.CreateZExt(Res, II->getType());
- return replaceInstUsesWith(CI, Res);
- }
+ auto IsI1Vec = [&](Value *V) {
+ auto *VTy = dyn_cast<VectorType>(V->getType());
+ return VTy && VTy->getElementType() == Builder.getInt1Ty();
+ };
+
+ // vector_reduce_mul(zext(<n x i1>)) --> zext(vector_reduce_and(<n x i1>))
+ Value *Vect;
+ if (match(Arg, m_ZExt(m_Value(Vect))) && IsI1Vec(Vect)) {
+ Value *Res = Builder.CreateAndReduce(Vect);
+ return replaceInstUsesWith(CI, Builder.CreateZExt(Res, II->getType()));
+ }
+
+ // vector_reduce_mul(sext(<n x i1>)) -->
+ // sext(vector_reduce_and(<n x i1>)) if n is odd
+ // zext(vector_reduce_and(<n x i1>)) if n is even.
+ // This is because if the vector is all `true`, we are multiplying n -1s.
+ // Therefore the answer is -1 if n is odd, or 1 if n is even.
+ if (match(Arg, m_SExt(m_Value(Vect)))) {
+ if (auto *VTy = dyn_cast<FixedVectorType>(Vect->getType());
+ VTy && VTy->getElementType() == Builder.getInt1Ty()) {
+ Value *Res = Builder.CreateAndReduce(Vect);
+ Res = (VTy->getNumElements() & 1)
+ ? Builder.CreateSExt(Res, II->getType())
+ : Builder.CreateZExt(Res, II->getType());
+ return replaceInstUsesWith(CI, Res);
+ }
+ }
+
+ // vector_reduce_mul(<n x i1>) --> vector_reduce_and(<n x i1>)
+ if (IsI1Vec(Arg)) {
+ return replaceInstUsesWith(CI, Builder.CreateAndReduce(Arg));
}
}
[[fallthrough]];
diff --git a/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll b/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll
index f70820801602c..c867e076d9b4a 100644
--- a/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll
+++ b/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll
@@ -95,8 +95,33 @@ define i64 @reduce_mul_zext_external_use(<8 x i1> %x) {
ret i64 %res
}
+define i8 @reduce_mul_sext_odd_lanes(<3 x i1> %x) {
+; CHECK-LABEL: @reduce_mul_sext_odd_lanes(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <3 x i1> [[X:%.*]] to i3
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i3 [[TMP1]], -1
+; CHECK-NEXT: [[RES:%.*]] = sext i1 [[TMP2]] to i8
+; CHECK-NEXT: ret i8 [[RES]]
+;
+ %sext = sext <3 x i1> %x to <3 x i8>
+ %res = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %sext)
+ ret i8 %res
+}
+
+define i8 @reduce_mul_zext_odd_lanes(<3 x i1> %x) {
+; CHECK-LABEL: @reduce_mul_zext_odd_lanes(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <3 x i1> [[X:%.*]] to i3
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i3 [[TMP1]], -1
+; CHECK-NEXT: [[RES:%.*]] = zext i1 [[TMP2]] to i8
+; CHECK-NEXT: ret i8 [[RES]]
+;
+ %zext = zext <3 x i1> %x to <3 x i8>
+ %res = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %zext)
+ ret i8 %res
+}
+
declare i1 @llvm.vector.reduce.mul.v8i32(<8 x i1> %a)
declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %a)
declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %a)
declare i8 @llvm.vector.reduce.mul.v128i8(<128 x i8> %a)
+declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %a)
>From 008728fb5c0a0a6d45fcaac7a512df729e7c009e Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Sun, 24 May 2026 02:34:24 -0700
Subject: [PATCH 2/2] Review comments
---
.../InstCombine/InstCombineCalls.cpp | 40 ++++++----------
.../InstCombine/reduction-mul-sext-zext-i1.ll | 48 +++++++++++++++----
2 files changed, 53 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b567d5a0b665b..09541cfff7642 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -4119,38 +4119,26 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
return II;
}
- auto IsI1Vec = [&](Value *V) {
- auto *VTy = dyn_cast<VectorType>(V->getType());
- return VTy && VTy->getElementType() == Builder.getInt1Ty();
- };
-
- // vector_reduce_mul(zext(<n x i1>)) --> zext(vector_reduce_and(<n x i1>))
+ // vector_reduce_mul(zext(<n x i1>)), or
+ // vector_reduce_mul(sext(<n x i1>)) (if n is even) -->
+ // zext(vector_reduce_and(<n x i1>)).
+ // (The sext case doesn't work if n is odd because multiplying an odd
+ // number of -1's produces -1, not 1.)
Value *Vect;
- if (match(Arg, m_ZExt(m_Value(Vect))) && IsI1Vec(Vect)) {
+ bool IsZext = match(Arg, m_ZExt(m_Value(Vect))) &&
+ Vect->getType()->isIntOrIntVectorTy(1);
+ bool IsSext =
+ match(Arg, m_SExt(m_Value(Vect))) &&
+ Vect->getType()->isIntOrIntVectorTy(1) &&
+ cast<VectorType>(Vect->getType())->getElementCount().isKnownEven();
+ if (IsZext || IsSext) {
Value *Res = Builder.CreateAndReduce(Vect);
- return replaceInstUsesWith(CI, Builder.CreateZExt(Res, II->getType()));
- }
-
- // vector_reduce_mul(sext(<n x i1>)) -->
- // sext(vector_reduce_and(<n x i1>)) if n is odd
- // zext(vector_reduce_and(<n x i1>)) if n is even.
- // This is because if the vector is all `true`, we are multiplying n -1s.
- // Therefore the answer is -1 if n is odd, or 1 if n is even.
- if (match(Arg, m_SExt(m_Value(Vect)))) {
- if (auto *VTy = dyn_cast<FixedVectorType>(Vect->getType());
- VTy && VTy->getElementType() == Builder.getInt1Ty()) {
- Value *Res = Builder.CreateAndReduce(Vect);
- Res = (VTy->getNumElements() & 1)
- ? Builder.CreateSExt(Res, II->getType())
- : Builder.CreateZExt(Res, II->getType());
- return replaceInstUsesWith(CI, Res);
- }
+ return CastInst::Create(Instruction::ZExt, Res, II->getType());
}
// vector_reduce_mul(<n x i1>) --> vector_reduce_and(<n x i1>)
- if (IsI1Vec(Arg)) {
+ if (Arg->getType()->isIntOrIntVectorTy(1))
return replaceInstUsesWith(CI, Builder.CreateAndReduce(Arg));
- }
}
[[fallthrough]];
}
diff --git a/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll b/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll
index c867e076d9b4a..7a39d4124808f 100644
--- a/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll
+++ b/llvm/test/Transforms/InstCombine/reduction-mul-sext-zext-i1.ll
@@ -95,11 +95,12 @@ define i64 @reduce_mul_zext_external_use(<8 x i1> %x) {
ret i64 %res
}
+; Can't fold to `zext(reduce.and)`, because, since our input has an odd number
+; of elements, a value of all `true` produces an output of -1, not 1.
define i8 @reduce_mul_sext_odd_lanes(<3 x i1> %x) {
; CHECK-LABEL: @reduce_mul_sext_odd_lanes(
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <3 x i1> [[X:%.*]] to i3
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i3 [[TMP1]], -1
-; CHECK-NEXT: [[RES:%.*]] = sext i1 [[TMP2]] to i8
+; CHECK-NEXT: [[SEXT:%.*]] = sext <3 x i1> [[X:%.*]] to <3 x i8>
+; CHECK-NEXT: [[RES:%.*]] = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> [[SEXT]])
; CHECK-NEXT: ret i8 [[RES]]
;
%sext = sext <3 x i1> %x to <3 x i8>
@@ -119,9 +120,38 @@ define i8 @reduce_mul_zext_odd_lanes(<3 x i1> %x) {
ret i8 %res
}
-declare i1 @llvm.vector.reduce.mul.v8i32(<8 x i1> %a)
-declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
-declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %a)
-declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %a)
-declare i8 @llvm.vector.reduce.mul.v128i8(<128 x i8> %a)
-declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %a)
+define i32 @reduce_mul_sext_scalable_even(<vscale x 4 x i1> %x) {
+; CHECK-LABEL: @reduce_mul_sext_scalable_even(
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.vector.reduce.and.nxv4i1(<vscale x 4 x i1> [[X:%.*]])
+; CHECK-NEXT: [[RES:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %sext = sext <vscale x 4 x i1> %x to <vscale x 4 x i32>
+ %res = call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> %sext)
+ ret i32 %res
+}
+
+; Can't fold to `zext(reduce.and)`, because we don't know if the input has an
+; odd or even number of elements, and this fold is only valid for an even
+; number.
+define i32 @reduce_mul_sext_scalable_unknown_parity(<vscale x 3 x i1> %x) {
+; CHECK-LABEL: @reduce_mul_sext_scalable_unknown_parity(
+; CHECK-NEXT: [[SEXT:%.*]] = sext <vscale x 3 x i1> [[X:%.*]] to <vscale x 3 x i32>
+; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.vector.reduce.mul.nxv3i32(<vscale x 3 x i32> [[SEXT]])
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %sext = sext <vscale x 3 x i1> %x to <vscale x 3 x i32>
+ %res = call i32 @llvm.vector.reduce.mul.nxv3i32(<vscale x 3 x i32> %sext)
+ ret i32 %res
+}
+
+define i32 @reduce_mul_zext_scalable(<vscale x 3 x i1> %x) {
+; CHECK-LABEL: @reduce_mul_zext_scalable(
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.vector.reduce.and.nxv3i1(<vscale x 3 x i1> [[X:%.*]])
+; CHECK-NEXT: [[RES:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %zext = zext <vscale x 3 x i1> %x to <vscale x 3 x i32>
+ %res = call i32 @llvm.vector.reduce.mul.nxv3i32(<vscale x 3 x i32> %zext)
+ ret i32 %res
+}
More information about the llvm-commits
mailing list