[llvm] 47f32bd - [VectorCombine] Fold reduction chains with equivalent bitcast sources (#212084)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 03:20:51 PDT 2026
Author: Jinpeng Wang
Date: 2026-08-10T10:20:45Z
New Revision: 47f32bddd3496cea2eac0df5f46108a704dde51d
URL: https://github.com/llvm/llvm-project/commit/47f32bddd3496cea2eac0df5f46108a704dde51d
DIFF: https://github.com/llvm/llvm-project/commit/47f32bddd3496cea2eac0df5f46108a704dde51d.diff
LOG: [VectorCombine] Fold reduction chains with equivalent bitcast sources (#212084)
## Description
`foldShuffleChainsToReduce` currently identifies leaf sources by SSA
value identity.
This can prevent a shuffle reduction chain from being folded when
multiple bitcast instructions represent the same vector source.
This happens when an earlier VectorCombine transform moves a bitcast
through a shuffle and creates a new bitcast of the same underlying
value.
The reduction matcher then sees the original and newly created bitcasts
as separate sources and rejects the fold.
This PR treats bitcast sources as equivalent when they have the same
result type and the same operand.
It then merges the demanded lanes of equivalent sources while preserving
duplicate-lane semantics:
* Reject duplicate or overlapping lanes for non-idempotent reductions.
* Allow overlapping lanes for idempotent reductions.
* Continue to reject sources with different underlying values or result
types.
This allows the default optimization pipeline to recognize and fold the
horizontal reductions in both `hsum_i32_4` and `hsum_i32_8` in #210897
## Tests
* Add VectorCombine tests for:
* equivalent bitcast sources with disjoint demanded lanes;
* overlapping lanes for non-idempotent reductions;
* duplicates within a later equivalent source;
* bitcasts of different underlying values;
* overlapping lanes for idempotent reductions.
Fixes #210897.
Added:
Modified:
llvm/lib/Transforms/Vectorize/VectorCombine.cpp
llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 726f564b1aad9..b88472f69c548 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -219,6 +219,12 @@ static Value *peekThroughBitcasts(Value *V) {
return V;
}
+/// Helper to peek through bitcasts to the same value.
+static bool isEquivBitcast(Value *X, Value *Y) {
+ return X->getType() == Y->getType() &&
+ peekThroughBitcasts(X) == peekThroughBitcasts(Y);
+}
+
static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI) {
// Do not widen load if atomic/volatile or under asan/hwasan/memtag/tsan.
// The widened load may load data from dirty regions or create data races
@@ -3740,19 +3746,13 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
if (!FrontV)
return false;
- // Helper to peek through bitcasts to the same value.
- auto IsEquiv = [&](Value *X, Value *Y) {
- return X->getType() == Y->getType() &&
- peekThroughBitcasts(X) == peekThroughBitcasts(Y);
- };
-
// Look for an identity value.
if (FrontLane == 0 &&
cast<FixedVectorType>(FrontV->getType())->getNumElements() ==
Item.size() &&
- all_of(drop_begin(enumerate(Item)), [IsEquiv, Item](const auto &E) {
+ all_of(drop_begin(enumerate(Item)), [Item](const auto &E) {
Value *FrontV = Item.front().first;
- return !E.value().first || (IsEquiv(E.value().first, FrontV) &&
+ return !E.value().first || (isEquivBitcast(E.value().first, FrontV) &&
E.value().second == (int)E.index());
})) {
IdentityLeafs.insert(std::make_pair(FrontV, From));
@@ -4267,11 +4267,23 @@ bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
auto It = Demands.find(S);
if (It == Demands.end() || It->second.Lanes.isZero())
continue;
- if (Cut || (!IsIdempotent && !It->second.Duplicates.isZero())) {
+ if (!IsIdempotent && !It->second.Duplicates.isZero()) {
+ Cut.reset();
+ break;
+ }
+ if (!Cut) {
+ Cut = ReductionCut{S, It->second.Lanes};
+ continue;
+ }
+ if (!isEquivBitcast(Cut->Src, S)) {
+ Cut.reset();
+ break;
+ }
+ if (!IsIdempotent && !(Cut->Elts & It->second.Lanes).isZero()) {
Cut.reset();
break;
}
- Cut = ReductionCut{S, It->second.Lanes};
+ Cut->Elts |= It->second.Lanes;
}
if (!Cut) {
for (Value *V : Nodes) {
diff --git a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
index f00ce63159d17..d38d5bef6dd18 100644
--- a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
+++ b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
@@ -435,3 +435,129 @@ define i32 @test_reduce_reconvergent_intermediate_v4i32(<4 x i32> %a) {
%e = extractelement <2 x i32> %c, i64 0
ret i32 %e
}
+
+; Equivalent bitcast sources with disjoint demanded lanes can be merged into
+; one complete reduction.
+define i32 @test_reduce_equivalent_bitcast_sources(<2 x i64> %a) {
+; CHECK-LABEL: define i32 @test_reduce_equivalent_bitcast_sources(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) {
+; CHECK-NEXT: [[BC0:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BC0]])
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %bc0 = bitcast <2 x i64> %a to <4 x i32>
+ %hi = shufflevector <4 x i32> %bc0, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
+
+ %bc1 = bitcast <2 x i64> %a to <4 x i32>
+ %sum0 = add <4 x i32> %hi, %bc1
+
+ %sh = shufflevector <4 x i32> %sum0, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %sum1 = add <4 x i32> %sum0, %sh
+ %r = extractelement <4 x i32> %sum1, i64 0
+ ret i32 %r
+}
+
+; Equivalent sources demand lane 0 through two
diff erent paths. Add is not
+; idempotent, so merging the lane sets would lose one contribution.
+define i32 @test_no_reduce_equivalent_bitcast_sources_overlap(<2 x i64> %a) {
+; CHECK-LABEL: define i32 @test_no_reduce_equivalent_bitcast_sources_overlap(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) {
+; CHECK-NEXT: [[BC0:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[LHS:%.*]] = shufflevector <4 x i32> [[BC0]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[BC1:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[RHS:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+; CHECK-NEXT: [[SUM0:%.*]] = add <4 x i32> [[LHS]], [[RHS]]
+; CHECK-NEXT: [[SH:%.*]] = shufflevector <4 x i32> [[SUM0]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[SUM1:%.*]] = add <4 x i32> [[SUM0]], [[SH]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[SUM1]], i64 0
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %bc0 = bitcast <2 x i64> %a to <4 x i32>
+ %lhs = shufflevector <4 x i32> %bc0, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+
+ %bc1 = bitcast <2 x i64> %a to <4 x i32>
+ %rhs = shufflevector <4 x i32> %bc1, <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+
+ %sum0 = add <4 x i32> %lhs, %rhs
+ %sh = shufflevector <4 x i32> %sum0, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %sum1 = add <4 x i32> %sum0, %sh
+ %r = extractelement <4 x i32> %sum1, i64 0
+ ret i32 %r
+}
+
+; The second equivalent source has an internally duplicated demanded lane.
+; Its lane set does not overlap the first source, so this specifically checks
+; the per-source Duplicates field rather than cross-source overlap.
+define i32 @test_no_reduce_later_equivalent_source_has_duplicates(<2 x i64> %a) {
+; CHECK-LABEL: define i32 @test_no_reduce_later_equivalent_source_has_duplicates(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) {
+; CHECK-NEXT: [[BC0:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[LHS:%.*]] = shufflevector <4 x i32> [[BC0]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[BC1:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[RHS:%.*]] = shufflevector <4 x i32> [[BC1]], <4 x i32> poison, <4 x i32> <i32 2, i32 2, i32 poison, i32 poison>
+; CHECK-NEXT: [[SUM0:%.*]] = add <4 x i32> [[LHS]], [[RHS]]
+; CHECK-NEXT: [[SH:%.*]] = shufflevector <4 x i32> [[SUM0]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[SUM1:%.*]] = add <4 x i32> [[SUM0]], [[SH]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[SUM1]], i64 0
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %bc0 = bitcast <2 x i64> %a to <4 x i32>
+ %lhs = shufflevector <4 x i32> %bc0, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+
+ %bc1 = bitcast <2 x i64> %a to <4 x i32>
+ %rhs = shufflevector <4 x i32> %bc1, <4 x i32> poison, <4 x i32> <i32 2, i32 2, i32 poison, i32 poison>
+
+ %sum0 = add <4 x i32> %lhs, %rhs
+ %sh = shufflevector <4 x i32> %sum0, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %sum1 = add <4 x i32> %sum0, %sh
+ %r = extractelement <4 x i32> %sum1, i64 0
+ ret i32 %r
+}
+
+; Bitcasts with the same result type but
diff erent underlying values are
+;
diff erent logical sources and cannot be merged.
+define i32 @test_no_reduce_
diff erent_bitcast_sources(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: define i32 @test_no_reduce_
diff erent_bitcast_sources(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT: [[BC0:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[HI:%.*]] = shufflevector <4 x i32> [[BC0]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT: [[BC1:%.*]] = bitcast <2 x i64> [[B]] to <4 x i32>
+; CHECK-NEXT: [[SUM0:%.*]] = add <4 x i32> [[HI]], [[BC1]]
+; CHECK-NEXT: [[SH:%.*]] = shufflevector <4 x i32> [[SUM0]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[SUM1:%.*]] = add <4 x i32> [[SUM0]], [[SH]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[SUM1]], i64 0
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %bc0 = bitcast <2 x i64> %a to <4 x i32>
+ %hi = shufflevector <4 x i32> %bc0, <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
+
+ %bc1 = bitcast <2 x i64> %b to <4 x i32>
+ %sum0 = add <4 x i32> %hi, %bc1
+
+ %sh = shufflevector <4 x i32> %sum0, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %sum1 = add <4 x i32> %sum0, %sh
+ %r = extractelement <4 x i32> %sum1, i64 0
+ ret i32 %r
+}
+
+; Overlapping lanes are safe for an idempotent operation.
+define i32 @test_reduce_idempotent_equivalent_bitcast_sources_overlap(<2 x i64> %a) {
+; CHECK-LABEL: define i32 @test_reduce_idempotent_equivalent_bitcast_sources_overlap(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) {
+; CHECK-NEXT: [[BC0:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT: [[PARTIAL:%.*]] = shufflevector <4 x i32> [[BC0]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.or.v3i32(<3 x i32> [[PARTIAL]])
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %bc0 = bitcast <2 x i64> %a to <4 x i32>
+ %lhs = shufflevector <4 x i32> %bc0, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+
+ %bc1 = bitcast <2 x i64> %a to <4 x i32>
+ %rhs = shufflevector <4 x i32> %bc1, <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
+
+ %or0 = or <4 x i32> %lhs, %rhs
+ %sh = shufflevector <4 x i32> %or0, <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+ %or1 = or <4 x i32> %or0, %sh
+ %r = extractelement <4 x i32> %or1, i64 0
+ ret i32 %r
+}
More information about the llvm-commits
mailing list