[llvm] Fold patterns which uses v4i32 type for equality comparison on v2i64 type (PR #184328)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 3 03:51:14 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Fuad Ismail (fuad1502)
<details>
<summary>Changes</summary>
This PR partially recognize v2i64 comparison patterns described in #<!-- -->32961. In particular, this PR handles the equality comparison (`eq`, `ne`).
The recognized pattern along with the transformation proof can be seen here: https://alive2.llvm.org/ce/z/8QqwW1
New lit test:
- Transforms/VectorCombine/fold-v4i32-eq-shuffle-and-to-v2i64-eq.ll
---
Full diff: https://github.com/llvm/llvm-project/pull/184328.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VectorCombine.cpp (+97)
- (added) llvm/test/Transforms/VectorCombine/fold-v4i32-eq-shuffle-and-to-v2i64-eq.ll (+174)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 1f37e435b8080..fd2ca3ef7c901 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -27,12 +27,15 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -153,6 +156,7 @@ class VectorCombine {
bool foldEquivalentReductionCmp(Instruction &I);
bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
bool foldInterleaveIntrinsics(Instruction &I);
+ bool foldEqualShuffleAnd(Instruction &I);
bool shrinkType(Instruction &I);
bool shrinkLoadForShuffles(Instruction &I);
bool shrinkPhiOfShuffles(Instruction &I);
@@ -5435,6 +5439,93 @@ bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
return true;
}
+// Prior to SSE4.1, performing equality comparison on v2i64 types require a
+// comparison on v4i32 types using the following pattern:
+//
+// %3 = icmp eq <4 x i32> %1, %2
+//
+// %4 = sext <4 x i1> %3 to <4 x i32>
+//
+// %5 = shufflevector <4 x i32> %4, <4 x i32> poison, <4 x i32> <i32 1, i32 0,
+// i32 3, i32 2>
+//
+// %6 = select <4 x i1> %3, <4 x i32> %5, <4 x i32> zeroinitializer
+//
+// OR
+//
+// %6 = and <4 x i32> %sext, %shuffle
+//
+// We should detect such patterns and fold them into:
+//
+// %3 = bitcast <4 x i32> %1 to <2 x i64>
+//
+// %4 = bitcast <4 x i32> %2 to <2 x i64>
+//
+// %5 = icmp eq <2 x i64> %3, %4
+//
+// %6 = bitcast <2 x i64> %5 to <4 x i32>
+//
+bool VectorCombine::foldEqualShuffleAnd(Instruction &I) {
+ Value *Equal, *Shuffle, *L, *R;
+ CmpPredicate Pred;
+ SmallVector<int> Mask = {1, 0, 3, 2};
+
+ // Check pattern existance
+ if (!match(&I, m_CombineOr(m_c_And(m_SExt(m_Value(Equal)),
+ m_SExtOrSelf(m_Value(Shuffle))),
+ m_Select(m_Value(Equal),
+ m_SExtOrSelf(m_Value(Shuffle)),
+ m_ZeroInt()))) ||
+ !match(Shuffle, m_Shuffle(m_SExtOrSelf(m_Specific(Equal)), m_Poison(),
+ m_SpecificMask(Mask))) ||
+ !match(Equal, m_ICmp(Pred, m_Value(L), m_Value(R))) ||
+ !CmpInst::isEquality(Pred))
+ return false;
+
+ // Check argument type
+ auto *OldVecType = cast<VectorType>(L->getType());
+
+ if (OldVecType->isScalableTy() ||
+ !OldVecType->getElementType()->isIntegerTy())
+ return false;
+
+ int ElementCount = OldVecType->getElementCount().getFixedValue();
+ int ElementBitWidth = OldVecType->getElementType()->getIntegerBitWidth();
+
+ if (ElementCount != 4 || ElementBitWidth != 32)
+ return false;
+
+ // Check uses outside pattern
+ if (!Shuffle->hasOneUse())
+ return false;
+
+ for (auto *U : Equal->users()) {
+ if (U == &I || U == Shuffle)
+ continue;
+ if (!isa<llvm::CastInst>(U))
+ return false;
+ for (auto *U : U->users())
+ if (U != &I && U != Shuffle)
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "VC: Found equal-shuffle-and pattern" << '\n');
+
+ // Perform folding
+ IRBuilder Builder(&I);
+ auto *NewElementType = IntegerType::get(I.getContext(), ElementBitWidth * 2);
+ auto *NewVecType = VectorType::get(NewElementType, ElementCount / 2, false);
+ auto *BitCastL = Builder.CreateBitCast(L, NewVecType);
+ auto *BitCastR = Builder.CreateBitCast(R, NewVecType);
+ auto *Cmp = Builder.CreateICmp(Pred, BitCastL, BitCastR);
+ auto *SExt = Builder.CreateSExt(Cmp, NewVecType);
+ auto *BitCastCmp = Builder.CreateBitCast(SExt, OldVecType);
+
+ replaceValue(I, *BitCastCmp);
+
+ return false;
+}
+
// Attempt to shrink loads that are only used by shufflevector instructions.
bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
auto *OldLoad = dyn_cast<LoadInst>(&I);
@@ -5777,11 +5868,17 @@ bool VectorCombine::run() {
return true;
if (foldBitOpOfCastConstant(I))
return true;
+ if (foldEqualShuffleAnd(I))
+ return true;
break;
case Instruction::PHI:
if (shrinkPhiOfShuffles(I))
return true;
break;
+ case Instruction::Select:
+ if (foldEqualShuffleAnd(I))
+ return true;
+ break;
default:
if (shrinkType(I))
return true;
diff --git a/llvm/test/Transforms/VectorCombine/fold-v4i32-eq-shuffle-and-to-v2i64-eq.ll b/llvm/test/Transforms/VectorCombine/fold-v4i32-eq-shuffle-and-to-v2i64-eq.ll
new file mode 100644
index 0000000000000..4e13afd360673
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/fold-v4i32-eq-shuffle-and-to-v2i64-eq.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=vector-combine -S | FileCheck %s
+
+define <4 x i32> @cmpeq_epi64_select(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_select(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext <2 x i1> [[TMP3]] to <2 x i64>
+; CHECK-NEXT: [[SELECT:%.*]] = bitcast <2 x i64> [[TMP4]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[SELECT]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %select = select <4 x i1> %cmp, <4 x i32> %shuffle, <4 x i32> zeroinitializer
+ ret <4 x i32> %select
+}
+
+define <4 x i32> @cmpeq_epi64_and(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_and(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext <2 x i1> [[TMP3]] to <2 x i64>
+; CHECK-NEXT: [[AND:%.*]] = bitcast <2 x i64> [[TMP4]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[AND]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %and = and <4 x i32> %sext, %shuffle
+ ret <4 x i32> %and
+}
+
+define <4 x i32> @cmpeq_epi64_commutated_and(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_commutated_and(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[B]] to <2 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sext <2 x i1> [[TMP3]] to <2 x i64>
+; CHECK-NEXT: [[AND:%.*]] = bitcast <2 x i64> [[TMP4]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[AND]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %and = and <4 x i32> %shuffle, %sext
+ ret <4 x i32> %and
+}
+
+declare void @use.v4i1(<4 x i1>)
+declare void @use.v4i32(<4 x i32>)
+
+define <4 x i32> @cmpeq_epi64_multi_use_cmp(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_multi_use_cmp(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: call void @use.v4i1(<4 x i1> [[CMP]])
+; CHECK-NEXT: [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[AND:%.*]] = and <4 x i32> [[SHUFFLE]], [[SEXT]]
+; CHECK-NEXT: ret <4 x i32> [[AND]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ call void @use.v4i1(<4 x i1> %cmp)
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %and = and <4 x i32> %shuffle, %sext
+ ret <4 x i32> %and
+}
+
+define <4 x i32> @cmpeq_epi64_multi_use_sext(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_multi_use_sext(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NEXT: call void @use.v4i32(<4 x i32> [[SEXT]])
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[AND:%.*]] = and <4 x i32> [[SHUFFLE]], [[SEXT]]
+; CHECK-NEXT: ret <4 x i32> [[AND]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ call void @use.v4i32(<4 x i32> %sext)
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %and = and <4 x i32> %shuffle, %sext
+ ret <4 x i32> %and
+}
+
+define <4 x i32> @cmpeq_epi64_multi_use_shuffle(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_multi_use_shuffle(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: call void @use.v4i32(<4 x i32> [[SHUFFLE]])
+; CHECK-NEXT: [[AND:%.*]] = and <4 x i32> [[SHUFFLE]], [[SEXT]]
+; CHECK-NEXT: ret <4 x i32> [[AND]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ call void @use.v4i32(<4 x i32> %shuffle)
+ %and = and <4 x i32> %shuffle, %sext
+ ret <4 x i32> %and
+}
+
+define <4 x i32> @cmpeq_epi64_select_neg_0(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_select_neg_0(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i1> [[CMP]], <4 x i1> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[SHUFFLE:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[SELECT:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[SHUFFLE]], <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x i32> [[SELECT]]
+;
+ %cmp = icmp sgt <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %select = select <4 x i1> %cmp, <4 x i32> %shuffle, <4 x i32> zeroinitializer
+ ret <4 x i32> %select
+}
+
+define <4 x i32> @cmpeq_epi64_and_neg_1(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_and_neg_1(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[SEXT:%.*]] = zext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[AND:%.*]] = and <4 x i32> [[SEXT]], [[SHUFFLE]]
+; CHECK-NEXT: ret <4 x i32> [[AND]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = zext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %and = and <4 x i32> %sext, %shuffle
+ ret <4 x i32> %and
+}
+
+define <4 x i32> @cmpeq_epi64_select_neg_2(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_select_neg_2(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i1> [[CMP]], <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[SHUFFLE:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[SELECT:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[SHUFFLE]], <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x i32> [[SELECT]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ %select = select <4 x i1> %cmp, <4 x i32> %shuffle, <4 x i32> zeroinitializer
+ ret <4 x i32> %select
+}
+
+define <4 x i32> @cmpeq_epi64_select_neg_3(<4 x i32> noundef %a, <4 x i32> noundef %b) {
+; CHECK-LABEL: define <4 x i32> @cmpeq_epi64_select_neg_3(
+; CHECK-SAME: <4 x i32> noundef [[A:%.*]], <4 x i32> noundef [[B:%.*]]) {
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[SEXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[SEXT]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[SELECT:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[SHUFFLE]], <4 x i32> [[SEXT]]
+; CHECK-NEXT: ret <4 x i32> [[SELECT]]
+;
+ %cmp = icmp eq <4 x i32> %a, %b
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %shuffle = shufflevector <4 x i32> %sext, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %select = select <4 x i1> %cmp, <4 x i32> %shuffle, <4 x i32> %sext
+ ret <4 x i32> %select
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/184328
More information about the llvm-commits
mailing list