[llvm] [VectorCombine] Handle widening/narrowing bitcasts in foldShuffleToIdentity (PR #187870)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 09:05:39 PDT 2026
https://github.com/AbdallahRashed updated https://github.com/llvm/llvm-project/pull/187870
>From e64af7231550a8244280f575c9463ace4702abb7 Mon Sep 17 00:00:00 2001
From: AbdallahRashed <abdallah.mrashed at gmail.com>
Date: Sun, 26 Apr 2026 18:48:21 +0200
Subject: [PATCH] [VectorCombine] Handle widening/narrowing bitcasts in
foldShuffleToIdentity
Track lane indices through vector bitcasts that change element count
in foldShuffleToIdentity. Widening bitcasts (e.g. <2 x i32> -> <4 x i16>)
compress R consecutive destination lanes into one source lane, while
narrowing bitcasts (e.g. <4 x i16> -> <2 x i32>) expand each destination
lane into R source lanes.
Also fix identity check, splat mask, and destination type construction
to use Item.size() instead of Ty->getNumElements(), since the Item
vector changes size when passing through element-count-changing bitcasts.
When foldShuffleToIdentity only traverses a single element-count-changing
bitcast (NumVisited == 2), it just commutes the bitcast and shuffle
without any real simplification. Skip this case to avoid an infinite
loop with foldBitcastShuffle which does the reverse transform.
Fixes #96884.
---
.../Transforms/Vectorize/VectorCombine.cpp | 134 ++++++++++-
.../Transforms/PhaseOrdering/X86/pr67803.ll | 11 +-
.../AArch64/shuffletoidentity.ll | 71 ++++++
.../X86/shuffletoidentity-bitcast.ll | 217 ++++++++++++++++++
4 files changed, 419 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/Transforms/VectorCombine/X86/shuffletoidentity-bitcast.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 29eced7ee9de1..845a012dae700 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -3520,7 +3520,7 @@ generateNewInstTree(ArrayRef<InstLane> Item, Use *From, FixedVectorType *Ty,
return FrontV;
}
if (SplatLeafs.contains(std::make_pair(FrontV, From))) {
- SmallVector<int, 16> Mask(Ty->getNumElements(), FrontLane);
+ SmallVector<int, 16> Mask(Item.size(), FrontLane);
return Builder.CreateShuffleVector(FrontV, Mask);
}
if (ConcatLeafs.contains(std::make_pair(FrontV, From))) {
@@ -3544,6 +3544,50 @@ generateNewInstTree(ArrayRef<InstLane> Item, Use *From, FixedVectorType *Ty,
}
auto *I = cast<Instruction>(FrontV);
+
+ // Handle vector bitcasts that change element count. We cannot use
+ // generateInstLaneVectorFromOperand for these because the lane indices
+ // don't map 1:1 through the bitcast.
+ if (auto *BitCast = dyn_cast<BitCastInst>(I)) {
+ auto *BCDstTy = dyn_cast<FixedVectorType>(BitCast->getDestTy());
+ auto *BCSrcTy = dyn_cast<FixedVectorType>(BitCast->getSrcTy());
+ if (BCDstTy && BCSrcTy &&
+ BCDstTy->getNumElements() != BCSrcTy->getNumElements()) {
+ unsigned DstElts = BCDstTy->getNumElements();
+ unsigned SrcElts = BCSrcTy->getNumElements();
+ SmallVector<InstLane> NewItem;
+ if (DstElts > SrcElts) {
+ // Widening: compress operand Item.
+ unsigned R = DstElts / SrcElts;
+ for (unsigned Idx = 0, E = Item.size(); Idx < E; Idx += R) {
+ auto [V, Lane] = Item[Idx];
+ if (!V)
+ NewItem.push_back({nullptr, PoisonMaskElem});
+ else
+ NewItem.push_back(lookThroughShuffles(
+ cast<Instruction>(V)->getOperand(0), Lane / R));
+ }
+ } else {
+ // Narrowing: expand operand Item.
+ unsigned R = SrcElts / DstElts;
+ for (auto [V, Lane] : Item) {
+ if (!V) {
+ for (unsigned J = 0; J < R; ++J)
+ NewItem.push_back({nullptr, PoisonMaskElem});
+ } else {
+ Value *Op = cast<Instruction>(V)->getOperand(0);
+ for (unsigned J = 0; J < R; ++J)
+ NewItem.push_back(lookThroughShuffles(Op, Lane * R + J));
+ }
+ }
+ }
+ Value *Op = generateNewInstTree(NewItem, &BitCast->getOperandUse(0), Ty,
+ IdentityLeafs, SplatLeafs, ConcatLeafs,
+ Builder, TTI);
+ return Builder.CreateBitCast(
+ Op, FixedVectorType::get(BCDstTy->getScalarType(), Item.size()));
+ }
+ }
auto *II = dyn_cast<IntrinsicInst>(I);
unsigned NumOps = I->getNumOperands() - (II ? 1 : 0);
SmallVector<Value *> Ops(NumOps);
@@ -3564,7 +3608,7 @@ generateNewInstTree(ArrayRef<InstLane> Item, Use *From, FixedVectorType *Ty,
ValueList.push_back(Lane.first);
Type *DstTy =
- FixedVectorType::get(I->getType()->getScalarType(), Ty->getNumElements());
+ FixedVectorType::get(I->getType()->getScalarType(), Item.size());
if (auto *BI = dyn_cast<BinaryOperator>(I)) {
auto *Value = Builder.CreateBinOp((Instruction::BinaryOps)BI->getOpcode(),
Ops[0], Ops[1]);
@@ -3614,6 +3658,7 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
Worklist.push_back(std::make_pair(Start, &*I.use_begin()));
DenseSet<std::pair<Value *, Use *>> IdentityLeafs, SplatLeafs, ConcatLeafs;
unsigned NumVisited = 0;
+ bool TraversedElCountChangingBitcast = false;
while (!Worklist.empty()) {
if (++NumVisited > MaxInstrsToScan)
@@ -3637,7 +3682,7 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
// Look for an identity value.
if (FrontLane == 0 &&
cast<FixedVectorType>(FrontV->getType())->getNumElements() ==
- Ty->getNumElements() &&
+ Item.size() &&
all_of(drop_begin(enumerate(Item)), [IsEquiv, Item](const auto &E) {
Value *FrontV = Item.front().first;
return !E.value().first || (IsEquiv(E.value().first, FrontV) &&
@@ -3718,14 +3763,75 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
&cast<Instruction>(FrontV)->getOperandUse(0));
continue;
} else if (auto *BitCast = dyn_cast<BitCastInst>(FrontV)) {
- // TODO: Handle vector widening/narrowing bitcasts.
- auto *DstTy = dyn_cast<FixedVectorType>(BitCast->getDestTy());
- auto *SrcTy = dyn_cast<FixedVectorType>(BitCast->getSrcTy());
- if (DstTy && SrcTy &&
- SrcTy->getNumElements() == DstTy->getNumElements()) {
- Worklist.emplace_back(generateInstLaneVectorFromOperand(Item, 0),
- &BitCast->getOperandUse(0));
- continue;
+ auto *BCDstTy = dyn_cast<FixedVectorType>(BitCast->getDestTy());
+ auto *BCSrcTy = dyn_cast<FixedVectorType>(BitCast->getSrcTy());
+ if (BCDstTy && BCSrcTy) {
+ unsigned DstElts = BCDstTy->getNumElements();
+ unsigned SrcElts = BCSrcTy->getNumElements();
+ if (DstElts == SrcElts) {
+ // Same element count - simple pass-through.
+ Worklist.emplace_back(generateInstLaneVectorFromOperand(Item, 0),
+ &BitCast->getOperandUse(0));
+ continue;
+ }
+ if (DstElts > SrcElts && DstElts % SrcElts == 0) {
+ // Widening bitcast (e.g. <2 x i32> -> <4 x i16>). Compress
+ // consecutive groups of R destination lanes into one source
+ // lane.
+ unsigned R = DstElts / SrcElts;
+ SmallVector<InstLane> NItem;
+ bool Valid = true;
+ for (unsigned Idx = 0, E = Item.size(); Idx < E; Idx += R) {
+ auto [V0, L0] = Item[Idx];
+ if (!V0) {
+ if (any_of(ArrayRef(Item).slice(Idx + 1, R - 1),
+ [](InstLane IL) { return IL.first != nullptr; })) {
+ Valid = false;
+ break;
+ }
+ NItem.push_back({nullptr, PoisonMaskElem});
+ continue;
+ }
+ if (L0 % R != 0) {
+ Valid = false;
+ break;
+ }
+ for (unsigned J = 1; J < R; ++J) {
+ auto [VJ, LJ] = Item[Idx + J];
+ if (!VJ || VJ != V0 || LJ != L0 + (int)J) {
+ Valid = false;
+ break;
+ }
+ }
+ if (!Valid)
+ break;
+ NItem.push_back(lookThroughShuffles(
+ cast<Instruction>(V0)->getOperand(0), L0 / R));
+ }
+ if (Valid) {
+ TraversedElCountChangingBitcast = true;
+ Worklist.emplace_back(NItem, &BitCast->getOperandUse(0));
+ continue;
+ }
+ } else if (SrcElts > DstElts && SrcElts % DstElts == 0) {
+ // Narrowing bitcast (e.g. <4 x i16> -> <2 x i32>). Expand
+ // each destination lane into R source lanes.
+ unsigned R = SrcElts / DstElts;
+ SmallVector<InstLane> NItem;
+ for (auto [V, Lane] : Item) {
+ if (!V) {
+ for (unsigned J = 0; J < R; ++J)
+ NItem.push_back({nullptr, PoisonMaskElem});
+ continue;
+ }
+ Value *Op = cast<Instruction>(V)->getOperand(0);
+ for (unsigned J = 0; J < R; ++J)
+ NItem.push_back(lookThroughShuffles(Op, Lane * R + J));
+ }
+ TraversedElCountChangingBitcast = true;
+ Worklist.emplace_back(NItem, &BitCast->getOperandUse(0));
+ continue;
+ }
}
} else if (auto *Sel = dyn_cast<SelectInst>(FrontV)) {
Worklist.emplace_back(generateInstLaneVectorFromOperand(Item, 0),
@@ -3768,6 +3874,12 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
if (NumVisited <= 1)
return false;
+ // If the only non-leaf node traversed was a single bitcast that changes
+ // element count, the fold would just commute the bitcast and shuffle.
+ // foldBitcastShuffle does the reverse transform, causing an infinite loop.
+ if (NumVisited == 2 && TraversedElCountChangingBitcast)
+ return false;
+
LLVM_DEBUG(dbgs() << "Found a superfluous identity shuffle: " << I << "\n");
// If we got this far, we know the shuffles are superfluous and can be
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll
index 7b690341fb805..f142f5ee1596e 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) {
; CHECK-LABEL: @PR67803(
@@ -77,3 +77,8 @@ entry:
%shuffle.i23 = shufflevector <2 x i64> %18, <2 x i64> %26, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x i64> %shuffle.i23
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; AVX2: {{.*}}
+; AVX512: {{.*}}
+; SSE: {{.*}}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index 72b75148f027e..fddc130975ba2 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -1356,3 +1356,74 @@ define <48 x i8> @concat_splats(<64 x i8> %wide.vec, <64 x i8> %wide.vec117) {
declare <4 x i64> @llvm.fshl.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
declare void @use(<4 x i8>)
+
+; Narrowing bitcasts (i16->i32) around a sub, then widening bitcasts (i32->i16)
+; back, split into halves and re-joined. Should fold to a single bitcast+sub+bitcast.
+define <8 x i16> @bitcast_narrowwiden_sub(<8 x i16> %a) {
+; CHECK-LABEL: @bitcast_narrowwiden_sub(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = sub <4 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %lo = shufflevector <8 x i16> %a, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %hi = shufflevector <8 x i16> %a, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %bc_lo = bitcast <4 x i16> %lo to <2 x i32>
+ %bc_hi = bitcast <4 x i16> %hi to <2 x i32>
+ %neg_lo = sub <2 x i32> zeroinitializer, %bc_lo
+ %neg_hi = sub <2 x i32> zeroinitializer, %bc_hi
+ %bc_lo2 = bitcast <2 x i32> %neg_lo to <4 x i16>
+ %bc_hi2 = bitcast <2 x i32> %neg_hi to <4 x i16>
+ %result = shufflevector <4 x i16> %bc_lo2, <4 x i16> %bc_hi2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ ret <8 x i16> %result
+}
+
+; Two-operand add through narrowing+widening bitcasts.
+define <8 x i16> @bitcast_narrowwiden_add(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @bitcast_narrowwiden_add(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i16> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TMP4]]
+;
+ %a_lo = shufflevector <8 x i16> %a, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %a_hi = shufflevector <8 x i16> %a, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %b_lo = shufflevector <8 x i16> %b, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %b_hi = shufflevector <8 x i16> %b, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %bc_a_lo = bitcast <4 x i16> %a_lo to <2 x i32>
+ %bc_a_hi = bitcast <4 x i16> %a_hi to <2 x i32>
+ %bc_b_lo = bitcast <4 x i16> %b_lo to <2 x i32>
+ %bc_b_hi = bitcast <4 x i16> %b_hi to <2 x i32>
+ %add_lo = add <2 x i32> %bc_a_lo, %bc_b_lo
+ %add_hi = add <2 x i32> %bc_a_hi, %bc_b_hi
+ %bc_lo = bitcast <2 x i32> %add_lo to <4 x i16>
+ %bc_hi = bitcast <2 x i32> %add_hi to <4 x i16>
+ %result = shufflevector <4 x i16> %bc_lo, <4 x i16> %bc_hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ ret <8 x i16> %result
+}
+
+; Ratio of 4: i8 -> i32 narrowing bitcast with add, then widening back.
+define <16 x i8> @bitcast_narrow4_widen4_add(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @bitcast_narrow4_widen4_add(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT: ret <16 x i8> [[TMP4]]
+;
+ %a_lo = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %a_hi = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %b_lo = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %b_hi = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %bc_a_lo = bitcast <8 x i8> %a_lo to <2 x i32>
+ %bc_a_hi = bitcast <8 x i8> %a_hi to <2 x i32>
+ %bc_b_lo = bitcast <8 x i8> %b_lo to <2 x i32>
+ %bc_b_hi = bitcast <8 x i8> %b_hi to <2 x i32>
+ %add_lo = add <2 x i32> %bc_a_lo, %bc_b_lo
+ %add_hi = add <2 x i32> %bc_a_hi, %bc_b_hi
+ %bc_lo = bitcast <2 x i32> %add_lo to <8 x i8>
+ %bc_hi = bitcast <2 x i32> %add_hi to <8 x i8>
+ %result = shufflevector <8 x i8> %bc_lo, <8 x i8> %bc_hi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ ret <16 x i8> %result
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffletoidentity-bitcast.ll b/llvm/test/Transforms/VectorCombine/X86/shuffletoidentity-bitcast.ll
new file mode 100644
index 0000000000000..43c06caab8cb4
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffletoidentity-bitcast.ll
@@ -0,0 +1,217 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=vector-combine -S %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt -passes=vector-combine -S %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+; Tests for foldShuffleToIdentity handling of bitcasts that change element count.
+; Motivated by issue #96884: SSE/AVX code uses vXi64 for __m128i/__m256i types,
+; requiring bitcasts to different element-count types for operations.
+
+; Narrowing bitcast: split <4 x i32> through <2 x i64> bitcasts and rejoin.
+define <4 x i32> @bitcast_i32_to_i64_add(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @bitcast_i32_to_i64_add(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i32> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i64> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP4]]
+;
+ %a_lo = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+ %a_hi = shufflevector <4 x i32> %a, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+ %b_lo = shufflevector <4 x i32> %b, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+ %b_hi = shufflevector <4 x i32> %b, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+ %bc_a_lo = bitcast <2 x i32> %a_lo to <1 x i64>
+ %bc_a_hi = bitcast <2 x i32> %a_hi to <1 x i64>
+ %bc_b_lo = bitcast <2 x i32> %b_lo to <1 x i64>
+ %bc_b_hi = bitcast <2 x i32> %b_hi to <1 x i64>
+ %add_lo = add <1 x i64> %bc_a_lo, %bc_b_lo
+ %add_hi = add <1 x i64> %bc_a_hi, %bc_b_hi
+ %bc_lo = bitcast <1 x i64> %add_lo to <2 x i32>
+ %bc_hi = bitcast <1 x i64> %add_hi to <2 x i32>
+ %result = shufflevector <2 x i32> %bc_lo, <2 x i32> %bc_hi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x i32> %result
+}
+
+; Widening bitcast: split <2 x i64> through <4 x i32> bitcasts and rejoin.
+; This is the typical SSE pattern: __m128i is <2 x i64>, operations on <4 x i32>.
+define <2 x i64> @bitcast_i64_to_i32_xor(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcast_i64_to_i32_xor(
+; CHECK-NEXT: [[B:%.*]] = xor <2 x i64> [[A:%.*]], [[B1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[B]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[TMP2]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[TMP4]]
+;
+ %a_lo = shufflevector <2 x i64> %a, <2 x i64> poison, <1 x i32> <i32 0>
+ %a_hi = shufflevector <2 x i64> %a, <2 x i64> poison, <1 x i32> <i32 1>
+ %b_lo = shufflevector <2 x i64> %b, <2 x i64> poison, <1 x i32> <i32 0>
+ %b_hi = shufflevector <2 x i64> %b, <2 x i64> poison, <1 x i32> <i32 1>
+ %bc_a_lo = bitcast <1 x i64> %a_lo to <2 x i32>
+ %bc_a_hi = bitcast <1 x i64> %a_hi to <2 x i32>
+ %bc_b_lo = bitcast <1 x i64> %b_lo to <2 x i32>
+ %bc_b_hi = bitcast <1 x i64> %b_hi to <2 x i32>
+ %xor_lo = xor <2 x i32> %bc_a_lo, %bc_b_lo
+ %xor_hi = xor <2 x i32> %bc_a_hi, %bc_b_hi
+ %bc_lo = bitcast <2 x i32> %xor_lo to <1 x i64>
+ %bc_hi = bitcast <2 x i32> %xor_hi to <1 x i64>
+ %result = shufflevector <1 x i64> %bc_lo, <1 x i64> %bc_hi, <2 x i32> <i32 0, i32 1>
+ ret <2 x i64> %result
+}
+
+; i64 -> i16: wider ratio (4x), split and rejoin through <8 x i16> operations.
+define <2 x i64> @bitcast_i64_to_i16_shl(<2 x i64> %a) {
+; SSE-LABEL: @bitcast_i64_to_i16_shl(
+; SSE-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <8 x i16>
+; SSE-NEXT: [[BC_LO:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[A]] to <8 x i16>
+; SSE-NEXT: [[BC_HI:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT: [[SHL_LO:%.*]] = shl <4 x i16> [[BC_LO]], splat (i16 3)
+; SSE-NEXT: [[SHL_HI:%.*]] = shl <4 x i16> [[BC_HI]], splat (i16 3)
+; SSE-NEXT: [[RES_LO:%.*]] = bitcast <4 x i16> [[SHL_LO]] to <1 x i64>
+; SSE-NEXT: [[RES_HI:%.*]] = bitcast <4 x i16> [[SHL_HI]] to <1 x i64>
+; SSE-NEXT: [[RESULT:%.*]] = shufflevector <1 x i64> [[RES_LO]], <1 x i64> [[RES_HI]], <2 x i32> <i32 0, i32 1>
+; SSE-NEXT: ret <2 x i64> [[RESULT]]
+;
+; AVX-LABEL: @bitcast_i64_to_i16_shl(
+; AVX-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <8 x i16>
+; AVX-NEXT: [[TMP2:%.*]] = shl <8 x i16> [[TMP1]], splat (i16 3)
+; AVX-NEXT: [[RESULT:%.*]] = bitcast <8 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT: ret <2 x i64> [[RESULT]]
+;
+ %a_lo = shufflevector <2 x i64> %a, <2 x i64> poison, <1 x i32> <i32 0>
+ %a_hi = shufflevector <2 x i64> %a, <2 x i64> poison, <1 x i32> <i32 1>
+ %bc_lo = bitcast <1 x i64> %a_lo to <4 x i16>
+ %bc_hi = bitcast <1 x i64> %a_hi to <4 x i16>
+ %shl_lo = shl <4 x i16> %bc_lo, splat (i16 3)
+ %shl_hi = shl <4 x i16> %bc_hi, splat (i16 3)
+ %res_lo = bitcast <4 x i16> %shl_lo to <1 x i64>
+ %res_hi = bitcast <4 x i16> %shl_hi to <1 x i64>
+ %result = shufflevector <1 x i64> %res_lo, <1 x i64> %res_hi, <2 x i32> <i32 0, i32 1>
+ ret <2 x i64> %result
+}
+
+; i64 -> i8: ratio of 8, split __m128i into halves, byte-shift, rejoin.
+define <2 x i64> @bitcast_i64_to_i8_add(<2 x i64> %a, <2 x i64> %b) {
+; SSE-LABEL: @bitcast_i64_to_i8_add(
+; SSE-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <16 x i8>
+; SSE-NEXT: [[BC_A_LO:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
+; SSE-NEXT: [[BC_A_HI:%.*]] = shufflevector <16 x i8> [[TMP2]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT: [[TMP3:%.*]] = bitcast <2 x i64> [[B:%.*]] to <16 x i8>
+; SSE-NEXT: [[BC_B_LO:%.*]] = shufflevector <16 x i8> [[TMP3]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT: [[TMP4:%.*]] = bitcast <2 x i64> [[B]] to <16 x i8>
+; SSE-NEXT: [[BC_B_HI:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT: [[ADD_LO:%.*]] = add <8 x i8> [[BC_A_LO]], [[BC_B_LO]]
+; SSE-NEXT: [[ADD_HI:%.*]] = add <8 x i8> [[BC_A_HI]], [[BC_B_HI]]
+; SSE-NEXT: [[BC_LO:%.*]] = bitcast <8 x i8> [[ADD_LO]] to <1 x i64>
+; SSE-NEXT: [[BC_HI:%.*]] = bitcast <8 x i8> [[ADD_HI]] to <1 x i64>
+; SSE-NEXT: [[RESULT:%.*]] = shufflevector <1 x i64> [[BC_LO]], <1 x i64> [[BC_HI]], <2 x i32> <i32 0, i32 1>
+; SSE-NEXT: ret <2 x i64> [[RESULT]]
+;
+; AVX-LABEL: @bitcast_i64_to_i8_add(
+; AVX-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <16 x i8>
+; AVX-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[B:%.*]] to <16 x i8>
+; AVX-NEXT: [[TMP3:%.*]] = add <16 x i8> [[TMP1]], [[TMP2]]
+; AVX-NEXT: [[RESULT:%.*]] = bitcast <16 x i8> [[TMP3]] to <2 x i64>
+; AVX-NEXT: ret <2 x i64> [[RESULT]]
+;
+ %a_lo = shufflevector <2 x i64> %a, <2 x i64> poison, <1 x i32> <i32 0>
+ %a_hi = shufflevector <2 x i64> %a, <2 x i64> poison, <1 x i32> <i32 1>
+ %b_lo = shufflevector <2 x i64> %b, <2 x i64> poison, <1 x i32> <i32 0>
+ %b_hi = shufflevector <2 x i64> %b, <2 x i64> poison, <1 x i32> <i32 1>
+ %bc_a_lo = bitcast <1 x i64> %a_lo to <8 x i8>
+ %bc_a_hi = bitcast <1 x i64> %a_hi to <8 x i8>
+ %bc_b_lo = bitcast <1 x i64> %b_lo to <8 x i8>
+ %bc_b_hi = bitcast <1 x i64> %b_hi to <8 x i8>
+ %add_lo = add <8 x i8> %bc_a_lo, %bc_b_lo
+ %add_hi = add <8 x i8> %bc_a_hi, %bc_b_hi
+ %bc_lo = bitcast <8 x i8> %add_lo to <1 x i64>
+ %bc_hi = bitcast <8 x i8> %add_hi to <1 x i64>
+ %result = shufflevector <1 x i64> %bc_lo, <1 x i64> %bc_hi, <2 x i32> <i32 0, i32 1>
+ ret <2 x i64> %result
+}
+
+; AVX pattern: <4 x i64> (__m256i) split into two <2 x i64> halves,
+; bitcast to <4 x i32>, operate, bitcast back, rejoin.
+define <4 x i64> @avx_bitcast_i64_to_i32_and(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @avx_bitcast_i64_to_i32_and(
+; CHECK-NEXT: [[B:%.*]] = and <4 x i64> [[A:%.*]], [[B1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x i64> [[B]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT: ret <4 x i64> [[TMP4]]
+;
+ %a_lo = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
+ %a_hi = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+ %b_lo = shufflevector <4 x i64> %b, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
+ %b_hi = shufflevector <4 x i64> %b, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+ %bc_a_lo = bitcast <2 x i64> %a_lo to <4 x i32>
+ %bc_a_hi = bitcast <2 x i64> %a_hi to <4 x i32>
+ %bc_b_lo = bitcast <2 x i64> %b_lo to <4 x i32>
+ %bc_b_hi = bitcast <2 x i64> %b_hi to <4 x i32>
+ %and_lo = and <4 x i32> %bc_a_lo, %bc_b_lo
+ %and_hi = and <4 x i32> %bc_a_hi, %bc_b_hi
+ %bc_lo = bitcast <4 x i32> %and_lo to <2 x i64>
+ %bc_hi = bitcast <4 x i32> %and_hi to <2 x i64>
+ %result = shufflevector <2 x i64> %bc_lo, <2 x i64> %bc_hi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x i64> %result
+}
+
+; Compare through bitcast: split, bitcast, icmp, select, bitcast back, rejoin.
+define <4 x i64> @avx_bitcast_select(<4 x i64> %a, <4 x i64> %b) {
+; SSE-LABEL: @avx_bitcast_select(
+; SSE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
+; SSE-NEXT: [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
+; SSE-NEXT: [[TMP3:%.*]] = icmp sgt <8 x i32> [[TMP1]], [[TMP2]]
+; SSE-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP1]], <8 x i32> [[TMP2]]
+; SSE-NEXT: [[RESULT:%.*]] = bitcast <8 x i32> [[TMP4]] to <4 x i64>
+; SSE-NEXT: ret <4 x i64> [[RESULT]]
+;
+; AVX-LABEL: @avx_bitcast_select(
+; AVX-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
+; AVX-NEXT: [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
+; AVX-NEXT: [[TMP3:%.*]] = icmp sgt <8 x i32> [[TMP1]], [[TMP2]]
+; AVX-NEXT: [[TMP4:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
+; AVX-NEXT: [[TMP5:%.*]] = bitcast <4 x i64> [[B]] to <8 x i32>
+; AVX-NEXT: [[TMP6:%.*]] = select <8 x i1> [[TMP3]], <8 x i32> [[TMP4]], <8 x i32> [[TMP5]]
+; AVX-NEXT: [[RESULT:%.*]] = bitcast <8 x i32> [[TMP6]] to <4 x i64>
+; AVX-NEXT: ret <4 x i64> [[RESULT]]
+;
+ %a_lo = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
+ %a_hi = shufflevector <4 x i64> %a, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+ %b_lo = shufflevector <4 x i64> %b, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
+ %b_hi = shufflevector <4 x i64> %b, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+ %bc_a_lo = bitcast <2 x i64> %a_lo to <4 x i32>
+ %bc_a_hi = bitcast <2 x i64> %a_hi to <4 x i32>
+ %bc_b_lo = bitcast <2 x i64> %b_lo to <4 x i32>
+ %bc_b_hi = bitcast <2 x i64> %b_hi to <4 x i32>
+ %cmp_lo = icmp sgt <4 x i32> %bc_a_lo, %bc_b_lo
+ %cmp_hi = icmp sgt <4 x i32> %bc_a_hi, %bc_b_hi
+ %sel_lo = select <4 x i1> %cmp_lo, <4 x i32> %bc_a_lo, <4 x i32> %bc_b_lo
+ %sel_hi = select <4 x i1> %cmp_hi, <4 x i32> %bc_a_hi, <4 x i32> %bc_b_hi
+ %bc_lo = bitcast <4 x i32> %sel_lo to <2 x i64>
+ %bc_hi = bitcast <4 x i32> %sel_hi to <2 x i64>
+ %result = shufflevector <2 x i64> %bc_lo, <2 x i64> %bc_hi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x i64> %result
+}
+
+; Negative test: misaligned lanes in widening bitcast should not fold.
+define <8 x i16> @bitcast_widen_misaligned(<8 x i16> %a) {
+; CHECK-LABEL: @bitcast_widen_misaligned(
+; CHECK-NEXT: [[LO:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT: [[BC_LO:%.*]] = bitcast <4 x i16> [[LO]] to <2 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[A]] to <4 x i32>
+; CHECK-NEXT: [[BC_HI:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[BC_LO]], <2 x i32> [[BC_HI]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP2]], [[TMP2]]
+; CHECK-NEXT: [[RESULT:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[RESULT]]
+;
+ %lo = shufflevector <8 x i16> %a, <8 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+ %hi = shufflevector <8 x i16> %a, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %bc_lo = bitcast <4 x i16> %lo to <2 x i32>
+ %bc_hi = bitcast <4 x i16> %hi to <2 x i32>
+ %add_lo = add <2 x i32> %bc_lo, %bc_lo
+ %add_hi = add <2 x i32> %bc_hi, %bc_hi
+ %res_lo = bitcast <2 x i32> %add_lo to <4 x i16>
+ %res_hi = bitcast <2 x i32> %add_hi to <4 x i16>
+ %result = shufflevector <4 x i16> %res_lo, <4 x i16> %res_hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ ret <8 x i16> %result
+}
More information about the llvm-commits
mailing list