[llvm] [InstCombine][VectorCombine] Move bitcast vp.load fold into VectorCombine (PR #200321)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 04:15:22 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/200321
>From a75434b03eee0e116647551a456e7e061ebdbc89 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 29 May 2026 11:47:55 +0800
Subject: [PATCH 1/3] Move vp.load bitcast fold to vectorcombine
---
.../InstCombine/InstCombineCalls.cpp | 41 ----------------
.../Transforms/Vectorize/VectorCombine.cpp | 48 +++++++++++++++++++
.../RISCV}/fold-vp-load.ll | 32 +++++++++----
3 files changed, 70 insertions(+), 51 deletions(-)
rename llvm/test/Transforms/{InstCombine => VectorCombine/RISCV}/fold-vp-load.ll (74%)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index dd1444197cf5d..03378b43ce305 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3956,47 +3956,6 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
}
break;
}
- case Intrinsic::vp_load: {
- auto *VPI = cast<VPIntrinsic>(II);
- // Fold away bit casts of the loaded value by loading the desired type,
- // if the mask is all-ones.
- Value *Mask = VPI->getMaskParam();
- Value *EVL = VPI->getVectorLengthParam();
- if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue() ||
- !II->hasOneUse())
- break;
-
- const DataLayout &DL = II->getDataLayout();
- auto *Cast = dyn_cast<CastInst>(II->user_back());
- if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Cast->getDestTy()))
- break;
- VectorType *OrigVecTy = cast<VectorType>(II->getType());
- Align OrigAlign =
- DL.getValueOrABITypeAlignment(VPI->getPointerAlignment(), OrigVecTy);
- ElementCount OrigVecCnt = OrigVecTy->getElementCount();
- VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
- ElementCount NewVecCnt = NewVecTy->getElementCount();
-
- // Right now we only support cases where the NewVec is longer, because for
- // cases where it's shorter, we have to be sure that EVL can be exactly
- // divided, otherwise it might yield incorrect results or even page faults
- // (if we round-up during the division).
- if (OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
- NewVecCnt.hasKnownScalarFactor(OrigVecCnt)) {
- unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
- Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
- Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
- CallInst *NewVP = Builder.CreateIntrinsic(
- NewVecTy, Intrinsic::vp_load,
- {VPI->getMemoryPointerParam(), NewMask, NewEVL});
- // Preserve the original alignment.
- NewVP->addParamAttrs(
- 0, AttrBuilder(VPI->getContext()).addAlignmentAttr(OrigAlign));
- replaceInstUsesWith(*Cast, NewVP);
- return eraseInstFromFunction(*Cast);
- }
- break;
- }
case Intrinsic::experimental_vp_reverse: {
Value *X;
Value *Vec = II->getArgOperand(0);
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 5a1c53f5d7fd1..427db9554ab7d 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -156,6 +156,7 @@ class VectorCombine {
bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
bool foldInterleaveIntrinsics(Instruction &I);
bool foldDeinterleaveIntrinsics(Instruction &I);
+ bool foldBitcastOfVPLoad(Instruction &I);
bool shrinkType(Instruction &I);
bool shrinkLoadForShuffles(Instruction &I);
bool shrinkPhiOfShuffles(Instruction &I);
@@ -5823,6 +5824,51 @@ bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
return true;
}
+bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
+ const DataLayout &DL = I.getDataLayout();
+ auto *Cast = dyn_cast<CastInst>(&I);
+ if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Cast->getDestTy()))
+ return false;
+
+ auto *II = dyn_cast<VPIntrinsic>(I.getOperand(0));
+ if (!II || II->getIntrinsicID() != Intrinsic::vp_load)
+ return false;
+ // Fold away bit casts of the loaded value by loading the desired type,
+ // if the mask is all-ones.
+ Value *Mask = II->getMaskParam();
+ Value *EVL = II->getVectorLengthParam();
+ if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue() ||
+ !II->hasOneUse())
+ return false;
+
+ VectorType *OrigVecTy = cast<VectorType>(II->getType());
+ Align OrigAlign =
+ DL.getValueOrABITypeAlignment(II->getPointerAlignment(), OrigVecTy);
+ ElementCount OrigVecCnt = OrigVecTy->getElementCount();
+ VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
+ ElementCount NewVecCnt = NewVecTy->getElementCount();
+
+ // Right now we only support cases where the NewVec is longer, because for
+ // cases where it's shorter, we have to be sure that EVL can be exactly
+ // divided, otherwise it might yield incorrect results or even page faults
+ // (if we round-up during the division).
+ if (!(OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
+ NewVecCnt.hasKnownScalarFactor(OrigVecCnt)))
+ return false;
+
+ unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
+ Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
+ Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
+ CallInst *NewVP =
+ Builder.CreateIntrinsic(NewVecTy, Intrinsic::vp_load,
+ {II->getMemoryPointerParam(), NewMask, NewEVL});
+ // Preserve the original alignment.
+ NewVP->addParamAttrs(
+ 0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
+ replaceValue(*Cast, *NewVP);
+ return true;
+}
+
// Attempt to shrink loads that are only used by shufflevector instructions.
bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
auto *OldLoad = dyn_cast<LoadInst>(&I);
@@ -6102,6 +6148,8 @@ bool VectorCombine::run() {
return true;
if (foldInterleaveIntrinsics(I))
return true;
+ if (foldBitcastOfVPLoad(I))
+ return true;
}
if (foldDeinterleaveIntrinsics(I))
diff --git a/llvm/test/Transforms/InstCombine/fold-vp-load.ll b/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
similarity index 74%
rename from llvm/test/Transforms/InstCombine/fold-vp-load.ll
rename to llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
index f650a7c45a5c6..7bc7c7fbf9f7b 100644
--- a/llvm/test/Transforms/InstCombine/fold-vp-load.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -p instcombine < %s | FileCheck %s
+; RUN: opt -S -p vector-combine -mtriple riscv64 -mattr=+v < %s | FileCheck %s
define <vscale x 8 x i16> @bitcast_vp_load_scalable(ptr %p) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 16 [[P]], <vscale x 8 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
@@ -14,8 +14,8 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable(ptr %p) {
define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 16 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
@@ -26,8 +26,8 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(ptr %p, i32 %evl) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
@@ -38,8 +38,8 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(ptr %p, i32 %
define <8 x i16> @bitcast_vp_load_fixed_evl(ptr %p, i32 %evl) {
; CHECK-LABEL: define <8 x i16> @bitcast_vp_load_fixed_evl(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 16 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <8 x i16> [[R]]
;
@@ -50,7 +50,7 @@ define <8 x i16> @bitcast_vp_load_fixed_evl(ptr %p, i32 %evl) {
define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(ptr %p) {
; CHECK-LABEL: define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[P]], <vscale x 4 x i1> splat (i1 true), i32 3)
; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[L]] to <vscale x 2 x i64>
; CHECK-NEXT: ret <vscale x 2 x i64> [[R]]
@@ -62,7 +62,7 @@ define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(ptr %p) {
define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(ptr %p, <vscale x 4 x i1> %m) {
; CHECK-LABEL: define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(
-; CHECK-SAME: ptr [[P:%.*]], <vscale x 4 x i1> [[M:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]], <vscale x 4 x i1> [[M:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[P]], <vscale x 4 x i1> [[M]], i32 2)
; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[L]] to <vscale x 8 x i16>
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
@@ -71,3 +71,15 @@ define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(ptr %p, <vscal
%r = bitcast <vscale x 4 x i32> %l to <vscale x 8 x i16>
ret <vscale x 8 x i16> %r
}
+
+; Negative test - vp.load of i1 has an invalid cost
+define <vscale x 32 x i1> @bitcast_vp_load_i1(ptr %p) {
+; CHECK-LABEL: define <vscale x 32 x i1> @bitcast_vp_load_i1(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 32 x i1> @llvm.vp.load.nxv32i1.p0(ptr align 4 [[P]], <vscale x 32 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT: ret <vscale x 32 x i1> [[R]]
+;
+ %l = call <vscale x 4 x i8> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 2)
+ %r = bitcast <vscale x 4 x i8> %l to <vscale x 32 x i1>
+ ret <vscale x 32 x i1> %r
+}
>From 8bf5bf8fee25cc54417b6857356f2dfdec6bc078 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 29 May 2026 11:48:55 +0800
Subject: [PATCH 2/3] Add cost check
---
llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 16 ++++++++++++++++
.../VectorCombine/RISCV/fold-vp-load.ll | 3 ++-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 427db9554ab7d..c93b2b2f995b6 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -5856,6 +5856,22 @@ bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
NewVecCnt.hasKnownScalarFactor(OrigVecCnt)))
return false;
+ InstructionCost OldCost =
+ TTI.getMemIntrinsicInstrCost({Intrinsic::vp_load, OrigVecTy,
+ II->getMemoryPointerParam(), false,
+ OrigAlign},
+ CostKind) +
+ TTI.getCastInstrCost(Instruction::BitCast, Cast->getType(), OrigVecTy,
+ TTI::CastContextHint::None, CostKind);
+ InstructionCost NewCost = TTI.getMemIntrinsicInstrCost(
+ {Intrinsic::vp_load, NewVecTy, II->getMemoryPointerParam(), false,
+ OrigAlign},
+ CostKind);
+ LLVM_DEBUG(dbgs() << "foldBitcastOfVPLoad: OldCost=" << OldCost
+ << " NewCost=" << NewCost << "\n");
+ if (NewCost > OldCost || !NewCost.isValid())
+ return false;
+
unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll b/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
index 7bc7c7fbf9f7b..066ce0484b737 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
@@ -76,7 +76,8 @@ define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(ptr %p, <vscal
define <vscale x 32 x i1> @bitcast_vp_load_i1(ptr %p) {
; CHECK-LABEL: define <vscale x 32 x i1> @bitcast_vp_load_i1(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 32 x i1> @llvm.vp.load.nxv32i1.p0(ptr align 4 [[P]], <vscale x 32 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr [[P]], <vscale x 4 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i8> [[L]] to <vscale x 32 x i1>
; CHECK-NEXT: ret <vscale x 32 x i1> [[R]]
;
%l = call <vscale x 4 x i8> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 2)
>From a0599d0674445c2243a19e615153cf09e56897c7 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 1 Jun 2026 19:14:25 +0800
Subject: [PATCH 3/3] Use PatternMatch.h
---
llvm/lib/Transforms/Vectorize/VectorCombine.cpp | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 34fd7458c6d67..88ab6c09d5690 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -5833,15 +5833,12 @@ bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Cast->getDestTy()))
return false;
- auto *II = dyn_cast<VPIntrinsic>(I.getOperand(0));
- if (!II || II->getIntrinsicID() != Intrinsic::vp_load)
- return false;
// Fold away bit casts of the loaded value by loading the desired type,
// if the mask is all-ones.
- Value *Mask = II->getMaskParam();
- Value *EVL = II->getVectorLengthParam();
- if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue() ||
- !II->hasOneUse())
+ Value *EVL;
+ auto *II = dyn_cast<VPIntrinsic>(I.getOperand(0));
+ if (!II || !match(II, m_OneUse(m_Intrinsic<Intrinsic::vp_load>(
+ m_Value(), m_AllOnes(), m_Value(EVL)))))
return false;
VectorType *OrigVecTy = cast<VectorType>(II->getType());
More information about the llvm-commits
mailing list