[llvm] 2666b44 - [InstCombine][VectorCombine] Move bitcast vp.load fold into VectorCombine (#200321)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 04:53:00 PDT 2026
Author: Luke Lau
Date: 2026-06-01T11:52:52Z
New Revision: 2666b44bdc9d2098d394c0afce91a00437b75823
URL: https://github.com/llvm/llvm-project/commit/2666b44bdc9d2098d394c0afce91a00437b75823
DIFF: https://github.com/llvm/llvm-project/commit/2666b44bdc9d2098d394c0afce91a00437b75823.diff
LOG: [InstCombine][VectorCombine] Move bitcast vp.load fold into VectorCombine (#200321)
Fixes https://github.com/llvm/llvm-project/issues/199896
In #192173 we started folding bitcasts of vp.loads with an all ones mask
into a vp.load with the casted type. However on RISC-V a vp.load of an
i1 vector is illegal (since there's no masked variant of `vlm.v`), and
we have no way of checking this in InstCombine.
This moves the fold into VectorCombine so we can query TTI if the cost
is legal (and profitable)
As a side note, it may be possible to lower a vp.load of an i1 vector on
RISC-V to `vlm.v` **only** if the mask is all ones. But this means the
lowering would only be valid for certain values, which is difficult to
cost. And I'm not sure if it would be profitable anyway.
Added:
llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
Modified:
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
llvm/lib/Transforms/Vectorize/VectorCombine.cpp
Removed:
llvm/test/Transforms/InstCombine/fold-vp-load.ll
################################################################################
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index aeb40939cd10e..df693293b03dd 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3956,47 +3956,6 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
}
break;
}
- case Intrinsic::vp_load: {
- auto *VPI = cast<VPIntrinsic>(II);
- // Fold away bit casts of the loaded value by loading the desired type,
- // if the mask is all-ones.
- Value *Mask = VPI->getMaskParam();
- Value *EVL = VPI->getVectorLengthParam();
- if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue() ||
- !II->hasOneUse())
- break;
-
- const DataLayout &DL = II->getDataLayout();
- auto *Cast = dyn_cast<CastInst>(II->user_back());
- if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Cast->getDestTy()))
- break;
- VectorType *OrigVecTy = cast<VectorType>(II->getType());
- Align OrigAlign =
- DL.getValueOrABITypeAlignment(VPI->getPointerAlignment(), OrigVecTy);
- ElementCount OrigVecCnt = OrigVecTy->getElementCount();
- VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
- ElementCount NewVecCnt = NewVecTy->getElementCount();
-
- // Right now we only support cases where the NewVec is longer, because for
- // cases where it's shorter, we have to be sure that EVL can be exactly
- // divided, otherwise it might yield incorrect results or even page faults
- // (if we round-up during the division).
- if (OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
- NewVecCnt.hasKnownScalarFactor(OrigVecCnt)) {
- unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
- Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
- Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
- CallInst *NewVP = Builder.CreateIntrinsic(
- NewVecTy, Intrinsic::vp_load,
- {VPI->getMemoryPointerParam(), NewMask, NewEVL});
- // Preserve the original alignment.
- NewVP->addParamAttrs(
- 0, AttrBuilder(VPI->getContext()).addAlignmentAttr(OrigAlign));
- replaceInstUsesWith(*Cast, NewVP);
- return eraseInstFromFunction(*Cast);
- }
- break;
- }
case Intrinsic::experimental_vp_reverse: {
Value *X;
Value *Vec = II->getArgOperand(0);
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index da8a52a6b3dca..88ab6c09d5690 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -156,6 +156,7 @@ class VectorCombine {
bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
bool foldInterleaveIntrinsics(Instruction &I);
bool foldDeinterleaveIntrinsics(Instruction &I);
+ bool foldBitcastOfVPLoad(Instruction &I);
bool shrinkType(Instruction &I);
bool shrinkLoadForShuffles(Instruction &I);
bool shrinkPhiOfShuffles(Instruction &I);
@@ -5826,6 +5827,64 @@ bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
return true;
}
+bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
+ const DataLayout &DL = I.getDataLayout();
+ auto *Cast = dyn_cast<CastInst>(&I);
+ if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Cast->getDestTy()))
+ return false;
+
+ // Fold away bit casts of the loaded value by loading the desired type,
+ // if the mask is all-ones.
+ Value *EVL;
+ auto *II = dyn_cast<VPIntrinsic>(I.getOperand(0));
+ if (!II || !match(II, m_OneUse(m_Intrinsic<Intrinsic::vp_load>(
+ m_Value(), m_AllOnes(), m_Value(EVL)))))
+ return false;
+
+ VectorType *OrigVecTy = cast<VectorType>(II->getType());
+ Align OrigAlign =
+ DL.getValueOrABITypeAlignment(II->getPointerAlignment(), OrigVecTy);
+ ElementCount OrigVecCnt = OrigVecTy->getElementCount();
+ VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
+ ElementCount NewVecCnt = NewVecTy->getElementCount();
+
+ // Right now we only support cases where the NewVec is longer, because for
+ // cases where it's shorter, we have to be sure that EVL can be exactly
+ // divided, otherwise it might yield incorrect results or even page faults
+ // (if we round-up during the division).
+ if (!(OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
+ NewVecCnt.hasKnownScalarFactor(OrigVecCnt)))
+ return false;
+
+ InstructionCost OldCost =
+ TTI.getMemIntrinsicInstrCost({Intrinsic::vp_load, OrigVecTy,
+ II->getMemoryPointerParam(), false,
+ OrigAlign},
+ CostKind) +
+ TTI.getCastInstrCost(Instruction::BitCast, Cast->getType(), OrigVecTy,
+ TTI::CastContextHint::None, CostKind);
+ InstructionCost NewCost = TTI.getMemIntrinsicInstrCost(
+ {Intrinsic::vp_load, NewVecTy, II->getMemoryPointerParam(), false,
+ OrigAlign},
+ CostKind);
+ LLVM_DEBUG(dbgs() << "foldBitcastOfVPLoad: OldCost=" << OldCost
+ << " NewCost=" << NewCost << "\n");
+ if (NewCost > OldCost || !NewCost.isValid())
+ return false;
+
+ unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
+ Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
+ Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
+ CallInst *NewVP =
+ Builder.CreateIntrinsic(NewVecTy, Intrinsic::vp_load,
+ {II->getMemoryPointerParam(), NewMask, NewEVL});
+ // Preserve the original alignment.
+ NewVP->addParamAttrs(
+ 0, AttrBuilder(II->getContext()).addAlignmentAttr(OrigAlign));
+ replaceValue(*Cast, *NewVP);
+ return true;
+}
+
// Attempt to shrink loads that are only used by shufflevector instructions.
bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
auto *OldLoad = dyn_cast<LoadInst>(&I);
@@ -6105,6 +6164,8 @@ bool VectorCombine::run() {
return true;
if (foldInterleaveIntrinsics(I))
return true;
+ if (foldBitcastOfVPLoad(I))
+ return true;
}
if (foldDeinterleaveIntrinsics(I))
diff --git a/llvm/test/Transforms/InstCombine/fold-vp-load.ll b/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
similarity index 72%
rename from llvm/test/Transforms/InstCombine/fold-vp-load.ll
rename to llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
index f650a7c45a5c6..066ce0484b737 100644
--- a/llvm/test/Transforms/InstCombine/fold-vp-load.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/fold-vp-load.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -p instcombine < %s | FileCheck %s
+; RUN: opt -S -p vector-combine -mtriple riscv64 -mattr=+v < %s | FileCheck %s
define <vscale x 8 x i16> @bitcast_vp_load_scalable(ptr %p) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 16 [[P]], <vscale x 8 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
@@ -14,8 +14,8 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable(ptr %p) {
define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 16 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
@@ -26,8 +26,8 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(ptr %p, i32 %evl) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
@@ -38,8 +38,8 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(ptr %p, i32 %
define <8 x i16> @bitcast_vp_load_fixed_evl(ptr %p, i32 %evl) {
; CHECK-LABEL: define <8 x i16> @bitcast_vp_load_fixed_evl(
-; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i32 [[EVL]], 2
; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 16 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <8 x i16> [[R]]
;
@@ -50,7 +50,7 @@ define <8 x i16> @bitcast_vp_load_fixed_evl(ptr %p, i32 %evl) {
define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(ptr %p) {
; CHECK-LABEL: define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(
-; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[P]], <vscale x 4 x i1> splat (i1 true), i32 3)
; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[L]] to <vscale x 2 x i64>
; CHECK-NEXT: ret <vscale x 2 x i64> [[R]]
@@ -62,7 +62,7 @@ define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(ptr %p) {
define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(ptr %p, <vscale x 4 x i1> %m) {
; CHECK-LABEL: define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(
-; CHECK-SAME: ptr [[P:%.*]], <vscale x 4 x i1> [[M:%.*]]) {
+; CHECK-SAME: ptr [[P:%.*]], <vscale x 4 x i1> [[M:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[P]], <vscale x 4 x i1> [[M]], i32 2)
; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[L]] to <vscale x 8 x i16>
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
@@ -71,3 +71,16 @@ define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(ptr %p, <vscal
%r = bitcast <vscale x 4 x i32> %l to <vscale x 8 x i16>
ret <vscale x 8 x i16> %r
}
+
+; Negative test - vp.load of i1 has an invalid cost
+define <vscale x 32 x i1> @bitcast_vp_load_i1(ptr %p) {
+; CHECK-LABEL: define <vscale x 32 x i1> @bitcast_vp_load_i1(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr [[P]], <vscale x 4 x i1> splat (i1 true), i32 2)
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i8> [[L]] to <vscale x 32 x i1>
+; CHECK-NEXT: ret <vscale x 32 x i1> [[R]]
+;
+ %l = call <vscale x 4 x i8> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 2)
+ %r = bitcast <vscale x 4 x i8> %l to <vscale x 32 x i1>
+ ret <vscale x 32 x i1> %r
+}
More information about the llvm-commits
mailing list