[llvm] [InstCombine] Fold bitcast into vp.load (PR #192173)
Min-Yih Hsu via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 10:22:14 PDT 2026
https://github.com/mshockwave updated https://github.com/llvm/llvm-project/pull/192173
>From 02d44c6947bcf693e826b85894d4d3960bc7ea9d Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Tue, 14 Apr 2026 17:37:18 -0700
Subject: [PATCH 1/3] [InstCombine] Fold bitcast into vp.load
---
.../InstCombine/InstCombineCalls.cpp | 36 +++++++++++
.../Transforms/InstCombine/fold-vp-load.ll | 61 +++++++++++++++++++
2 files changed, 97 insertions(+)
create mode 100644 llvm/test/Transforms/InstCombine/fold-vp-load.ll
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index aa3f0dcc3c534..1cfc0290269d9 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3971,6 +3971,42 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
}
break;
}
+ case Intrinsic::vp_load: {
+ // Fold away bit casts of the loaded value by loading the desired type,
+ // if the mask is all-ones.
+ Value *Ptr = II->getArgOperand(0);
+ Value *Mask = II->getArgOperand(1);
+ Value *EVL = II->getArgOperand(2);
+ if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue() ||
+ !II->hasOneUse())
+ break;
+
+ auto *Cast = dyn_cast<CastInst>(II->user_back());
+ if (!Cast || !Cast->isNoopCast(II->getDataLayout()) ||
+ !isa<VectorType>(Cast->getDestTy()))
+ break;
+ VectorType *OrigVecTy = cast<VectorType>(II->getType());
+ ElementCount OrigVecCnt = OrigVecTy->getElementCount();
+ VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
+ ElementCount NewVecCnt = NewVecTy->getElementCount();
+
+ // Right now we only support cases where
+ // the NewVec is longer, because for cases where
+ // it's shorter, we have to be sure that EVL can be
+ // exactly divided, otherwise it might page fault / yield
+ // incorrect results.
+ if (OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
+ NewVecCnt.hasKnownScalarFactor(OrigVecCnt)) {
+ unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
+ Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
+ Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
+ Value *NewVP = Builder.CreateIntrinsic(NewVecTy, Intrinsic::vp_load,
+ {Ptr, NewMask, NewEVL});
+ replaceInstUsesWith(*Cast, NewVP);
+ return eraseInstFromFunction(*Cast);
+ }
+ break;
+ }
case Intrinsic::experimental_vp_reverse: {
Value *X;
Value *Vec = II->getArgOperand(0);
diff --git a/llvm/test/Transforms/InstCombine/fold-vp-load.ll b/llvm/test/Transforms/InstCombine/fold-vp-load.ll
new file mode 100644
index 0000000000000..891d748a10c93
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-vp-load.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -p instcombine < %s | FileCheck %s
+
+define <vscale x 8 x i16> @bitcast_vp_load_scalable(ptr %p) {
+; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr [[P]], <vscale x 8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %l = call <vscale x 4 x i32> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 2)
+ %r = bitcast <vscale x 4 x i32> %l to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %r
+}
+
+define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %l = call <vscale x 4 x i32> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+ %r = bitcast <vscale x 4 x i32> %l to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %r
+}
+
+define <8 x i16> @bitcast_vp_load_fixed_evl(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <8 x i16> @bitcast_vp_load_fixed_evl(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %l = call <4 x i32> @llvm.vp.load(ptr %p, <4 x i1> splat (i1 true), i32 %evl)
+ %r = bitcast <4 x i32> %l to <8 x i16>
+ ret <8 x i16> %r
+}
+
+define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(ptr %p) {
+; CHECK-LABEL: define <vscale x 2 x i64> @negative_bitcast_vp_load_scalable(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[P]], <vscale x 4 x i1> splat (i1 true), i32 3)
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[L]] to <vscale x 2 x i64>
+; CHECK-NEXT: ret <vscale x 2 x i64> [[R]]
+;
+ %l = call <vscale x 4 x i32> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 3)
+ %r = bitcast <vscale x 4 x i32> %l to <vscale x 2 x i64>
+ ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(ptr %p, <vscale x 4 x i1> %m) {
+; CHECK-LABEL: define <vscale x 8 x i16> @negative_bitcast_vp_load_scalable_mask(
+; CHECK-SAME: ptr [[P:%.*]], <vscale x 4 x i1> [[M:%.*]]) {
+; CHECK-NEXT: [[L:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr [[P]], <vscale x 4 x i1> [[M]], i32 2)
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[L]] to <vscale x 8 x i16>
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %l = call <vscale x 4 x i32> @llvm.vp.load(ptr %p, <vscale x 4 x i1> %m, i32 2)
+ %r = bitcast <vscale x 4 x i32> %l to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %r
+}
>From 6bd4fe4a9dc89bbb20295526d2b0d81dc6b662d7 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 15 Apr 2026 10:19:45 -0700
Subject: [PATCH 2/3] fixup! Preserve the alignment
---
.../InstCombine/InstCombineCalls.cpp | 20 ++++++++++++-------
.../Transforms/InstCombine/fold-vp-load.ll | 18 ++++++++++++++---
2 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 1cfc0290269d9..38434150fd3eb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3972,20 +3972,22 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
break;
}
case Intrinsic::vp_load: {
+ auto *VPI = cast<VPIntrinsic>(II);
// Fold away bit casts of the loaded value by loading the desired type,
// if the mask is all-ones.
- Value *Ptr = II->getArgOperand(0);
- Value *Mask = II->getArgOperand(1);
- Value *EVL = II->getArgOperand(2);
+ Value *Mask = VPI->getMaskParam();
+ Value *EVL = VPI->getVectorLengthParam();
if (!isa<Constant>(Mask) || !cast<Constant>(Mask)->isAllOnesValue() ||
!II->hasOneUse())
break;
+ const DataLayout &DL = II->getDataLayout();
auto *Cast = dyn_cast<CastInst>(II->user_back());
- if (!Cast || !Cast->isNoopCast(II->getDataLayout()) ||
- !isa<VectorType>(Cast->getDestTy()))
+ if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Cast->getDestTy()))
break;
VectorType *OrigVecTy = cast<VectorType>(II->getType());
+ Align OrigAlign =
+ DL.getValueOrABITypeAlignment(VPI->getPointerAlignment(), OrigVecTy);
ElementCount OrigVecCnt = OrigVecTy->getElementCount();
VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
ElementCount NewVecCnt = NewVecTy->getElementCount();
@@ -4000,8 +4002,12 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
Value *NewEVL = Builder.CreateNUWMul(EVL, Builder.getInt32(Factor));
Value *NewMask = Builder.CreateVectorSplat(NewVecCnt, Builder.getTrue());
- Value *NewVP = Builder.CreateIntrinsic(NewVecTy, Intrinsic::vp_load,
- {Ptr, NewMask, NewEVL});
+ CallInst *NewVP = Builder.CreateIntrinsic(
+ NewVecTy, Intrinsic::vp_load,
+ {VPI->getMemoryPointerParam(), NewMask, NewEVL});
+ // Preserve the original alignment.
+ NewVP->addParamAttrs(
+ 0, AttrBuilder(VPI->getContext()).addAlignmentAttr(OrigAlign));
replaceInstUsesWith(*Cast, NewVP);
return eraseInstFromFunction(*Cast);
}
diff --git a/llvm/test/Transforms/InstCombine/fold-vp-load.ll b/llvm/test/Transforms/InstCombine/fold-vp-load.ll
index 891d748a10c93..f650a7c45a5c6 100644
--- a/llvm/test/Transforms/InstCombine/fold-vp-load.ll
+++ b/llvm/test/Transforms/InstCombine/fold-vp-load.ll
@@ -4,7 +4,7 @@
define <vscale x 8 x i16> @bitcast_vp_load_scalable(ptr %p) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr [[P]], <vscale x 8 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 16 [[P]], <vscale x 8 x i1> splat (i1 true), i32 4)
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
%l = call <vscale x 4 x i32> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 2)
@@ -16,7 +16,7 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(
; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
-; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 16 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
;
%l = call <vscale x 4 x i32> @llvm.vp.load(ptr %p, <vscale x 4 x i1> splat (i1 true), i32 %evl)
@@ -24,11 +24,23 @@ define <vscale x 8 x i16> @bitcast_vp_load_scalable_evl(ptr %p, i32 %evl) {
ret <vscale x 8 x i16> %r
}
+define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(ptr %p, i32 %evl) {
+; CHECK-LABEL: define <vscale x 8 x i16> @bitcast_vp_load_scalable_preserve_align(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 4 [[P]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[R]]
+;
+ %l = call <vscale x 4 x i32> @llvm.vp.load(ptr align 4 %p, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+ %r = bitcast <vscale x 4 x i32> %l to <vscale x 8 x i16>
+ ret <vscale x 8 x i16> %r
+}
+
define <8 x i16> @bitcast_vp_load_fixed_evl(ptr %p, i32 %evl) {
; CHECK-LABEL: define <8 x i16> @bitcast_vp_load_fixed_evl(
; CHECK-SAME: ptr [[P:%.*]], i32 [[EVL:%.*]]) {
; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[EVL]], 1
-; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT: [[R:%.*]] = call <8 x i16> @llvm.vp.load.v8i16.p0(ptr align 16 [[P]], <8 x i1> splat (i1 true), i32 [[TMP1]])
; CHECK-NEXT: ret <8 x i16> [[R]]
;
%l = call <4 x i32> @llvm.vp.load(ptr %p, <4 x i1> splat (i1 true), i32 %evl)
>From 2e8047b408a51b7d158b0fd67babb29911d573a8 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Thu, 16 Apr 2026 10:21:29 -0700
Subject: [PATCH 3/3] fixup! Update the comments
---
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 38434150fd3eb..d97c66025cee7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3992,11 +3992,10 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
VectorType *NewVecTy = cast<VectorType>(Cast->getDestTy());
ElementCount NewVecCnt = NewVecTy->getElementCount();
- // Right now we only support cases where
- // the NewVec is longer, because for cases where
- // it's shorter, we have to be sure that EVL can be
- // exactly divided, otherwise it might page fault / yield
- // incorrect results.
+ // Right now we only support cases where the NewVec is longer, because for
+ // cases where it's shorter, we have to be sure that EVL can be exactly
+ // divided, otherwise it might yield incorrect results or even page faults
+ // (if we round-up during the division).
if (OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
NewVecCnt.hasKnownScalarFactor(OrigVecCnt)) {
unsigned Factor = NewVecCnt.getKnownScalarFactor(OrigVecCnt);
More information about the llvm-commits
mailing list