[llvm] [RISCV][IA] Factor out code for extracting operands from mem insts [nfc] (PR #149344)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 18 11:03:12 PDT 2025
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/149344
>From 920477953b7f07642fc6ac9bdf62522bc123d71c Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Thu, 17 Jul 2025 08:58:33 -0700
Subject: [PATCH 1/4] [RISCV][IA] Factor out code for extracting operands from
mem insts [nfc]
We're going to end up repeating the operand extraction four times once
all of the routines have been updated to support both plain load/store
and vp.load/vp.store. I plan to add masked.load/masked.store in the
near future, and we'd need to add that to each of the four cases.
Instead, factor out a single copy of the operand normalization.
---
.../Target/RISCV/RISCVInterleavedAccess.cpp | 108 +++++++++---------
1 file changed, 52 insertions(+), 56 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 0d4f24172b574..1336145db53db 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -102,6 +102,52 @@ static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
return false;
}
+static bool getMemOperands(IRBuilderBase &Builder, unsigned Factor,
+ VectorType *VTy, const DataLayout &DL, Type *XLenTy,
+ Instruction *I, Value *&Ptr, Value *&Mask,
+ Value *&VL, Align &Alignment) {
+ ElementCount EC = VTy->getElementCount();
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ assert(LI->isSimple());
+ Ptr = LI->getPointerOperand();
+ Alignment = LI->getAlign();
+ assert(!Mask && "Unexpected mask on a load\n");
+ Mask = Builder.getAllOnesMask(EC);
+ VL = isa<FixedVectorType>(VTy) ? Builder.CreateElementCount(XLenTy, EC)
+ : Constant::getAllOnesValue(XLenTy);
+ return true;
+ }
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ assert(SI->isSimple());
+ Ptr = SI->getPointerOperand();
+ Alignment = SI->getAlign();
+ assert(!Mask && "Unexpected mask on a store");
+ Mask = Builder.getAllOnesMask(EC);
+ VL = isa<FixedVectorType>(VTy) ? Builder.CreateElementCount(XLenTy, EC)
+ : Constant::getAllOnesValue(XLenTy);
+ return true;
+ }
+ auto *VPLdSt = cast<VPIntrinsic>(I);
+ assert((VPLdSt->getIntrinsicID() == Intrinsic::vp_load ||
+ VPLdSt->getIntrinsicID() == Intrinsic::vp_store) &&
+ "Unexpected intrinsic");
+ Ptr = VPLdSt->getMemoryPointerParam();
+ Alignment = VPLdSt->getPointerAlignment().value_or(
+ DL.getABITypeAlign(VTy->getElementType()));
+
+ assert(Mask && "vp.load needs a mask!");
+
+ Value *WideEVL = VPLdSt->getVectorLengthParam();
+ // Conservatively check if EVL is a multiple of factor, otherwise some
+ // (trailing) elements might be lost after the transformation.
+ if (!isMultipleOfN(WideEVL, I->getDataLayout(), Factor))
+ return false;
+
+ auto *FactorC = ConstantInt::get(WideEVL->getType(), Factor);
+ VL = Builder.CreateZExt(Builder.CreateExactUDiv(WideEVL, FactorC), XLenTy);
+ return true;
+}
+
/// Lower an interleaved load into a vlsegN intrinsic.
///
/// E.g. Lower an interleaved load (Factor = 2):
@@ -271,34 +317,9 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
Value *Ptr, *VL;
Align Alignment;
- if (auto *LI = dyn_cast<LoadInst>(Load)) {
- assert(LI->isSimple());
- Ptr = LI->getPointerOperand();
- Alignment = LI->getAlign();
- assert(!Mask && "Unexpected mask on a load\n");
- Mask = Builder.getAllOnesMask(ResVTy->getElementCount());
- VL = isa<FixedVectorType>(ResVTy)
- ? Builder.CreateElementCount(XLenTy, ResVTy->getElementCount())
- : Constant::getAllOnesValue(XLenTy);
- } else {
- auto *VPLoad = cast<VPIntrinsic>(Load);
- assert(VPLoad->getIntrinsicID() == Intrinsic::vp_load &&
- "Unexpected intrinsic");
- Ptr = VPLoad->getMemoryPointerParam();
- Alignment = VPLoad->getPointerAlignment().value_or(
- DL.getABITypeAlign(ResVTy->getElementType()));
-
- assert(Mask && "vp.load needs a mask!");
-
- Value *WideEVL = VPLoad->getVectorLengthParam();
- // Conservatively check if EVL is a multiple of factor, otherwise some
- // (trailing) elements might be lost after the transformation.
- if (!isMultipleOfN(WideEVL, Load->getDataLayout(), Factor))
- return false;
-
- auto *FactorC = ConstantInt::get(WideEVL->getType(), Factor);
- VL = Builder.CreateZExt(Builder.CreateExactUDiv(WideEVL, FactorC), XLenTy);
- }
+ if (!getMemOperands(Builder, Factor, ResVTy, DL, XLenTy, Load, Ptr, Mask, VL,
+ Alignment))
+ return false;
Type *PtrTy = Ptr->getType();
unsigned AS = PtrTy->getPointerAddressSpace();
@@ -360,34 +381,9 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
Value *Ptr, *VL;
Align Alignment;
- if (auto *SI = dyn_cast<StoreInst>(Store)) {
- assert(SI->isSimple());
- Ptr = SI->getPointerOperand();
- Alignment = SI->getAlign();
- assert(!Mask && "Unexpected mask on a store");
- Mask = Builder.getAllOnesMask(InVTy->getElementCount());
- VL = isa<FixedVectorType>(InVTy)
- ? Builder.CreateElementCount(XLenTy, InVTy->getElementCount())
- : Constant::getAllOnesValue(XLenTy);
- } else {
- auto *VPStore = cast<VPIntrinsic>(Store);
- assert(VPStore->getIntrinsicID() == Intrinsic::vp_store &&
- "Unexpected intrinsic");
- Ptr = VPStore->getMemoryPointerParam();
- Alignment = VPStore->getPointerAlignment().value_or(
- DL.getABITypeAlign(InVTy->getElementType()));
-
- assert(Mask && "vp.store needs a mask!");
-
- Value *WideEVL = VPStore->getVectorLengthParam();
- // Conservatively check if EVL is a multiple of factor, otherwise some
- // (trailing) elements might be lost after the transformation.
- if (!isMultipleOfN(WideEVL, DL, Factor))
- return false;
-
- auto *FactorC = ConstantInt::get(WideEVL->getType(), Factor);
- VL = Builder.CreateZExt(Builder.CreateExactUDiv(WideEVL, FactorC), XLenTy);
- }
+ if (!getMemOperands(Builder, Factor, InVTy, DL, XLenTy, Store, Ptr, Mask, VL,
+ Alignment))
+ return false;
Type *PtrTy = Ptr->getType();
unsigned AS = Ptr->getType()->getPointerAddressSpace();
if (!isLegalInterleavedAccessType(InVTy, Factor, Alignment, AS, DL))
>From ea2c9b077133e2e6c4a38dd69d4c649932aaf49b Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Thu, 17 Jul 2025 17:50:45 -0700
Subject: [PATCH 2/4] New case
---
.../Target/RISCV/RISCVInterleavedAccess.cpp | 29 ++-----------------
1 file changed, 3 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 0ca6028c929b9..ae12ecea8b850 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -173,32 +173,9 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
Value *Ptr, *VL;
Align Alignment;
- if (auto *LI = dyn_cast<LoadInst>(Load)) {
- assert(LI->isSimple());
- Ptr = LI->getPointerOperand();
- Alignment = LI->getAlign();
- assert(!Mask && "Unexpected mask on a load\n");
- Mask = Builder.getAllOnesMask(VTy->getElementCount());
- VL = Builder.CreateElementCount(XLenTy, VTy->getElementCount());
- } else {
- auto *VPLoad = cast<VPIntrinsic>(Load);
- assert(VPLoad->getIntrinsicID() == Intrinsic::vp_load &&
- "Unexpected intrinsic");
- Ptr = VPLoad->getMemoryPointerParam();
- Alignment = VPLoad->getPointerAlignment().value_or(
- DL.getABITypeAlign(VTy->getElementType()));
-
- assert(Mask && "vp.load needs a mask!");
-
- Value *WideEVL = VPLoad->getVectorLengthParam();
- // Conservatively check if EVL is a multiple of factor, otherwise some
- // (trailing) elements might be lost after the transformation.
- if (!isMultipleOfN(WideEVL, DL, Factor))
- return false;
-
- auto *FactorC = ConstantInt::get(WideEVL->getType(), Factor);
- VL = Builder.CreateZExt(Builder.CreateExactUDiv(WideEVL, FactorC), XLenTy);
- }
+ if (!getMemOperands(Builder, Factor, VTy, DL, XLenTy, Load, Ptr, Mask, VL,
+ Alignment))
+ return false;
Type *PtrTy = Ptr->getType();
unsigned AS = PtrTy->getPointerAddressSpace();
>From f36fba9bd21c6ccd01c2c24338fa6d447944a1e1 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 18 Jul 2025 10:54:31 -0700
Subject: [PATCH 3/4] Address review comments
---
.../Target/RISCV/RISCVInterleavedAccess.cpp | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index ae12ecea8b850..69da0fd349f09 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -102,16 +102,21 @@ static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
return false;
}
-static bool getMemOperands(IRBuilderBase &Builder, unsigned Factor,
- VectorType *VTy, const DataLayout &DL, Type *XLenTy,
+/// Do the common operand retrieval and validition required by the
+/// routines below.
+static bool getMemOperands(unsigned Factor,
+ VectorType *VTy, Type *XLenTy,
Instruction *I, Value *&Ptr, Value *&Mask,
Value *&VL, Align &Alignment) {
+
+ IRBuilder<> Builder(I);
+ const DataLayout &DL = I->getDataLayout();
ElementCount EC = VTy->getElementCount();
if (auto *LI = dyn_cast<LoadInst>(I)) {
assert(LI->isSimple());
Ptr = LI->getPointerOperand();
Alignment = LI->getAlign();
- assert(!Mask && "Unexpected mask on a load\n");
+ assert(!Mask && "Unexpected mask on a load");
Mask = Builder.getAllOnesMask(EC);
VL = isa<FixedVectorType>(VTy) ? Builder.CreateElementCount(XLenTy, EC)
: Constant::getAllOnesValue(XLenTy);
@@ -135,7 +140,7 @@ static bool getMemOperands(IRBuilderBase &Builder, unsigned Factor,
Alignment = VPLdSt->getPointerAlignment().value_or(
DL.getABITypeAlign(VTy->getElementType()));
- assert(Mask && "vp.load needs a mask!");
+ assert(Mask && "vp.load and vp.store needs a mask!");
Value *WideEVL = VPLdSt->getVectorLengthParam();
// Conservatively check if EVL is a multiple of factor, otherwise some
@@ -173,7 +178,7 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
Value *Ptr, *VL;
Align Alignment;
- if (!getMemOperands(Builder, Factor, VTy, DL, XLenTy, Load, Ptr, Mask, VL,
+ if (!getMemOperands(Factor, VTy, XLenTy, Load, Ptr, Mask, VL,
Alignment))
return false;
@@ -319,7 +324,7 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
Value *Ptr, *VL;
Align Alignment;
- if (!getMemOperands(Builder, Factor, ResVTy, DL, XLenTy, Load, Ptr, Mask, VL,
+ if (!getMemOperands(Factor, ResVTy, XLenTy, Load, Ptr, Mask, VL,
Alignment))
return false;
@@ -383,7 +388,7 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
Value *Ptr, *VL;
Align Alignment;
- if (!getMemOperands(Builder, Factor, InVTy, DL, XLenTy, Store, Ptr, Mask, VL,
+ if (!getMemOperands(Factor, InVTy, XLenTy, Store, Ptr, Mask, VL,
Alignment))
return false;
Type *PtrTy = Ptr->getType();
>From 154960990186b762620a3096a5a30ee0eb7c7f8d Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Fri, 18 Jul 2025 10:55:11 -0700
Subject: [PATCH 4/4] clang-format
---
llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 69da0fd349f09..dd68a5556cdb5 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -103,9 +103,8 @@ static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
}
/// Do the common operand retrieval and validition required by the
-/// routines below.
-static bool getMemOperands(unsigned Factor,
- VectorType *VTy, Type *XLenTy,
+/// routines below.
+static bool getMemOperands(unsigned Factor, VectorType *VTy, Type *XLenTy,
Instruction *I, Value *&Ptr, Value *&Mask,
Value *&VL, Align &Alignment) {
@@ -178,8 +177,7 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
Value *Ptr, *VL;
Align Alignment;
- if (!getMemOperands(Factor, VTy, XLenTy, Load, Ptr, Mask, VL,
- Alignment))
+ if (!getMemOperands(Factor, VTy, XLenTy, Load, Ptr, Mask, VL, Alignment))
return false;
Type *PtrTy = Ptr->getType();
@@ -324,8 +322,7 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
Value *Ptr, *VL;
Align Alignment;
- if (!getMemOperands(Factor, ResVTy, XLenTy, Load, Ptr, Mask, VL,
- Alignment))
+ if (!getMemOperands(Factor, ResVTy, XLenTy, Load, Ptr, Mask, VL, Alignment))
return false;
Type *PtrTy = Ptr->getType();
@@ -388,8 +385,7 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
Value *Ptr, *VL;
Align Alignment;
- if (!getMemOperands(Factor, InVTy, XLenTy, Store, Ptr, Mask, VL,
- Alignment))
+ if (!getMemOperands(Factor, InVTy, XLenTy, Store, Ptr, Mask, VL, Alignment))
return false;
Type *PtrTy = Ptr->getType();
unsigned AS = Ptr->getType()->getPointerAddressSpace();
More information about the llvm-commits
mailing list