[llvm] 02bb33c - [RISCV] Check for alignment when lowering interleaved/deinterleaved loads/stores
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 7 07:34:32 PDT 2023
Author: Luke Lau
Date: 2023-07-07T15:34:24+01:00
New Revision: 02bb33c3ce7a83d47244ae16c8b4c625aba187a2
URL: https://github.com/llvm/llvm-project/commit/02bb33c3ce7a83d47244ae16c8b4c625aba187a2
DIFF: https://github.com/llvm/llvm-project/commit/02bb33c3ce7a83d47244ae16c8b4c625aba187a2.diff
LOG: [RISCV] Check for alignment when lowering interleaved/deinterleaved loads/stores
As noted by @reames, we should be checking that the memory access is aligned to
the element size (or the unaligned vector memory access feature is enabled)
before lowering vlseg/vsseg intrinsics via the interleaved access pass.
Reviewed By: reames
Differential Revision: https://reviews.llvm.org/D154536
Added:
Modified:
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.h
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 1180c39ee7fa3b..7c15e5d5e78aa1 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -16689,13 +16689,16 @@ Value *RISCVTargetLowering::getIRStackGuard(IRBuilderBase &IRB) const {
}
bool RISCVTargetLowering::isLegalInterleavedAccessType(
- VectorType *VTy, unsigned Factor, const DataLayout &DL) const {
+ VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace,
+ const DataLayout &DL) const {
EVT VT = getValueType(DL, VTy);
// Don't lower vlseg/vsseg for vector types that can't be split.
if (!isTypeLegal(VT))
return false;
- if (!isLegalElementTypeForRVV(VT.getScalarType()))
+ if (!isLegalElementTypeForRVV(VT.getScalarType()) ||
+ !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace,
+ Alignment))
return false;
MVT ContainerVT = VT.getSimpleVT();
@@ -16762,7 +16765,8 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
IRBuilder<> Builder(LI);
auto *VTy = cast<FixedVectorType>(Shuffles[0]->getType());
- if (!isLegalInterleavedAccessType(VTy, Factor,
+ if (!isLegalInterleavedAccessType(VTy, Factor, LI->getAlign(),
+ LI->getPointerAddressSpace(),
LI->getModule()->getDataLayout()))
return false;
@@ -16815,7 +16819,8 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
// Given SVI : <n*factor x ty>, then VTy : <n x ty>
auto *VTy = FixedVectorType::get(ShuffleVTy->getElementType(),
ShuffleVTy->getNumElements() / Factor);
- if (!isLegalInterleavedAccessType(VTy, Factor,
+ if (!isLegalInterleavedAccessType(VTy, Factor, SI->getAlign(),
+ SI->getPointerAddressSpace(),
SI->getModule()->getDataLayout()))
return false;
@@ -16859,7 +16864,8 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *DI,
VectorType *VTy = cast<VectorType>(DI->getOperand(0)->getType());
VectorType *ResVTy = cast<VectorType>(DI->getType()->getContainedType(0));
- if (!isLegalInterleavedAccessType(ResVTy, Factor,
+ if (!isLegalInterleavedAccessType(ResVTy, Factor, LI->getAlign(),
+ LI->getPointerAddressSpace(),
LI->getModule()->getDataLayout()))
return false;
@@ -16908,7 +16914,8 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(IntrinsicInst *II,
VectorType *VTy = cast<VectorType>(II->getType());
VectorType *InVTy = cast<VectorType>(II->getOperand(0)->getType());
- if (!isLegalInterleavedAccessType(InVTy, Factor,
+ if (!isLegalInterleavedAccessType(InVTy, Factor, SI->getAlign(),
+ SI->getPointerAddressSpace(),
SI->getModule()->getDataLayout()))
return false;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 87d6c9f3957217..7621911f697695 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -742,7 +742,8 @@ class RISCVTargetLowering : public TargetLowering {
/// Returns whether or not generating a interleaved load/store intrinsic for
/// this type will be legal.
- bool isLegalInterleavedAccessType(VectorType *, unsigned Factor,
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace,
const DataLayout &) const;
/// Return true if a stride load store of the given result type and
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 6e26241275ab05..81a910ae57131e 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -471,7 +471,8 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
// it's getMemoryOpCost returns a really expensive cost for types like
// <6 x i8>, which show up when doing interleaves of Factor=3 etc.
// Should the memory op cost of these be cheaper?
- if (TLI->isLegalInterleavedAccessType(LegalFVTy, Factor, DL)) {
+ if (TLI->isLegalInterleavedAccessType(LegalFVTy, Factor, Alignment,
+ AddressSpace, DL)) {
InstructionCost LegalMemCost = getMemoryOpCost(
Opcode, LegalFVTy, Alignment, AddressSpace, CostKind);
return LT.first + LegalMemCost;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
index d9a13afbce6caa..bf2642e0a38dd0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
@@ -82,12 +82,16 @@ define {<16 x i8>, <16 x i8>} @vector_deinterleave_load_v16i8_v32i8(ptr %p) {
ret {<16 x i8>, <16 x i8>} %retval
}
-; FIXME: Shouldn't be lowered to vlseg because it's unaligned
+; Shouldn't be lowered to vlseg because it's unaligned
define {<8 x i16>, <8 x i16>} @vector_deinterleave_load_v8i16_v16i16_align1(ptr %p) {
-; CHECK-LABEL: vector_deinterleave_load_v8i16_v16i16:
+; CHECK-LABEL: vector_deinterleave_load_v8i16_v16i16_align1:
; CHECK: # %bb.0:
+; CHECK-NEXT: li a1, 32
+; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT: vle8.v v10, (a0)
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vlseg2e16.v v8, (a0)
+; CHECK-NEXT: vnsrl.wi v8, v10, 0
+; CHECK-NEXT: vnsrl.wi v9, v10, 16
; CHECK-NEXT: ret
%vec = load <16 x i16>, ptr %p, align 1
%retval = call {<8 x i16>, <8 x i16>} @llvm.experimental.vector.deinterleave2.v16i16(<16 x i16> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
index 25c7e851f422ba..a0024b0ddba764 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
@@ -29,12 +29,17 @@ define void @vector_interleave_store_v32i1_v16i1(<16 x i1> %a, <16 x i1> %b, ptr
ret void
}
-; FIXME: Shouldn't be lowered to vsseg because it's unaligned
+; Shouldn't be lowered to vsseg because it's unaligned
define void @vector_interleave_store_v16i16_v8i16_align1(<8 x i16> %a, <8 x i16> %b, ptr %p) {
; CHECK-LABEL: vector_interleave_store_v16i16_v8i16_align1:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vsseg2e16.v v8, (a0)
+; CHECK-NEXT: vwaddu.vv v10, v8, v9
+; CHECK-NEXT: li a1, -1
+; CHECK-NEXT: vwmaccu.vx v10, a1, v9
+; CHECK-NEXT: li a1, 32
+; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT: vse8.v v10, (a0)
; CHECK-NEXT: ret
%res = call <16 x i16> @llvm.experimental.vector.interleave2.v16i16(<8 x i16> %a, <8 x i16> %b)
store <16 x i16> %res, ptr %p, align 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index 326fd78c3cc3e2..1f8270a401d9d1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -39,12 +39,14 @@ define {<vscale x 16 x i8>, <vscale x 16 x i8>} @vector_deinterleave_load_nxv16i
ret {<vscale x 16 x i8>, <vscale x 16 x i8>} %retval
}
-; FIXME: Shouldn't be lowered to vlseg because it's unaligned
+; Shouldn't be lowered to vlseg because it's unaligned
define {<vscale x 8 x i16>, <vscale x 8 x i16>} @vector_deinterleave_load_nxv8i16_nxv16i16_align1(ptr %p) {
; CHECK-LABEL: vector_deinterleave_load_nxv8i16_nxv16i16_align1:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT: vlseg2e16.v v8, (a0)
+; CHECK-NEXT: vl4r.v v12, (a0)
+; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma
+; CHECK-NEXT: vnsrl.wi v8, v12, 0
+; CHECK-NEXT: vnsrl.wi v10, v12, 16
; CHECK-NEXT: ret
%vec = load <vscale x 16 x i16>, ptr %p, align 1
%retval = call {<vscale x 8 x i16>, <vscale x 8 x i16>} @llvm.experimental.vector.deinterleave2.nxv16i16(<vscale x 16 x i16> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
index 909dc3461c5ca9..ea47df594e268d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
@@ -32,12 +32,15 @@ define void @vector_interleave_store_nxv32i1_nxv16i1(<vscale x 16 x i1> %a, <vsc
ret void
}
-; FIXME: Shouldn't be lowered to vsseg because it's unaligned
+; Shouldn't be lowered to vsseg because it's unaligned
define void @vector_interleave_store_nxv16i16_nxv8i16_align1(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, ptr %p) {
; CHECK-LABEL: vector_interleave_store_nxv16i16_nxv8i16_align1:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT: vsseg2e16.v v8, (a0)
+; CHECK-NEXT: vwaddu.vv v12, v8, v10
+; CHECK-NEXT: li a1, -1
+; CHECK-NEXT: vwmaccu.vx v12, a1, v10
+; CHECK-NEXT: vs4r.v v12, (a0)
; CHECK-NEXT: ret
%res = call <vscale x 16 x i16> @llvm.experimental.vector.interleave2.nxv16i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
store <vscale x 16 x i16> %res, ptr %p, align 1
More information about the llvm-commits
mailing list