[llvm] [RISCV] Match strided vector bases in RISCVGatherScatterLowering (PR #93972)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Fri May 31 11:08:06 PDT 2024
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/93972
>From 3f0acb27e71a0414b1d52f6a6e42fe687a2118e8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 31 May 2024 10:24:34 +0100
Subject: [PATCH 1/3] [RISCV] Match strided vector bases in
RISCVGatherScatterLowering
Currently we only match GEPs with a scalar base pointer, but a common pattern that's emitted from the loop vectorizer is a strided vector base plus some sort of scalar offset:
%base = getelementptr i64, ptr %p, <vscale x 1 x i64> %step
%gep = getelementptr i64, <vscale x 1 x ptr> %base, i64 %offset
This is common for accesses into a struct e.g. f[i].b below:
struct F { int a; char b; };
void foo(struct F *f) {
for (int i = 0; i < 1024; i += 2) {
f[i].a++;
f[i].b++;
}
}
This patch handles this case in RISCVGatherScatterLowering by recursing on the base pointer if it's a vector.
With this we can convert roughly 80% of the indexed loads and stores emitted to strided loads and stores on SPEC CPU 2017, -O3 -march=rva22u64_v
---
.../Target/RISCV/RISCVGatherScatterLowering.cpp | 16 ++++++++++++++++
.../test/CodeGen/RISCV/rvv/strided-load-store.ll | 12 ++++--------
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index f0bd25f167d80..f7cca854d2767 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -349,6 +349,22 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
SmallVector<Value *, 2> Ops(GEP->operands());
+ // If the base pointer is a vector, check if it's strided.
+ if (GEP->getPointerOperand()->getType()->isVectorTy()) {
+ auto [BaseBase, Stride] = determineBaseAndStride(
+ cast<Instruction>(GEP->getPointerOperand()), Builder);
+ // If GEP's offset is scalar then we can add it to the base pointer's base.
+ auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); };
+ if (BaseBase && all_of(GEP->indices(), IsScalar)) {
+ Builder.SetInsertPoint(GEP);
+ SmallVector<Value *> Indices(GEP->indices());
+ Value *OffsetBase =
+ Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices, "",
+ GEP->isInBounds());
+ return {OffsetBase, Stride};
+ }
+ }
+
// Base pointer needs to be a scalar.
Value *ScalarBase = Ops[0];
if (ScalarBase->getType()->isVectorTy()) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
index 8733c5dc83d65..70412de1d0e91 100644
--- a/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
@@ -301,10 +301,8 @@ define void @constant_stride(<vscale x 1 x i64> %x, ptr %p, i64 %stride) {
define <vscale x 1 x i64> @vector_base_scalar_offset(ptr %p, i64 %offset) {
; CHECK-LABEL: @vector_base_scalar_offset(
-; CHECK-NEXT: [[STEP:%.*]] = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
-; CHECK-NEXT: [[PTRS1:%.*]] = getelementptr i64, ptr [[P:%.*]], <vscale x 1 x i64> [[STEP]]
-; CHECK-NEXT: [[PTRS2:%.*]] = getelementptr i64, <vscale x 1 x ptr> [[PTRS1]], i64 [[OFFSET:%.*]]
-; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> [[PTRS2]], i32 8, <vscale x 1 x i1> shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> poison, i1 true, i64 0), <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i64> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET:%.*]]
+; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.riscv.masked.strided.load.nxv1i64.p0.i64(<vscale x 1 x i64> poison, ptr [[TMP1]], i64 8, <vscale x 1 x i1> shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> poison, i1 true, i64 0), <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer))
; CHECK-NEXT: ret <vscale x 1 x i64> [[X]]
;
%step = call <vscale x 1 x i64> @llvm.experimental.stepvector.nxv1i64()
@@ -321,10 +319,8 @@ define <vscale x 1 x i64> @vector_base_scalar_offset(ptr %p, i64 %offset) {
define <vscale x 1 x i64> @splat_base_scalar_offset(ptr %p, i64 %offset) {
; CHECK-LABEL: @splat_base_scalar_offset(
-; CHECK-NEXT: [[HEAD:%.*]] = insertelement <vscale x 1 x ptr> poison, ptr [[P:%.*]], i32 0
-; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 1 x ptr> [[HEAD]], <vscale x 1 x ptr> poison, <vscale x 1 x i32> zeroinitializer
-; CHECK-NEXT: [[PTRS:%.*]] = getelementptr i64, <vscale x 1 x ptr> [[SPLAT]], i64 [[OFFSET:%.*]]
-; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.masked.gather.nxv1i64.nxv1p0(<vscale x 1 x ptr> [[PTRS]], i32 8, <vscale x 1 x i1> shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> poison, i1 true, i64 0), <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer), <vscale x 1 x i64> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET:%.*]]
+; CHECK-NEXT: [[X:%.*]] = call <vscale x 1 x i64> @llvm.riscv.masked.strided.load.nxv1i64.p0.i64(<vscale x 1 x i64> poison, ptr [[TMP1]], i64 0, <vscale x 1 x i1> shufflevector (<vscale x 1 x i1> insertelement (<vscale x 1 x i1> poison, i1 true, i64 0), <vscale x 1 x i1> poison, <vscale x 1 x i32> zeroinitializer))
; CHECK-NEXT: ret <vscale x 1 x i64> [[X]]
;
%head = insertelement <vscale x 1 x ptr> poison, ptr %p, i32 0
>From 62eb32d05e39aa4dcc3fe73d53f5addb43e4ba14 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 31 May 2024 17:34:36 +0100
Subject: [PATCH 2/3] Check that the pointer operand is an Instruction
---
llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index f7cca854d2767..84fc755c9f260 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -350,9 +350,10 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
SmallVector<Value *, 2> Ops(GEP->operands());
// If the base pointer is a vector, check if it's strided.
- if (GEP->getPointerOperand()->getType()->isVectorTy()) {
- auto [BaseBase, Stride] = determineBaseAndStride(
- cast<Instruction>(GEP->getPointerOperand()), Builder);
+ Value *Base = GEP->getPointerOperand();
+ if (auto *BaseInst = dyn_cast<Instruction>(Base);
+ BaseInst && BaseInst->getType()->isVectorTy()) {
+ auto [BaseBase, Stride] = determineBaseAndStride(BaseInst, Builder);
// If GEP's offset is scalar then we can add it to the base pointer's base.
auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); };
if (BaseBase && all_of(GEP->indices(), IsScalar)) {
@@ -366,7 +367,7 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
}
// Base pointer needs to be a scalar.
- Value *ScalarBase = Ops[0];
+ Value *ScalarBase = Base;
if (ScalarBase->getType()->isVectorTy()) {
ScalarBase = getSplatValue(ScalarBase);
if (!ScalarBase)
>From aa6a40d11f2ba95c38dd9e94c824d28f3a7c5a18 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 31 May 2024 19:07:01 +0100
Subject: [PATCH 3/3] Move down call to determineBaseAndStride to avoid
generating code without passing the checks
---
.../RISCV/RISCVGatherScatterLowering.cpp | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index 84fc755c9f260..254587512f38a 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -353,16 +353,19 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
Value *Base = GEP->getPointerOperand();
if (auto *BaseInst = dyn_cast<Instruction>(Base);
BaseInst && BaseInst->getType()->isVectorTy()) {
- auto [BaseBase, Stride] = determineBaseAndStride(BaseInst, Builder);
// If GEP's offset is scalar then we can add it to the base pointer's base.
auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); };
- if (BaseBase && all_of(GEP->indices(), IsScalar)) {
- Builder.SetInsertPoint(GEP);
- SmallVector<Value *> Indices(GEP->indices());
- Value *OffsetBase =
- Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices, "",
- GEP->isInBounds());
- return {OffsetBase, Stride};
+ if (all_of(GEP->indices(), IsScalar)) {
+ auto [BaseBase, Stride] = determineBaseAndStride(BaseInst, Builder);
+ if (BaseBase) {
+ Builder.SetInsertPoint(GEP);
+ SmallVector<Value *> Indices(GEP->indices());
+ Value *OffsetBase =
+ Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices,
+ "", GEP->isInBounds());
+ OffsetBase->takeName(GEP);
+ return {OffsetBase, Stride};
+ }
}
}
More information about the llvm-commits
mailing list