[llvm] [RISCV] Enable LoopDataPrefetch pass (PR #66201)
Wang Pengcheng via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 17 21:19:48 PDT 2023
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/66201
>From 1a289719fb9827579adfa7c7369b1ced9757ea33 Mon Sep 17 00:00:00 2001
From: wangpc <wangpengcheng.pp at bytedance.com>
Date: Fri, 8 Sep 2023 17:59:24 +0800
Subject: [PATCH 1/3] [RISCV] Enable LoopDataPrefetch pass
So that we can benefit from data prefetching when `Zicbop` extension
is supported.
Tune information for data prefetching are added in `RISCVTuneInfo`.
This PR is stacked on #66193.
---
llvm/lib/Target/RISCV/RISCVProcessors.td | 10 ++++-
llvm/lib/Target/RISCV/RISCVSubtarget.h | 22 ++++++++++
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 9 ++++
llvm/test/CodeGen/RISCV/O3-pipeline.ll | 10 ++++-
.../LoopDataPrefetch/RISCV/basic.ll | 44 +++++++++++++++++++
.../LoopDataPrefetch/RISCV/lit.local.cfg | 2 +
6 files changed, 94 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
create mode 100644 llvm/test/Transforms/LoopDataPrefetch/RISCV/lit.local.cfg
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index e4008d145ffa572..f61d45c93bca819 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -13,12 +13,20 @@
class RISCVTuneInfo {
bits<8> PrefFunctionAlignment = 1;
bits<8> PrefLoopAlignment = 1;
+
+ // Information needed by LoopDataPrefetch.
+ bits<16> CacheLineSize = 0;
+ bits<16> PrefetchDistance = 0;
+ bits<16> MinPrefetchStride = 1;
+ bits<16> MaxPrefetchIterationsAhead = 65535;
}
def RISCVTuneInfoTable : GenericTable {
let FilterClass = "RISCVTuneInfo";
let CppTypeName = "RISCVTuneInfo";
- let Fields = ["Name", "PrefFunctionAlignment", "PrefLoopAlignment"];
+ let Fields = ["Name", "PrefFunctionAlignment", "PrefLoopAlignment",
+ "CacheLineSize", "PrefetchDistance",
+ "MinPrefetchStride", "MaxPrefetchIterationsAhead"];
}
def getRISCVTuneInfo : SearchIndex {
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 6b915e61c136086..a956481db99724a 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -38,6 +38,12 @@ struct RISCVTuneInfo {
const char *Name;
uint8_t PrefFunctionAlignment;
uint8_t PrefLoopAlignment;
+
+ // Information needed by LoopDataPrefetch.
+ uint16_t CacheLineSize;
+ uint16_t PrefetchDistance;
+ uint16_t MinPrefetchStride;
+ uint16_t MaxPrefetchIterationsAhead;
};
#define GET_RISCVTuneInfoTable_DECL
@@ -246,6 +252,22 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
&Mutations) const override;
bool useAA() const override;
+
+ unsigned getCacheLineSize() const override {
+ return TuneInfo->CacheLineSize;
+ };
+ unsigned getPrefetchDistance() const override {
+ return TuneInfo->PrefetchDistance;
+ };
+ unsigned getMinPrefetchStride(unsigned NumMemAccesses,
+ unsigned NumStridedMemAccesses,
+ unsigned NumPrefetches,
+ bool HasCall) const override {
+ return TuneInfo->MinPrefetchStride;
+ };
+ unsigned getMaxPrefetchIterationsAhead() const override {
+ return TuneInfo->MaxPrefetchIterationsAhead;
+ };
};
} // End llvm namespace
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 1281528ea511a4d..825f5e5dca31272 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -34,6 +34,7 @@
#include "llvm/Support/FormattedStream.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/Scalar.h"
#include <optional>
using namespace llvm;
@@ -83,6 +84,11 @@ static cl::opt<bool>
cl::desc("Enable sinking and folding of instruction copies"),
cl::init(false), cl::Hidden);
+static cl::opt<bool>
+ EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
+ cl::desc("Enable the loop data prefetch pass"),
+ cl::init(true));
+
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
@@ -305,6 +311,9 @@ void RISCVPassConfig::addIRPasses() {
addPass(createAtomicExpandPass());
if (getOptLevel() != CodeGenOptLevel::None) {
+ if (EnableLoopDataPrefetch)
+ addPass(createLoopDataPrefetchPass());
+
addPass(createRISCVGatherScatterLoweringPass());
addPass(createInterleavedAccessPass());
addPass(createRISCVCodeGenPreparePass());
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 277951782ce5ccb..edf2cab5b6235c3 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -12,10 +12,10 @@
; CHECK-NEXT: Target Pass Configuration
; CHECK-NEXT: Machine Module Information
; CHECK-NEXT: Target Transform Information
-; CHECK-NEXT: Type-Based Alias Analysis
-; CHECK-NEXT: Scoped NoAlias Alias Analysis
; CHECK-NEXT: Assumption Cache Tracker
; CHECK-NEXT: Profile summary info
+; CHECK-NEXT: Type-Based Alias Analysis
+; CHECK-NEXT: Scoped NoAlias Alias Analysis
; CHECK-NEXT: Create Garbage Collector Module Metadata
; CHECK-NEXT: Machine Branch Probability Analysis
; CHECK-NEXT: Default Regalloc Eviction Advisor
@@ -28,6 +28,12 @@
; CHECK-NEXT: Expand Atomic instructions
; CHECK-NEXT: Dominator Tree Construction
; CHECK-NEXT: Natural Loop Information
+; CHECK-NEXT: Canonicalize natural loops
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Optimization Remark Emitter
+; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Loop Data Prefetch
; CHECK-NEXT: RISC-V gather/scatter lowering
; CHECK-NEXT: Interleaved Access Pass
; CHECK-NEXT: RISC-V CodeGenPrepare
diff --git a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
new file mode 100644
index 000000000000000..0f8c59fd27a93c3
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -mtriple=riscv64 -riscv-enable-loop-data-prefetch \
+; RUN: -cache-line-size=64 -prefetch-distance=64 \
+; RUN: -passes=loop-data-prefetch -S < %s | FileCheck %s
+
+define void @foo(ptr nocapture %a, ptr nocapture readonly %b) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr nocapture [[A:%.*]], ptr nocapture readonly [[B:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 64
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: call void @llvm.prefetch.p0(ptr [[SCEVGEP]], i32 0, i32 3, i32 1)
+; CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[ADD:%.*]] = fadd double [[TMP2]], 1.000000e+00
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store double [[ADD]], ptr [[ARRAYIDX2]], align 8
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK: for.end:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds double, ptr %b, i64 %indvars.iv
+ %0 = load double, ptr %arrayidx, align 8
+ %add = fadd double %0, 1.000000e+00
+ %arrayidx2 = getelementptr inbounds double, ptr %a, i64 %indvars.iv
+ store double %add, ptr %arrayidx2, align 8
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp eq i64 %indvars.iv.next, 1600
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopDataPrefetch/RISCV/lit.local.cfg b/llvm/test/Transforms/LoopDataPrefetch/RISCV/lit.local.cfg
new file mode 100644
index 000000000000000..17351748513d988
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/RISCV/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "RISCV" in config.root.targets:
+ config.unsupported = True
>From b16bf4a68d21d9e0ed2af84c1e8c615d21c4577b Mon Sep 17 00:00:00 2001
From: wangpc <wangpengcheng.pp at bytedance.com>
Date: Wed, 20 Sep 2023 15:07:16 +0800
Subject: [PATCH 2/3] fixup! [RISCV] Enable LoopDataPrefetch pass
use bits<32> and change default value to UINT_MAX
---
llvm/lib/Target/RISCV/RISCVProcessors.td | 2 +-
llvm/lib/Target/RISCV/RISCVSubtarget.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index f61d45c93bca819..9aac8117b21044e 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -18,7 +18,7 @@ class RISCVTuneInfo {
bits<16> CacheLineSize = 0;
bits<16> PrefetchDistance = 0;
bits<16> MinPrefetchStride = 1;
- bits<16> MaxPrefetchIterationsAhead = 65535;
+ bits<32> MaxPrefetchIterationsAhead = -1;
}
def RISCVTuneInfoTable : GenericTable {
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index a956481db99724a..7427b93f5a61e76 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -43,7 +43,7 @@ struct RISCVTuneInfo {
uint16_t CacheLineSize;
uint16_t PrefetchDistance;
uint16_t MinPrefetchStride;
- uint16_t MaxPrefetchIterationsAhead;
+ unsigned MaxPrefetchIterationsAhead;
};
#define GET_RISCVTuneInfoTable_DECL
>From 419f4c284a9e4ccb3f60f687e0562d9c993c3b24 Mon Sep 17 00:00:00 2001
From: wangpc <wangpengcheng.pp at bytedance.com>
Date: Tue, 26 Sep 2023 13:10:06 +0800
Subject: [PATCH 3/3] fixup! [RISCV] Enable LoopDataPrefetch pass
Remove -riscv-enable-loop-data-prefetch
---
llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
index 0f8c59fd27a93c3..8e1123991a9cc3d 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/RISCV/basic.ll
@@ -1,6 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -mtriple=riscv64 -riscv-enable-loop-data-prefetch \
-; RUN: -cache-line-size=64 -prefetch-distance=64 \
+; RUN: opt -mtriple=riscv64 -cache-line-size=64 -prefetch-distance=64 \
; RUN: -passes=loop-data-prefetch -S < %s | FileCheck %s
define void @foo(ptr nocapture %a, ptr nocapture readonly %b) {
More information about the llvm-commits
mailing list