[llvm] [LoopDataPrefetch] Enable handling of target-specific memory intrinsics (PR #207358)
Shanzhi Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 20:05:31 PDT 2026
https://github.com/chenshanzhi updated https://github.com/llvm/llvm-project/pull/207358
>From 5ea4e6b44cfd2fd7faa4563bfd0b00b75c2ecd4f Mon Sep 17 00:00:00 2001
From: Shanzhi Chen <chenshanzhi at huawei.com>
Date: Fri, 3 Jul 2026 17:36:49 +0800
Subject: [PATCH 1/2] [LoopDataPrefetch] Enable handling of target-specific
memory intrinsics
Add support for target-specific memory intrinsics in LoopDataPrefetch.
This feature is enabled by default for AArch64 and disabled for other
targets.
On AArch64, this allows LoopDataPrefetch to optimize loop data accesses
such as `ld2`, `ld3`, and `ld4`.
Additionally, a new hidden option `-loop-prefetch-tgt-mem-intrinsics` is
added to explicitly override the default target behavior.
---
.../llvm/Analysis/TargetTransformInfo.h | 4 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 1 +
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 +
llvm/include/llvm/MC/MCSubtargetInfo.h | 5 ++
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 +
llvm/lib/MC/MCSubtargetInfo.cpp | 2 +
llvm/lib/Target/AArch64/AArch64Subtarget.h | 1 +
.../Transforms/Scalar/LoopDataPrefetch.cpp | 57 +++++++++---
.../AArch64/tgt-mem-intrinsics.ll | 86 +++++++++++++++++++
9 files changed, 152 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 10c0509460b95..14fb0b76519d9 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1463,6 +1463,10 @@ class TargetTransformInfo {
/// \return True if prefetching should also be done for writes.
LLVM_ABI bool enableWritePrefetching() const;
+ /// \return True if prefetching should also be done for target-specific
+ /// memory intrinsics.
+ LLVM_ABI bool enableTgtMemIntrinsicPrefetching() const;
+
/// \return if target want to issue a prefetch in address space \p AS.
LLVM_ABI bool shouldPrefetchAddressSpace(unsigned AS) const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5645cc63a6944..a453b3e1c2249 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -712,6 +712,7 @@ class LLVM_ABI TargetTransformInfoImplBase {
}
virtual unsigned getMaxPrefetchIterationsAhead() const { return UINT_MAX; }
virtual bool enableWritePrefetching() const { return false; }
+ virtual bool enableTgtMemIntrinsicPrefetching() const { return false; }
virtual bool shouldPrefetchAddressSpace(unsigned AS) const { return !AS; }
virtual InstructionCost getPartialReductionCost(
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 55163eb1976bf..e28943dcf86e0 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -880,6 +880,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return getST()->enableWritePrefetching();
}
+ bool enableTgtMemIntrinsicPrefetching() const override {
+ return getST()->enableTgtMemIntrinsicPrefetching();
+ }
+
bool shouldPrefetchAddressSpace(unsigned AS) const override {
return getST()->shouldPrefetchAddressSpace(AS);
}
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index 0393dea60b938..a0db55e37c859 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -342,6 +342,11 @@ class LLVM_ABI MCSubtargetInfo {
///
virtual bool enableWritePrefetching() const;
+ /// \return True if prefetching should also be done for target-specific
+ /// memory intrinsics.
+ ///
+ virtual bool enableTgtMemIntrinsicPrefetching() const;
+
/// Return the minimum stride necessary to trigger software
/// prefetching.
///
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index cdccd04f4c9ea..727e5d3270b30 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -912,6 +912,10 @@ bool TargetTransformInfo::enableWritePrefetching() const {
return TTIImpl->enableWritePrefetching();
}
+bool TargetTransformInfo::enableTgtMemIntrinsicPrefetching() const {
+ return TTIImpl->enableTgtMemIntrinsicPrefetching();
+}
+
bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
return TTIImpl->shouldPrefetchAddressSpace(AS);
}
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index 2cae4643641a3..bfde524a99511 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -406,6 +406,8 @@ bool MCSubtargetInfo::enableWritePrefetching() const {
return false;
}
+bool MCSubtargetInfo::enableTgtMemIntrinsicPrefetching() const { return false; }
+
unsigned MCSubtargetInfo::getMinPrefetchStride(unsigned NumMemAccesses,
unsigned NumStridedMemAccesses,
unsigned NumPrefetches,
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 101b0a700b41a..974a5c3647e84 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -286,6 +286,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
unsigned getMaxPrefetchIterationsAhead() const override {
return MaxPrefetchIterationsAhead;
}
+ bool enableTgtMemIntrinsicPrefetching() const override { return true; }
Align getPrefFunctionAlignment() const {
return PrefFunctionAlignment;
}
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index fe20c24318e12..c5800b119a0ea 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -40,6 +40,10 @@ static cl::opt<bool>
PrefetchWrites("loop-prefetch-writes", cl::Hidden, cl::init(false),
cl::desc("Prefetch write addresses"));
+static cl::opt<bool> PrefetchTgtMemIntrinsics(
+ "loop-prefetch-tgt-mem-intrinsics", cl::Hidden, cl::init(false),
+ cl::desc("Prefetch addresses from target-specific memory intrinsics"));
+
static cl::opt<unsigned>
PrefetchDistance("prefetch-distance",
cl::desc("Number of instructions to prefetch ahead"),
@@ -102,6 +106,12 @@ class LoopDataPrefetch {
return TTI->enableWritePrefetching();
}
+ bool doPrefetchTgtMemIntrinsics() {
+ if (PrefetchTgtMemIntrinsics.getNumOccurrences() > 0)
+ return PrefetchTgtMemIntrinsics;
+ return TTI->enableTgtMemIntrinsicPrefetching();
+ }
+
AssumptionCache *AC;
DominatorTree *DT;
LoopInfo *LI;
@@ -239,22 +249,26 @@ struct Prefetch {
bool Writes = false;
/// The (first seen) prefetched instruction.
Instruction *MemI = nullptr;
+ /// The pointer that the instruction is loading from or storing to.
+ Value *PtrVal = nullptr;
/// Constructor to create a new Prefetch for \p I.
- Prefetch(const SCEVAddRecExpr *L, Instruction *I) : LSCEVAddRec(L) {
- addInstruction(I);
+ Prefetch(const SCEVAddRecExpr *L, Instruction *I, Value *Ptr, bool WriteMem)
+ : LSCEVAddRec(L) {
+ addInstruction(I, Ptr, WriteMem);
};
/// Add the instruction \param I to this prefetch. If it's not the first
/// one, 'InsertPt' and 'Writes' will be updated as required.
/// \param PtrDiff the known constant address difference to the first added
/// instruction.
- void addInstruction(Instruction *I, DominatorTree *DT = nullptr,
- int64_t PtrDiff = 0) {
+ void addInstruction(Instruction *I, Value *Ptr, bool WriteMem,
+ DominatorTree *DT = nullptr, int64_t PtrDiff = 0) {
if (!InsertPt) {
MemI = I;
+ PtrVal = Ptr;
InsertPt = I;
- Writes = isa<StoreInst>(I);
+ Writes = WriteMem;
} else {
BasicBlock *PrefBB = InsertPt->getParent();
BasicBlock *InsBB = I->getParent();
@@ -264,7 +278,7 @@ struct Prefetch {
InsertPt = DomBB->getTerminator();
}
- if (isa<StoreInst>(I) && PtrDiff == 0)
+ if (WriteMem && PtrDiff == 0)
Writes = true;
}
}
@@ -326,15 +340,35 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
for (auto &I : *BB) {
Value *PtrValue;
Instruction *MemI;
+ bool WriteMem;
if (LoadInst *LMemI = dyn_cast<LoadInst>(&I)) {
MemI = LMemI;
PtrValue = LMemI->getPointerOperand();
+ WriteMem = false;
} else if (StoreInst *SMemI = dyn_cast<StoreInst>(&I)) {
if (!doPrefetchWrites()) continue;
MemI = SMemI;
PtrValue = SMemI->getPointerOperand();
- } else continue;
+ WriteMem = true;
+ } else if (IntrinsicInst *IntrI = dyn_cast<IntrinsicInst>(&I)) {
+ if (!doPrefetchTgtMemIntrinsics())
+ continue;
+ MemIntrinsicInfo IntrInfo;
+ bool IsTgtMemIntrinsic = TTI->getTgtMemIntrinsic(IntrI, IntrInfo);
+ if (!IsTgtMemIntrinsic)
+ continue;
+ if (!IntrInfo.PtrVal)
+ continue;
+ if (!IntrInfo.isUnordered())
+ continue;
+ if (IntrInfo.WriteMem && !doPrefetchWrites())
+ continue;
+ MemI = IntrI;
+ PtrValue = IntrInfo.PtrVal;
+ WriteMem = IntrInfo.WriteMem;
+ } else
+ continue;
unsigned PtrAddrSpace = PtrValue->getType()->getPointerAddressSpace();
if (!TTI->shouldPrefetchAddressSpace(PtrAddrSpace))
@@ -359,14 +393,14 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
dyn_cast<SCEVConstant>(PtrDiff)) {
int64_t PD = std::abs(ConstPtrDiff->getValue()->getSExtValue());
if (PD < (int64_t) TTI->getCacheLineSize()) {
- Pref.addInstruction(MemI, DT, PD);
+ Pref.addInstruction(MemI, PtrValue, WriteMem, DT, PD);
DupPref = true;
break;
}
}
}
if (!DupPref)
- Prefetches.push_back(Prefetch(LSCEVAddRec, MemI));
+ Prefetches.push_back(Prefetch(LSCEVAddRec, MemI, PtrValue, WriteMem));
}
unsigned TargetMinStride =
@@ -409,9 +443,8 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
ConstantInt::get(I32, 3),
ConstantInt::get(I32, 1)});
++NumPrefetches;
- LLVM_DEBUG(dbgs() << " Access: "
- << *P.MemI->getOperand(isa<LoadInst>(P.MemI) ? 0 : 1)
- << ", SCEV: " << *P.LSCEVAddRec << "\n");
+ LLVM_DEBUG(dbgs() << " Access: " << *P.PtrVal
+ << ", SCEV: " << *P.LSCEVAddRec << "\n");
ORE->emit([&]() {
return OptimizationRemark(DEBUG_TYPE, "Prefetched", P.MemI)
<< "prefetched memory access";
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
new file mode 100644
index 0000000000000..64dc3cf7fef8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64 -cache-line-size=64 -prefetch-distance=64 \
+; RUN: -passes=loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -cache-line-size=64 -prefetch-distance=64 \
+; RUN: -loop-prefetch-tgt-mem-intrinsics=false -passes=loop-data-prefetch -S < %s | FileCheck %s --check-prefix=DISABLED
+
+define dso_local void @separate_rgb(ptr noundef %rgb, ptr noundef %r, ptr noundef %g, ptr noundef %b) {
+; CHECK-LABEL: define dso_local void @separate_rgb(
+; CHECK-SAME: ptr noundef [[RGB:%.*]], ptr noundef [[R:%.*]], ptr noundef [[G:%.*]], ptr noundef [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP0]], 336
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[RGB]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[RGB]], i64 [[TMP2]]
+; CHECK-NEXT: call void @llvm.prefetch.p0(ptr [[SCEVGEP]], i32 0, i32 3, i32 1)
+; CHECK-NEXT: [[VLD3:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[ADD_PTR]])
+; CHECK-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+; CHECK-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+; CHECK-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[ADD_PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[R]], i64 [[TMP1]]
+; CHECK-NEXT: store <16 x i8> [[VLD3_FCA_0_EXTRACT]], ptr [[ADD_PTR3]], align 1
+; CHECK-NEXT: [[ADD_PTR9:%.*]] = getelementptr inbounds nuw i8, ptr [[G]], i64 [[TMP1]]
+; CHECK-NEXT: store <16 x i8> [[VLD3_FCA_1_EXTRACT]], ptr [[ADD_PTR9]], align 1
+; CHECK-NEXT: [[ADD_PTR15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: store <16 x i8> [[VLD3_FCA_2_EXTRACT]], ptr [[ADD_PTR15]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+; DISABLED-LABEL: define dso_local void @separate_rgb(
+; DISABLED-SAME: ptr noundef [[RGB:%.*]], ptr noundef [[R:%.*]], ptr noundef [[G:%.*]], ptr noundef [[B:%.*]]) {
+; DISABLED-NEXT: [[ENTRY:.*]]:
+; DISABLED-NEXT: br label %[[FOR_BODY:.*]]
+; DISABLED: [[FOR_COND_CLEANUP:.*]]:
+; DISABLED-NEXT: ret void
+; DISABLED: [[FOR_BODY]]:
+; DISABLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DISABLED-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
+; DISABLED-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[RGB]], i64 [[TMP0]]
+; DISABLED-NEXT: [[VLD3:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[ADD_PTR]])
+; DISABLED-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+; DISABLED-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+; DISABLED-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+; DISABLED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 4
+; DISABLED-NEXT: [[ADD_PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[R]], i64 [[TMP1]]
+; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_0_EXTRACT]], ptr [[ADD_PTR3]], align 1
+; DISABLED-NEXT: [[ADD_PTR9:%.*]] = getelementptr inbounds nuw i8, ptr [[G]], i64 [[TMP1]]
+; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_1_EXTRACT]], ptr [[ADD_PTR9]], align 1
+; DISABLED-NEXT: [[ADD_PTR15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP1]]
+; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_2_EXTRACT]], ptr [[ADD_PTR15]], align 1
+; DISABLED-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DISABLED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
+; DISABLED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = mul nuw nsw i64 %indvars.iv, 48
+ %add.ptr = getelementptr inbounds nuw i8, ptr %rgb, i64 %0
+ %vld3 = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr %add.ptr)
+ %vld3.fca.0.extract = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %vld3, 0
+ %vld3.fca.1.extract = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %vld3, 1
+ %vld3.fca.2.extract = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %vld3, 2
+ %1 = shl nuw nsw i64 %indvars.iv, 4
+ %add.ptr3 = getelementptr inbounds nuw i8, ptr %r, i64 %1
+ store <16 x i8> %vld3.fca.0.extract, ptr %add.ptr3, align 1
+ %add.ptr9 = getelementptr inbounds nuw i8, ptr %g, i64 %1
+ store <16 x i8> %vld3.fca.1.extract, ptr %add.ptr9, align 1
+ %add.ptr15 = getelementptr inbounds nuw i8, ptr %b, i64 %1
+ store <16 x i8> %vld3.fca.2.extract, ptr %add.ptr15, align 1
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1600
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
>From bad1ec2ad6d9e27e3d68e18b0071e6935e3df6b6 Mon Sep 17 00:00:00 2001
From: Shanzhi Chen <chenshanzhi at huawei.com>
Date: Tue, 7 Jul 2026 10:52:56 +0800
Subject: [PATCH 2/2] fixup! [LoopDataPrefetch] Enable handling of
target-specific memory intrinsics
---
.../llvm/Analysis/TargetTransformInfo.h | 4 ---
.../llvm/Analysis/TargetTransformInfoImpl.h | 1 -
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 ---
llvm/include/llvm/MC/MCSubtargetInfo.h | 5 ----
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 ---
llvm/lib/MC/MCSubtargetInfo.cpp | 2 --
llvm/lib/Target/AArch64/AArch64Subtarget.h | 1 -
.../Transforms/Scalar/LoopDataPrefetch.cpp | 14 +++-------
.../AArch64/tgt-mem-intrinsics.ll | 27 -------------------
9 files changed, 3 insertions(+), 59 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 14fb0b76519d9..10c0509460b95 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1463,10 +1463,6 @@ class TargetTransformInfo {
/// \return True if prefetching should also be done for writes.
LLVM_ABI bool enableWritePrefetching() const;
- /// \return True if prefetching should also be done for target-specific
- /// memory intrinsics.
- LLVM_ABI bool enableTgtMemIntrinsicPrefetching() const;
-
/// \return if target want to issue a prefetch in address space \p AS.
LLVM_ABI bool shouldPrefetchAddressSpace(unsigned AS) const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index a453b3e1c2249..5645cc63a6944 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -712,7 +712,6 @@ class LLVM_ABI TargetTransformInfoImplBase {
}
virtual unsigned getMaxPrefetchIterationsAhead() const { return UINT_MAX; }
virtual bool enableWritePrefetching() const { return false; }
- virtual bool enableTgtMemIntrinsicPrefetching() const { return false; }
virtual bool shouldPrefetchAddressSpace(unsigned AS) const { return !AS; }
virtual InstructionCost getPartialReductionCost(
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e28943dcf86e0..55163eb1976bf 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -880,10 +880,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return getST()->enableWritePrefetching();
}
- bool enableTgtMemIntrinsicPrefetching() const override {
- return getST()->enableTgtMemIntrinsicPrefetching();
- }
-
bool shouldPrefetchAddressSpace(unsigned AS) const override {
return getST()->shouldPrefetchAddressSpace(AS);
}
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index a0db55e37c859..0393dea60b938 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -342,11 +342,6 @@ class LLVM_ABI MCSubtargetInfo {
///
virtual bool enableWritePrefetching() const;
- /// \return True if prefetching should also be done for target-specific
- /// memory intrinsics.
- ///
- virtual bool enableTgtMemIntrinsicPrefetching() const;
-
/// Return the minimum stride necessary to trigger software
/// prefetching.
///
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 727e5d3270b30..cdccd04f4c9ea 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -912,10 +912,6 @@ bool TargetTransformInfo::enableWritePrefetching() const {
return TTIImpl->enableWritePrefetching();
}
-bool TargetTransformInfo::enableTgtMemIntrinsicPrefetching() const {
- return TTIImpl->enableTgtMemIntrinsicPrefetching();
-}
-
bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
return TTIImpl->shouldPrefetchAddressSpace(AS);
}
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index bfde524a99511..2cae4643641a3 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -406,8 +406,6 @@ bool MCSubtargetInfo::enableWritePrefetching() const {
return false;
}
-bool MCSubtargetInfo::enableTgtMemIntrinsicPrefetching() const { return false; }
-
unsigned MCSubtargetInfo::getMinPrefetchStride(unsigned NumMemAccesses,
unsigned NumStridedMemAccesses,
unsigned NumPrefetches,
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 974a5c3647e84..101b0a700b41a 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -286,7 +286,6 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
unsigned getMaxPrefetchIterationsAhead() const override {
return MaxPrefetchIterationsAhead;
}
- bool enableTgtMemIntrinsicPrefetching() const override { return true; }
Align getPrefFunctionAlignment() const {
return PrefFunctionAlignment;
}
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index c5800b119a0ea..18e19290f7f15 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -24,6 +24,7 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Scalar.h"
@@ -40,10 +41,6 @@ static cl::opt<bool>
PrefetchWrites("loop-prefetch-writes", cl::Hidden, cl::init(false),
cl::desc("Prefetch write addresses"));
-static cl::opt<bool> PrefetchTgtMemIntrinsics(
- "loop-prefetch-tgt-mem-intrinsics", cl::Hidden, cl::init(false),
- cl::desc("Prefetch addresses from target-specific memory intrinsics"));
-
static cl::opt<unsigned>
PrefetchDistance("prefetch-distance",
cl::desc("Number of instructions to prefetch ahead"),
@@ -106,12 +103,6 @@ class LoopDataPrefetch {
return TTI->enableWritePrefetching();
}
- bool doPrefetchTgtMemIntrinsics() {
- if (PrefetchTgtMemIntrinsics.getNumOccurrences() > 0)
- return PrefetchTgtMemIntrinsics;
- return TTI->enableTgtMemIntrinsicPrefetching();
- }
-
AssumptionCache *AC;
DominatorTree *DT;
LoopInfo *LI;
@@ -352,7 +343,8 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
PtrValue = SMemI->getPointerOperand();
WriteMem = true;
} else if (IntrinsicInst *IntrI = dyn_cast<IntrinsicInst>(&I)) {
- if (!doPrefetchTgtMemIntrinsics())
+ const Triple &TargetTriple = BB->getModule()->getTargetTriple();
+ if (!TargetTriple.isAArch64())
continue;
MemIntrinsicInfo IntrInfo;
bool IsTgtMemIntrinsic = TTI->getTgtMemIntrinsic(IntrI, IntrInfo);
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
index 64dc3cf7fef8e..4a65fd30c516e 100644
--- a/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
@@ -1,8 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=aarch64 -cache-line-size=64 -prefetch-distance=64 \
; RUN: -passes=loop-data-prefetch -S < %s | FileCheck %s
-; RUN: opt -mtriple=aarch64 -cache-line-size=64 -prefetch-distance=64 \
-; RUN: -loop-prefetch-tgt-mem-intrinsics=false -passes=loop-data-prefetch -S < %s | FileCheck %s --check-prefix=DISABLED
define dso_local void @separate_rgb(ptr noundef %rgb, ptr noundef %r, ptr noundef %g, ptr noundef %b) {
; CHECK-LABEL: define dso_local void @separate_rgb(
@@ -34,31 +32,6 @@ define dso_local void @separate_rgb(ptr noundef %rgb, ptr noundef %r, ptr nounde
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
;
-; DISABLED-LABEL: define dso_local void @separate_rgb(
-; DISABLED-SAME: ptr noundef [[RGB:%.*]], ptr noundef [[R:%.*]], ptr noundef [[G:%.*]], ptr noundef [[B:%.*]]) {
-; DISABLED-NEXT: [[ENTRY:.*]]:
-; DISABLED-NEXT: br label %[[FOR_BODY:.*]]
-; DISABLED: [[FOR_COND_CLEANUP:.*]]:
-; DISABLED-NEXT: ret void
-; DISABLED: [[FOR_BODY]]:
-; DISABLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; DISABLED-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
-; DISABLED-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[RGB]], i64 [[TMP0]]
-; DISABLED-NEXT: [[VLD3:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[ADD_PTR]])
-; DISABLED-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
-; DISABLED-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
-; DISABLED-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
-; DISABLED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 4
-; DISABLED-NEXT: [[ADD_PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[R]], i64 [[TMP1]]
-; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_0_EXTRACT]], ptr [[ADD_PTR3]], align 1
-; DISABLED-NEXT: [[ADD_PTR9:%.*]] = getelementptr inbounds nuw i8, ptr [[G]], i64 [[TMP1]]
-; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_1_EXTRACT]], ptr [[ADD_PTR9]], align 1
-; DISABLED-NEXT: [[ADD_PTR15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP1]]
-; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_2_EXTRACT]], ptr [[ADD_PTR15]], align 1
-; DISABLED-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; DISABLED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
-; DISABLED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
-;
entry:
br label %for.body
More information about the llvm-commits
mailing list