[llvm] [LoopDataPrefetch] Add support for target-specific memory intrinsics (PR #207358)
Shanzhi Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 04:59:23 PDT 2026
https://github.com/chenshanzhi updated https://github.com/llvm/llvm-project/pull/207358
>From 5ea4e6b44cfd2fd7faa4563bfd0b00b75c2ecd4f Mon Sep 17 00:00:00 2001
From: Shanzhi Chen <chenshanzhi at huawei.com>
Date: Fri, 3 Jul 2026 17:36:49 +0800
Subject: [PATCH] [LoopDataPrefetch] Enable handling of target-specific memory
intrinsics
Add support for target-specific memory intrinsics in LoopDataPrefetch.
This feature is enabled by default for AArch64 and disabled for other
targets.
On AArch64, this allows LoopDataPrefetch to optimize loop data accesses
such as `ld2`, `ld3`, and `ld4`.
Additionally, a new hidden option `-loop-prefetch-tgt-mem-intrinsics` is
added to explicitly override the default target behavior.
---
.../llvm/Analysis/TargetTransformInfo.h | 4 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 1 +
llvm/include/llvm/CodeGen/BasicTTIImpl.h | 4 +
llvm/include/llvm/MC/MCSubtargetInfo.h | 5 ++
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 +
llvm/lib/MC/MCSubtargetInfo.cpp | 2 +
llvm/lib/Target/AArch64/AArch64Subtarget.h | 1 +
.../Transforms/Scalar/LoopDataPrefetch.cpp | 57 +++++++++---
.../AArch64/tgt-mem-intrinsics.ll | 86 +++++++++++++++++++
9 files changed, 152 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 10c0509460b95..14fb0b76519d9 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1463,6 +1463,10 @@ class TargetTransformInfo {
/// \return True if prefetching should also be done for writes.
LLVM_ABI bool enableWritePrefetching() const;
+ /// \return True if prefetching should also be done for target-specific
+ /// memory intrinsics.
+ LLVM_ABI bool enableTgtMemIntrinsicPrefetching() const;
+
/// \return if target want to issue a prefetch in address space \p AS.
LLVM_ABI bool shouldPrefetchAddressSpace(unsigned AS) const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5645cc63a6944..a453b3e1c2249 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -712,6 +712,7 @@ class LLVM_ABI TargetTransformInfoImplBase {
}
virtual unsigned getMaxPrefetchIterationsAhead() const { return UINT_MAX; }
virtual bool enableWritePrefetching() const { return false; }
+ virtual bool enableTgtMemIntrinsicPrefetching() const { return false; }
virtual bool shouldPrefetchAddressSpace(unsigned AS) const { return !AS; }
virtual InstructionCost getPartialReductionCost(
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 55163eb1976bf..e28943dcf86e0 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -880,6 +880,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
return getST()->enableWritePrefetching();
}
+ bool enableTgtMemIntrinsicPrefetching() const override {
+ return getST()->enableTgtMemIntrinsicPrefetching();
+ }
+
bool shouldPrefetchAddressSpace(unsigned AS) const override {
return getST()->shouldPrefetchAddressSpace(AS);
}
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index 0393dea60b938..a0db55e37c859 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -342,6 +342,11 @@ class LLVM_ABI MCSubtargetInfo {
///
virtual bool enableWritePrefetching() const;
+ /// \return True if prefetching should also be done for target-specific
+ /// memory intrinsics.
+ ///
+ virtual bool enableTgtMemIntrinsicPrefetching() const;
+
/// Return the minimum stride necessary to trigger software
/// prefetching.
///
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index cdccd04f4c9ea..727e5d3270b30 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -912,6 +912,10 @@ bool TargetTransformInfo::enableWritePrefetching() const {
return TTIImpl->enableWritePrefetching();
}
+bool TargetTransformInfo::enableTgtMemIntrinsicPrefetching() const {
+ return TTIImpl->enableTgtMemIntrinsicPrefetching();
+}
+
bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
return TTIImpl->shouldPrefetchAddressSpace(AS);
}
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index 2cae4643641a3..bfde524a99511 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -406,6 +406,8 @@ bool MCSubtargetInfo::enableWritePrefetching() const {
return false;
}
+bool MCSubtargetInfo::enableTgtMemIntrinsicPrefetching() const { return false; }
+
unsigned MCSubtargetInfo::getMinPrefetchStride(unsigned NumMemAccesses,
unsigned NumStridedMemAccesses,
unsigned NumPrefetches,
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 101b0a700b41a..974a5c3647e84 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -286,6 +286,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
unsigned getMaxPrefetchIterationsAhead() const override {
return MaxPrefetchIterationsAhead;
}
+ bool enableTgtMemIntrinsicPrefetching() const override { return true; }
Align getPrefFunctionAlignment() const {
return PrefFunctionAlignment;
}
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index fe20c24318e12..c5800b119a0ea 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -40,6 +40,10 @@ static cl::opt<bool>
PrefetchWrites("loop-prefetch-writes", cl::Hidden, cl::init(false),
cl::desc("Prefetch write addresses"));
+static cl::opt<bool> PrefetchTgtMemIntrinsics(
+ "loop-prefetch-tgt-mem-intrinsics", cl::Hidden, cl::init(false),
+ cl::desc("Prefetch addresses from target-specific memory intrinsics"));
+
static cl::opt<unsigned>
PrefetchDistance("prefetch-distance",
cl::desc("Number of instructions to prefetch ahead"),
@@ -102,6 +106,12 @@ class LoopDataPrefetch {
return TTI->enableWritePrefetching();
}
+ bool doPrefetchTgtMemIntrinsics() {
+ if (PrefetchTgtMemIntrinsics.getNumOccurrences() > 0)
+ return PrefetchTgtMemIntrinsics;
+ return TTI->enableTgtMemIntrinsicPrefetching();
+ }
+
AssumptionCache *AC;
DominatorTree *DT;
LoopInfo *LI;
@@ -239,22 +249,26 @@ struct Prefetch {
bool Writes = false;
/// The (first seen) prefetched instruction.
Instruction *MemI = nullptr;
+ /// The pointer that the instruction is loading from or storing to.
+ Value *PtrVal = nullptr;
/// Constructor to create a new Prefetch for \p I.
- Prefetch(const SCEVAddRecExpr *L, Instruction *I) : LSCEVAddRec(L) {
- addInstruction(I);
+ Prefetch(const SCEVAddRecExpr *L, Instruction *I, Value *Ptr, bool WriteMem)
+ : LSCEVAddRec(L) {
+ addInstruction(I, Ptr, WriteMem);
};
/// Add the instruction \param I to this prefetch. If it's not the first
/// one, 'InsertPt' and 'Writes' will be updated as required.
/// \param PtrDiff the known constant address difference to the first added
/// instruction.
- void addInstruction(Instruction *I, DominatorTree *DT = nullptr,
- int64_t PtrDiff = 0) {
+ void addInstruction(Instruction *I, Value *Ptr, bool WriteMem,
+ DominatorTree *DT = nullptr, int64_t PtrDiff = 0) {
if (!InsertPt) {
MemI = I;
+ PtrVal = Ptr;
InsertPt = I;
- Writes = isa<StoreInst>(I);
+ Writes = WriteMem;
} else {
BasicBlock *PrefBB = InsertPt->getParent();
BasicBlock *InsBB = I->getParent();
@@ -264,7 +278,7 @@ struct Prefetch {
InsertPt = DomBB->getTerminator();
}
- if (isa<StoreInst>(I) && PtrDiff == 0)
+ if (WriteMem && PtrDiff == 0)
Writes = true;
}
}
@@ -326,15 +340,35 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
for (auto &I : *BB) {
Value *PtrValue;
Instruction *MemI;
+ bool WriteMem;
if (LoadInst *LMemI = dyn_cast<LoadInst>(&I)) {
MemI = LMemI;
PtrValue = LMemI->getPointerOperand();
+ WriteMem = false;
} else if (StoreInst *SMemI = dyn_cast<StoreInst>(&I)) {
if (!doPrefetchWrites()) continue;
MemI = SMemI;
PtrValue = SMemI->getPointerOperand();
- } else continue;
+ WriteMem = true;
+ } else if (IntrinsicInst *IntrI = dyn_cast<IntrinsicInst>(&I)) {
+ if (!doPrefetchTgtMemIntrinsics())
+ continue;
+ MemIntrinsicInfo IntrInfo;
+ bool IsTgtMemIntrinsic = TTI->getTgtMemIntrinsic(IntrI, IntrInfo);
+ if (!IsTgtMemIntrinsic)
+ continue;
+ if (!IntrInfo.PtrVal)
+ continue;
+ if (!IntrInfo.isUnordered())
+ continue;
+ if (IntrInfo.WriteMem && !doPrefetchWrites())
+ continue;
+ MemI = IntrI;
+ PtrValue = IntrInfo.PtrVal;
+ WriteMem = IntrInfo.WriteMem;
+ } else
+ continue;
unsigned PtrAddrSpace = PtrValue->getType()->getPointerAddressSpace();
if (!TTI->shouldPrefetchAddressSpace(PtrAddrSpace))
@@ -359,14 +393,14 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
dyn_cast<SCEVConstant>(PtrDiff)) {
int64_t PD = std::abs(ConstPtrDiff->getValue()->getSExtValue());
if (PD < (int64_t) TTI->getCacheLineSize()) {
- Pref.addInstruction(MemI, DT, PD);
+ Pref.addInstruction(MemI, PtrValue, WriteMem, DT, PD);
DupPref = true;
break;
}
}
}
if (!DupPref)
- Prefetches.push_back(Prefetch(LSCEVAddRec, MemI));
+ Prefetches.push_back(Prefetch(LSCEVAddRec, MemI, PtrValue, WriteMem));
}
unsigned TargetMinStride =
@@ -409,9 +443,8 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
ConstantInt::get(I32, 3),
ConstantInt::get(I32, 1)});
++NumPrefetches;
- LLVM_DEBUG(dbgs() << " Access: "
- << *P.MemI->getOperand(isa<LoadInst>(P.MemI) ? 0 : 1)
- << ", SCEV: " << *P.LSCEVAddRec << "\n");
+ LLVM_DEBUG(dbgs() << " Access: " << *P.PtrVal
+ << ", SCEV: " << *P.LSCEVAddRec << "\n");
ORE->emit([&]() {
return OptimizationRemark(DEBUG_TYPE, "Prefetched", P.MemI)
<< "prefetched memory access";
diff --git a/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll b/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
new file mode 100644
index 0000000000000..64dc3cf7fef8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopDataPrefetch/AArch64/tgt-mem-intrinsics.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64 -cache-line-size=64 -prefetch-distance=64 \
+; RUN: -passes=loop-data-prefetch -S < %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -cache-line-size=64 -prefetch-distance=64 \
+; RUN: -loop-prefetch-tgt-mem-intrinsics=false -passes=loop-data-prefetch -S < %s | FileCheck %s --check-prefix=DISABLED
+
+define dso_local void @separate_rgb(ptr noundef %rgb, ptr noundef %r, ptr noundef %g, ptr noundef %b) {
+; CHECK-LABEL: define dso_local void @separate_rgb(
+; CHECK-SAME: ptr noundef [[RGB:%.*]], ptr noundef [[R:%.*]], ptr noundef [[G:%.*]], ptr noundef [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP0]], 336
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[RGB]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[RGB]], i64 [[TMP2]]
+; CHECK-NEXT: call void @llvm.prefetch.p0(ptr [[SCEVGEP]], i32 0, i32 3, i32 1)
+; CHECK-NEXT: [[VLD3:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[ADD_PTR]])
+; CHECK-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+; CHECK-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+; CHECK-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT: [[ADD_PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[R]], i64 [[TMP1]]
+; CHECK-NEXT: store <16 x i8> [[VLD3_FCA_0_EXTRACT]], ptr [[ADD_PTR3]], align 1
+; CHECK-NEXT: [[ADD_PTR9:%.*]] = getelementptr inbounds nuw i8, ptr [[G]], i64 [[TMP1]]
+; CHECK-NEXT: store <16 x i8> [[VLD3_FCA_1_EXTRACT]], ptr [[ADD_PTR9]], align 1
+; CHECK-NEXT: [[ADD_PTR15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: store <16 x i8> [[VLD3_FCA_2_EXTRACT]], ptr [[ADD_PTR15]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+; DISABLED-LABEL: define dso_local void @separate_rgb(
+; DISABLED-SAME: ptr noundef [[RGB:%.*]], ptr noundef [[R:%.*]], ptr noundef [[G:%.*]], ptr noundef [[B:%.*]]) {
+; DISABLED-NEXT: [[ENTRY:.*]]:
+; DISABLED-NEXT: br label %[[FOR_BODY:.*]]
+; DISABLED: [[FOR_COND_CLEANUP:.*]]:
+; DISABLED-NEXT: ret void
+; DISABLED: [[FOR_BODY]]:
+; DISABLED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DISABLED-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], 48
+; DISABLED-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[RGB]], i64 [[TMP0]]
+; DISABLED-NEXT: [[VLD3:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[ADD_PTR]])
+; DISABLED-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+; DISABLED-NEXT: [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+; DISABLED-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+; DISABLED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 4
+; DISABLED-NEXT: [[ADD_PTR3:%.*]] = getelementptr inbounds nuw i8, ptr [[R]], i64 [[TMP1]]
+; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_0_EXTRACT]], ptr [[ADD_PTR3]], align 1
+; DISABLED-NEXT: [[ADD_PTR9:%.*]] = getelementptr inbounds nuw i8, ptr [[G]], i64 [[TMP1]]
+; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_1_EXTRACT]], ptr [[ADD_PTR9]], align 1
+; DISABLED-NEXT: [[ADD_PTR15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP1]]
+; DISABLED-NEXT: store <16 x i8> [[VLD3_FCA_2_EXTRACT]], ptr [[ADD_PTR15]], align 1
+; DISABLED-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DISABLED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1600
+; DISABLED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+
+for.cond.cleanup:
+ ret void
+
+for.body:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+ %0 = mul nuw nsw i64 %indvars.iv, 48
+ %add.ptr = getelementptr inbounds nuw i8, ptr %rgb, i64 %0
+ %vld3 = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr %add.ptr)
+ %vld3.fca.0.extract = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %vld3, 0
+ %vld3.fca.1.extract = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %vld3, 1
+ %vld3.fca.2.extract = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %vld3, 2
+ %1 = shl nuw nsw i64 %indvars.iv, 4
+ %add.ptr3 = getelementptr inbounds nuw i8, ptr %r, i64 %1
+ store <16 x i8> %vld3.fca.0.extract, ptr %add.ptr3, align 1
+ %add.ptr9 = getelementptr inbounds nuw i8, ptr %g, i64 %1
+ store <16 x i8> %vld3.fca.1.extract, ptr %add.ptr9, align 1
+ %add.ptr15 = getelementptr inbounds nuw i8, ptr %b, i64 %1
+ store <16 x i8> %vld3.fca.2.extract, ptr %add.ptr15, align 1
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, 1600
+ br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
More information about the llvm-commits
mailing list