[llvm] [SDAG][NVPTX] Cache control metadata support and lowering (PR #204067)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 16:56:13 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/204067
>From fc5e8f472004efe359001f3f0a6e99bae5d556d7 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 16 Jun 2026 06:09:07 +0000
Subject: [PATCH 01/33] refactor
---
llvm/include/llvm/CodeGen/CodeGenCommonISel.h | 5 +
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 5 +
llvm/include/llvm/CodeGen/MachineFunction.h | 20 +-
llvm/include/llvm/CodeGen/MachineMemOperand.h | 14 +-
llvm/include/llvm/CodeGen/SelectionDAG.h | 30 +-
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 22 +
llvm/lib/CodeGen/CodeGenCommonISel.cpp | 21 +
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 18 +-
llvm/lib/CodeGen/MIRParser/MILexer.cpp | 1 +
llvm/lib/CodeGen/MIRParser/MILexer.h | 1 +
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 12 +-
llvm/lib/CodeGen/MachineFunction.cpp | 22 +-
llvm/lib/CodeGen/MachineOperand.cpp | 15 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 61 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 20 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 2 +-
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp | 76 ++
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.h | 5 +
llvm/lib/Target/NVPTX/NVPTX.h | 74 +-
llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp | 7 +
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 173 +++-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 8 +
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 169 ++--
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 38 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 17 +
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 2 +-
.../floating-point-immediate-operands.mir | 12 +-
llvm/test/CodeGen/MIR/X86/memory-operands.mir | 6 +-
llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll | 54 +
.../CodeGen/NVPTX/cache-hint-sm-version.ll | 346 +++++++
llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll | 131 +++
.../CodeGen/NVPTX/load-store-cache-hint.ll | 935 ++++++++++++++++++
.../NVPTX/machinelicm-no-preheader.mir | 16 +-
llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll | 406 ++++++++
llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir | 8 +-
llvm/test/DebugInfo/NVPTX/inlinedAt_2.mir | 4 +-
llvm/tools/llvm-reduce/ReducerWorkItem.cpp | 2 +-
.../CodeGen/GlobalISel/GISelAliasTest.cpp | 2 +-
.../GlobalISel/MachineIRBuilderTest.cpp | 2 +-
39 files changed, 2584 insertions(+), 178 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
create mode 100644 llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
create mode 100644 llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll
create mode 100644 llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll
diff --git a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
index 9416fa83e9c5e..d0080bea5b1aa 100644
--- a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
+++ b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
@@ -19,6 +19,8 @@
namespace llvm {
class BasicBlock;
+class Instruction;
+class MDNode;
enum FPClassTest : unsigned;
/// Encapsulates all of the information needed to generate a stack protector
@@ -226,6 +228,9 @@ findSplitPointForStackProtector(MachineBasicBlock *BB,
/// simpler test.
LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp);
+LLVM_ABI const MDNode *getMemCacheHintMetadata(const Instruction &I,
+ unsigned OperandNo = 0);
+
/// Assuming the instruction \p MI is going to be deleted, attempt to salvage
/// debug users of \p MI by writing the effect of \p MI in a DIExpression.
LLVM_ABI void salvageDebugInfoForDbgValue(const MachineRegisterInfo &MRI,
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index e171aaa19a3db..2616c011d7997 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -194,6 +194,11 @@ class GAnyLoad : public GLoadStore {
return getMMO().getRanges();
}
+ /// Returns the cache hint metadata for this load.
+ const MDNode *getMemCacheHint() const {
+ return getMMO().getMemCacheHint();
+ }
+
static bool classof(const MachineInstr *MI) {
switch (MI->getOpcode()) {
case TargetOpcode::G_LOAD:
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index b3c814e1c3590..9ce8bb242fb42 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1115,34 +1115,38 @@ class LLVM_ABI MachineFunction {
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
- BaseAlignment, AAInfo, Ranges, SSID, Ordering,
- FailureOrdering);
+ BaseAlignment, AAInfo, Ranges, MemCacheHint,
+ SSID, Ordering, FailureOrdering);
}
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize Size,
Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
+ const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
- BaseAlignment, AAInfo, Ranges, SSID, Ordering,
- FailureOrdering);
+ BaseAlignment, AAInfo, Ranges, MemCacheHint,
+ SSID, Ordering, FailureOrdering);
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 79eedb8b74f6f..809fa5a4bc28e 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -182,6 +182,7 @@ class MachineMemOperand {
MachineAtomicInfo AtomicInfo;
AAMDNodes AAInfo;
const MDNode *Ranges;
+ const MDNode *MemCacheHint;
public:
/// Construct a MachineMemOperand object with the specified PtrInfo, flags,
@@ -195,14 +196,16 @@ class MachineMemOperand {
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic,
+ const MDNode *MemCacheHint = nullptr);
LLVM_ABI
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic,
+ const MDNode *MemCacheHint = nullptr);
const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
@@ -271,6 +274,9 @@ class MachineMemOperand {
/// Return the range tag for the memory reference.
const MDNode *getRanges() const { return Ranges; }
+ /// Return the cache hint metadata for the memory reference.
+ const MDNode *getMemCacheHint() const { return MemCacheHint; }
+
/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const {
return static_cast<SyncScope::ID>(AtomicInfo.SSID);
@@ -338,6 +344,9 @@ class MachineMemOperand {
/// Unset the tracked range metadata.
void clearRanges() { Ranges = nullptr; }
+ /// Unset the cache hint metadata.
+ void clearMemCacheHint() { MemCacheHint = nullptr; }
+
/// Support for operator<<.
/// @{
LLVM_ABI void print(raw_ostream &OS, ModuleSlotTracker &MST,
@@ -355,6 +364,7 @@ class MachineMemOperand {
LHS.getFlags() == RHS.getFlags() &&
LHS.getAAInfo() == RHS.getAAInfo() &&
LHS.getRanges() == RHS.getRanges() &&
+ LHS.getMemCacheHint() == RHS.getMemCacheHint() &&
LHS.getAlign() == RHS.getAlign() &&
LHS.getAddrSpace() == RHS.getAddrSpace() &&
LHS.getSuccessOrdering() == RHS.getSuccessOrdering() &&
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index abc2e1f0ebe4f..5e9f00d997dad 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1526,7 +1526,8 @@ class SelectionDAG {
EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr);
+ const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr);
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
MachineMemOperand *MMO);
LLVM_ABI SDValue
@@ -1534,7 +1535,8 @@ class SelectionDAG {
SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes());
+ const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *MemCacheHint = nullptr);
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
SDValue Chain, SDValue Ptr, EVT MemVT,
MachineMemOperand *MMO);
@@ -1546,17 +1548,19 @@ class SelectionDAG {
SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
EVT MemVT, Align Alignment,
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr);
+ const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr);
inline SDValue getLoad(
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
EVT MemVT, MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) {
+ const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr) {
// Ensures that codegen never sees a None Alignment.
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo,
- Ranges);
+ Ranges, MemCacheHint);
}
LLVM_ABI SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
@@ -1571,15 +1575,17 @@ class SelectionDAG {
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, Align Alignment,
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes());
+ const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *MemCacheHint = nullptr);
inline SDValue
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *MemCacheHint = nullptr) {
return getStore(Chain, dl, Val, Ptr, PtrInfo,
Alignment.value_or(getEVTAlign(Val.getValueType())),
- MMOFlags, AAInfo);
+ MMOFlags, AAInfo, MemCacheHint);
}
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, MachineMemOperand *MMO);
@@ -1587,16 +1593,18 @@ class SelectionDAG {
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes());
+ const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *MemCacheHint = nullptr);
inline SDValue
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, EVT SVT,
MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes()) {
+ const AAMDNodes &AAInfo = AAMDNodes(),
+ const MDNode *MemCacheHint = nullptr) {
return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
Alignment.value_or(getEVTAlign(SVT)), MMOFlags,
- AAInfo);
+ AAInfo, MemCacheHint);
}
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index b280bd48c94e9..79716099c2e05 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1469,6 +1469,11 @@ class MemSDNode : public SDNode {
/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return getMemOperand()->getRanges(); }
+ /// Returns the cache hint metadata for this memory access.
+ const MDNode *getMemCacheHint() const {
+ return getMemOperand()->getMemCacheHint();
+ }
+
/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const {
return getMemOperand()->getSyncScopeID();
@@ -1572,6 +1577,23 @@ class MemSDNode : public SDNode {
refineRanges(ArrayRef(NewMMO));
}
+ /// Refine cache hint metadata for all MMOs. The NewMMOs array must parallel
+ /// memoperands(). For each pair, if cache hints differ, the stored hint is
+ /// cleared.
+ void refineMemCacheHints(ArrayRef<MachineMemOperand *> NewMMOs) {
+ ArrayRef<MachineMemOperand *> MMOs = memoperands();
+ assert(NewMMOs.size() == MMOs.size() && "MMO count mismatch");
+ for (auto [MMO, NewMMO] : zip(MMOs, NewMMOs)) {
+ if (MMO->getMemCacheHint() &&
+ MMO->getMemCacheHint() != NewMMO->getMemCacheHint())
+ MMO->clearMemCacheHint();
+ }
+ }
+
+ void refineMemCacheHints(MachineMemOperand *NewMMO) {
+ refineMemCacheHints(ArrayRef(NewMMO));
+ }
+
const SDValue &getChain() const { return getOperand(0); }
const SDValue &getBasePtr() const {
diff --git a/llvm/lib/CodeGen/CodeGenCommonISel.cpp b/llvm/lib/CodeGen/CodeGenCommonISel.cpp
index 4cd2f6ae2fdb1..c07b6bd3dbd1e 100644
--- a/llvm/lib/CodeGen/CodeGenCommonISel.cpp
+++ b/llvm/lib/CodeGen/CodeGenCommonISel.cpp
@@ -17,12 +17,33 @@
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/Support/Casting.h"
#define DEBUG_TYPE "codegen-common"
using namespace llvm;
+const MDNode *llvm::getMemCacheHintMetadata(const Instruction &I,
+ unsigned OperandNo) {
+ const MDNode *MD = I.getMetadata(LLVMContext::MD_mem_cache_hint);
+ if (!MD)
+ return nullptr;
+
+ for (unsigned Idx = 0, E = MD->getNumOperands(); Idx != E; Idx += 2) {
+ const auto *OpNoCI = mdconst::extract<ConstantInt>(MD->getOperand(Idx));
+ const auto *Hint = cast<MDNode>(MD->getOperand(Idx + 1));
+ if (OpNoCI->getZExtValue() == OperandNo)
+ return Hint;
+ }
+
+ return nullptr;
+}
+
/// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
/// is 0.
MachineBasicBlock *
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 7a8f2a8431f9b..97295f81213fe 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1419,7 +1419,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI), AAInfo,
- LI.getMetadata(LLVMContext::MD_range), LI.getSyncScopeID(),
+ LI.getMetadata(LLVMContext::MD_range), nullptr, LI.getSyncScopeID(),
LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
return true;
@@ -1436,7 +1436,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
Align BaseAlign = getMemOpAlign(LI);
auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
commonAlignment(BaseAlign, Offsets[i]),
- AAInfo, nullptr, LI.getSyncScopeID(),
+ AAInfo, nullptr, nullptr, LI.getSyncScopeID(),
LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -1467,7 +1467,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(SI.getPointerOperand()), Flags,
MRI->getType(Vals[0]), getMemOpAlign(SI), SI.getAAMetadata(), nullptr,
- SI.getSyncScopeID(), SI.getOrdering());
+ nullptr, SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[0], Base, *MMO);
return true;
}
@@ -1483,7 +1483,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
Align BaseAlign = getMemOpAlign(SI);
auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),
commonAlignment(BaseAlign, Offsets[i]),
- SI.getAAMetadata(), nullptr,
+ SI.getAAMetadata(), nullptr, nullptr,
SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
}
@@ -2949,7 +2949,8 @@ bool IRTranslator::translateIntrinsic(
}
MIB.addMemOperand(MF->getMachineMemOperand(
MPI, Info.flags, MemTy, Alignment, CB.getAAMetadata(),
- /*Ranges=*/nullptr, Info.ssid, Info.order, Info.failureOrder));
+ /*Ranges=*/nullptr, /*MemCacheHint=*/nullptr, Info.ssid, Info.order,
+ Info.failureOrder));
}
if (CB.isConvergent()) {
@@ -3574,8 +3575,9 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
- getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
- I.getSuccessOrdering(), I.getFailureOrdering()));
+ getMemOpAlign(I), I.getAAMetadata(), nullptr, nullptr,
+ I.getSyncScopeID(), I.getSuccessOrdering(),
+ I.getFailureOrdering()));
return true;
}
@@ -3670,7 +3672,7 @@ bool IRTranslator::translateAtomicRMW(const User &U,
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, MRI->getType(Val), getMemOpAlign(I),
- I.getAAMetadata(), nullptr, I.getSyncScopeID(),
+ I.getAAMetadata(), nullptr, nullptr, I.getSyncScopeID(),
I.getOrdering()));
return true;
}
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
index 4188ff8f85131..9b29a39a90ae2 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp
@@ -640,6 +640,7 @@ static MIToken::TokenKind getMetadataKeywordKind(StringRef Identifier) {
.Case("!alias.scope", MIToken::md_alias_scope)
.Case("!noalias", MIToken::md_noalias)
.Case("!range", MIToken::md_range)
+ .Case("!mem.cache_hint", MIToken::md_mem_cache_hint)
.Case("!DIExpression", MIToken::md_diexpr)
.Case("!DILocation", MIToken::md_dilocation)
.Case("!noalias.addrspace", MIToken::md_noalias_addrspace)
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h
index 3915222896263..ec4014220c822 100644
--- a/llvm/lib/CodeGen/MIRParser/MILexer.h
+++ b/llvm/lib/CodeGen/MIRParser/MILexer.h
@@ -162,6 +162,7 @@ struct MIToken {
md_noalias,
md_noalias_addrspace,
md_range,
+ md_mem_cache_hint,
md_diexpr,
md_dilocation,
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 2924c0dda2390..194080f928da3 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3794,6 +3794,7 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
: 1;
AAMDNodes AAInfo;
MDNode *Range = nullptr;
+ MDNode *MemCacheHint = nullptr;
while (consumeIfPresent(MIToken::comma)) {
switch (Token.kind()) {
case MIToken::kw_align: {
@@ -3845,16 +3846,23 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
if (parseMDNode(Range))
return true;
break;
+ case MIToken::md_mem_cache_hint:
+ lex();
+ if (parseMDNode(MemCacheHint))
+ return true;
+ break;
// TODO: Report an error on duplicate metadata nodes.
default:
return error("expected 'align' or '!tbaa' or '!alias.scope' or "
- "'!noalias' or '!range' or '!noalias.addrspace'");
+ "'!noalias' or '!range' or '!mem.cache_hint' or "
+ "'!noalias.addrspace'");
}
}
if (expectAndConsume(MIToken::rparen))
return true;
Dest = MF.getMachineMemOperand(Ptr, Flags, MemoryType, Align(BaseAlignment),
- AAInfo, Range, SSID, Order, FailureOrder);
+ AAInfo, Range, MemCacheHint, SSID, Order,
+ FailureOrder);
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index b0ffdb36f21be..b0bed8e3a102c 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -567,7 +567,7 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- SyncScope::ID SSID, AtomicOrdering Ordering,
+ const MDNode *MemCacheHint, SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
assert((!Size.hasValue() ||
Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
@@ -575,17 +575,17 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
"LocationSize::beforeOrAfter()");
return new (Allocator)
MachineMemOperand(PtrInfo, F, Size, BaseAlignment, AAInfo, Ranges, SSID,
- Ordering, FailureOrdering);
+ Ordering, FailureOrdering, MemCacheHint);
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- SyncScope::ID SSID, AtomicOrdering Ordering,
+ const MDNode *MemCacheHint, SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges, SSID,
- Ordering, FailureOrdering);
+ Ordering, FailureOrdering, MemCacheHint);
}
MachineMemOperand *
@@ -599,7 +599,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator)
MachineMemOperand(PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
+ MMO->getMemCacheHint());
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
@@ -607,7 +608,8 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
return new (Allocator)
MachineMemOperand(PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
+ MMO->getMemCacheHint());
}
MachineMemOperand *
@@ -626,7 +628,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
MMO->getAAInfo(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
+ MMO->getMemCacheHint());
}
MachineMemOperand *
@@ -639,7 +642,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(), AAInfo,
MMO->getRanges(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering());
+ MMO->getFailureOrdering(), MMO->getMemCacheHint());
}
MachineMemOperand *
@@ -648,7 +651,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
+ MMO->getMemCacheHint());
}
MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfo(
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 518759952667e..712e2745b4131 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1173,9 +1173,10 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
LLT type, Align a, const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
+ AtomicOrdering FailureOrdering,
+ const MDNode *MemCacheHint)
: PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a),
- AAInfo(AAInfo), Ranges(Ranges) {
+ AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {
assert((PtrInfo.V.isNull() || isa<const PseudoSourceValue *>(PtrInfo.V) ||
isa<PointerType>(cast<const Value *>(PtrInfo.V)->getType())) &&
"invalid pointer value");
@@ -1194,14 +1195,16 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
const AAMDNodes &AAInfo,
const MDNode *Ranges, SyncScope::ID SSID,
AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
+ AtomicOrdering FailureOrdering,
+ const MDNode *MemCacheHint)
: MachineMemOperand(
ptrinfo, F,
!TS.isPrecise() ? LLT()
: TS.isScalable()
? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
: LLT::scalar(8 * TS.getValue().getKnownMinValue()),
- BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
+ BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering,
+ MemCacheHint) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
@@ -1367,6 +1370,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
OS << ", !range ";
getRanges()->printAsOperand(OS, MST);
}
+ if (getMemCacheHint()) {
+ OS << ", !mem.cache_hint ";
+ getMemCacheHint()->printAsOperand(OS, MST);
+ }
// FIXME: Implement addrspace printing/parsing in MIR.
// For now, print this even though parsing it is not available in MIR.
if (unsigned AS = getAddrSpace())
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c542f8e7cc20b..30c6e1b53c921 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -30,6 +30,7 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
@@ -1332,8 +1333,15 @@ SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
// to replace the dead one with the existing one. This can cause
// recursive merging of other unrelated nodes down the line.
Existing->intersectFlagsWith(N->getFlags());
- if (auto *MemNode = dyn_cast<MemSDNode>(Existing))
- MemNode->refineRanges(cast<MemSDNode>(N)->memoperands());
+ if (auto *MemNode = dyn_cast<MemSDNode>(Existing)) {
+ ArrayRef<MachineMemOperand *> NewMMOs =
+ cast<MemSDNode>(N)->memoperands();
+ // Range and cache hint metadata are not part of the DAG CSE key because
+ // we prefer to CSE even when metadata does not match. Merge potentially
+ // differing metadata conservatively.
+ MemNode->refineRanges(NewMMOs);
+ MemNode->refineMemCacheHints(NewMMOs);
+ }
ReplaceAllUsesWith(N, Existing);
// N is now dead. Inform the listeners and delete it.
@@ -9341,7 +9349,7 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
Align SrcAlign, bool isVol, bool AlwaysInline,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
- BatchAAResults *BatchAA) {
+ BatchAAResults *BatchAA, const CallInst *CI) {
// Turn a memcpy of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
if (Src.isUndef())
@@ -9409,6 +9417,10 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
MachineMemOperand::Flags MMOFlags =
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
+ const MDNode *DstMemCacheHint =
+ CI ? getMemCacheHintMetadata(*CI, /*OperandNo=*/0) : nullptr;
+ const MDNode *SrcMemCacheHint =
+ CI ? getMemCacheHintMetadata(*CI, /*OperandNo=*/1) : nullptr;
SmallVector<SDValue, 16> OutLoadChains;
SmallVector<SDValue, 16> OutStoreChains;
SmallVector<SDValue, 32> OutChains;
@@ -9449,7 +9461,7 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
Store = DAG.getStore(
Chain, dl, Value,
DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
- DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags, NewAAInfo);
+ DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags, NewAAInfo, DstMemCacheHint);
OutChains.push_back(Store);
}
}
@@ -9475,13 +9487,13 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
ISD::EXTLOAD, dl, NVT, Chain,
DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),
SrcPtrInfo.getWithOffset(SrcOff), VT,
- commonAlignment(SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
+ commonAlignment(SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo, SrcMemCacheHint);
OutLoadChains.push_back(Value.getValue(1));
Store = DAG.getTruncStore(
Chain, dl, Value,
DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
- DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags, NewAAInfo);
+ DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags, NewAAInfo, DstMemCacheHint);
OutStoreChains.push_back(Store);
}
SrcOff += VTSize;
@@ -9975,7 +9987,7 @@ SDValue SelectionDAG::getMemcpy(
SDValue Result = getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
- SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
+ SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, CI);
if (Result.getNode())
return Result;
}
@@ -9996,7 +10008,7 @@ SDValue SelectionDAG::getMemcpy(
assert(ConstantSize && "AlwaysInline requires a constant size!");
return getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
- SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
+ SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, CI);
}
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
@@ -10573,7 +10585,8 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
MachinePointerInfo PtrInfo, EVT MemVT,
Align Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo, const MDNode *Ranges) {
+ const AAMDNodes &AAInfo, const MDNode *Ranges,
+ const MDNode *MemCacheHint) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
@@ -10586,8 +10599,8 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
- Alignment, AAInfo, Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges, MemCacheHint);
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
}
@@ -10635,6 +10648,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
if (auto *E = cast_or_null<LoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
E->refineAlignment(MMO);
E->refineRanges(MMO);
+ E->refineMemCacheHints(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
@@ -10652,10 +10666,12 @@ SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
SDValue Ptr, MachinePointerInfo PtrInfo,
MaybeAlign Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo, const MDNode *Ranges) {
+ const AAMDNodes &AAInfo, const MDNode *Ranges,
+ const MDNode *MemCacheHint) {
SDValue Undef = getUNDEF(Ptr.getValueType());
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
- PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
+ PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
+ MemCacheHint);
}
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
@@ -10670,10 +10686,11 @@ SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
MachinePointerInfo PtrInfo, EVT MemVT,
MaybeAlign Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo) {
+ const AAMDNodes &AAInfo,
+ const MDNode *MemCacheHint) {
SDValue Undef = getUNDEF(Ptr.getValueType());
return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
- MemVT, Alignment, MMOFlags, AAInfo);
+ MemVT, Alignment, MMOFlags, AAInfo, nullptr, MemCacheHint);
}
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
@@ -10702,7 +10719,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, MachinePointerInfo PtrInfo,
Align Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo) {
+ const AAMDNodes &AAInfo,
+ const MDNode *MemCacheHint) {
assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
MMOFlags |= MachineMemOperand::MOStore;
@@ -10713,8 +10731,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
TypeSize Size = Val.getValueType().getStoreSize();
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MMOFlags, Size, Alignment, AAInfo, nullptr, MemCacheHint);
return getStore(Chain, dl, Val, Ptr, MMO);
}
@@ -10761,6 +10779,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
cast<StoreSDNode>(E)->refineAlignment(MMO);
+ cast<StoreSDNode>(E)->refineMemCacheHints(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
@@ -10778,7 +10797,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, MachinePointerInfo PtrInfo,
EVT SVT, Align Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo) {
+ const AAMDNodes &AAInfo,
+ const MDNode *MemCacheHint) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
@@ -10790,7 +10810,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo, nullptr,
+ MemCacheHint);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8aa184423c60c..7f41890d58584 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4732,6 +4732,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
Align Alignment = I.getAlign();
AAMDNodes AAInfo = I.getAAMetadata();
const MDNode *Ranges = getRangeMetadata(I);
+ const MDNode *MemCacheHint = getMemCacheHintMetadata(I);
bool isVolatile = I.isVolatile();
MachineMemOperand::Flags MMOFlags =
TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);
@@ -4789,7 +4790,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
- MMOFlags, AAInfo, Ranges);
+ MMOFlags, AAInfo, Ranges, MemCacheHint);
Chains[ChainI] = L.getValue(1);
if (MemVTs[i] != ValueVTs[i])
@@ -4920,6 +4921,8 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
SDLoc dl = getCurSDLoc();
Align Alignment = I.getAlign();
AAMDNodes AAInfo = I.getAAMetadata();
+ const MDNode *MemCacheHint =
+ getMemCacheHintMetadata(I, I.getPointerOperandIndex());
auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
@@ -4943,8 +4946,8 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
if (MemVTs[i] != ValueVTs[i])
Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
- SDValue St =
- DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
+ SDValue St = DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
+ AAInfo, MemCacheHint);
Chains[ChainI] = St;
}
@@ -5224,7 +5227,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
MachineMemOperand *MMO =
MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
MemVT.getStoreSize(), I.getAlign(), AAMDNodes(),
- nullptr, SSID, SuccessOrdering, FailureOrdering);
+ nullptr, nullptr, SSID, SuccessOrdering, FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
dl, MemVT, VTs, InChain,
@@ -5295,7 +5298,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
+ I.getAlign(), AAMDNodes(), nullptr, nullptr, SSID, Ordering);
SDValue L =
DAG.getAtomic(NT, dl, MemVT, InChain,
@@ -5342,7 +5345,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const MDNode *Ranges = getRangeMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), Ranges, SSID, Order);
+ I.getAlign(), AAMDNodes(), Ranges, nullptr, SSID, Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -5379,7 +5382,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
+ I.getAlign(), AAMDNodes(), nullptr, nullptr, SSID, Ordering);
SDValue Val = getValue(I.getValueOperand());
if (Val.getValueType() != MemVT)
@@ -5570,7 +5573,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));
MachineMemOperand *MMO = MF.getMachineMemOperand(
MPI, Info.flags, Size, Alignment, I.getAAMetadata(),
- /*Ranges=*/nullptr, Info.ssid, Info.order, Info.failureOrder);
+ /*Ranges=*/nullptr, /*MemCacheHint=*/nullptr, Info.ssid, Info.order,
+ Info.failureOrder);
MMOs.push_back(MMO);
}
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 6b5e5ef1a3c81..a6e82b3e1b377 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3169,7 +3169,7 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
MachineFunction &MF = DAG.getMachineFunction();
WideMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
- MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
+ MMO->getAAInfo(), MMO->getRanges(), nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index beed605c56bd4..fe1becb121bb1 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -386,6 +386,82 @@ void NVPTXInstPrinter::printAtomicCode(const MCInst *MI, int OpNum,
llvm_unreachable(formatv("Unknown Modifier: {}", Modifier).str().c_str());
}
+void NVPTXInstPrinter::printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &,
+ raw_ostream &O,
+ StringRef Modifier) {
+ const MCOperand &MO = MI->getOperand(OpNum);
+ unsigned Hint = MO.getImm();
+
+ // If no hint is set, print nothing.
+ if (Hint == 0)
+ return;
+
+ // Check if L2::cache_hint mode is active.
+ bool IsCacheHintMode = NVPTX::isL2CacheHintMode(Hint);
+
+ if (Modifier == "l1") {
+ auto L1 = NVPTX::decodeL1Eviction(Hint);
+ switch (L1) {
+ case NVPTX::L1Eviction::Normal:
+ return;
+ case NVPTX::L1Eviction::Unchanged:
+ O << ".L1::evict_unchanged";
+ return;
+ case NVPTX::L1Eviction::First:
+ O << ".L1::evict_first";
+ return;
+ case NVPTX::L1Eviction::Last:
+ O << ".L1::evict_last";
+ return;
+ case NVPTX::L1Eviction::NoAllocate:
+ O << ".L1::no_allocate";
+ return;
+ }
+ } else if (Modifier == "l2") {
+ auto L2 = NVPTX::decodeL2Eviction(Hint);
+ switch (L2) {
+ case NVPTX::L2Eviction::Normal:
+ break;
+ case NVPTX::L2Eviction::First:
+ O << ".L2::evict_first";
+ break;
+ case NVPTX::L2Eviction::Last:
+ O << ".L2::evict_last";
+ break;
+ }
+ if (IsCacheHintMode)
+ O << ".L2::cache_hint";
+ return;
+ } else if (Modifier == "prefetch") {
+ auto Prefetch = NVPTX::decodeL2Prefetch(Hint);
+ switch (Prefetch) {
+ case NVPTX::L2Prefetch::None:
+ return;
+ case NVPTX::L2Prefetch::Bytes64:
+ O << ".L2::64B";
+ return;
+ case NVPTX::L2Prefetch::Bytes128:
+ O << ".L2::128B";
+ return;
+ case NVPTX::L2Prefetch::Bytes256:
+ O << ".L2::256B";
+ return;
+ }
+ }
+}
+
+void NVPTXInstPrinter::printCachePolicy(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &,
+ raw_ostream &O) {
+ const MCOperand &MO = MI->getOperand(OpNum);
+ // If the operand is a register and valid, print ", $reg"
+ if (MO.isReg() && MO.getReg().isValid()) {
+ O << ", ";
+ printRegName(O, MO.getReg());
+ }
+}
+
void NVPTXInstPrinter::printMmaCode(const MCInst *MI, int OpNum,
const MCSubtargetInfo &, raw_ostream &O,
StringRef Modifier) {
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index 4d9115a1e1b9d..f977927b4d5ca 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -44,6 +44,11 @@ class NVPTXInstPrinter : public MCInstPrinter {
raw_ostream &O, StringRef Modifier = {});
void printAtomicCode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
+ void printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &STI, raw_ostream &O,
+ StringRef Modifier = {});
+ void printCachePolicy(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &STI, raw_ostream &O);
void printMmaCode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
void printMemOperand(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 57ed9bb43aeea..d047a303fb96e 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -14,6 +14,7 @@
#ifndef LLVM_LIB_TARGET_NVPTX_NVPTX_H
#define LLVM_LIB_TARGET_NVPTX_NVPTX_H
+#include "llvm/ADT/Bitfields.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
@@ -22,10 +23,13 @@
#include "llvm/Target/TargetMachine.h"
namespace llvm {
+class Function;
class FunctionPass;
+class MachineMemOperand;
class MachineFunctionPass;
class NVPTXTargetMachine;
class PassRegistry;
+class Value;
namespace NVPTXCC {
enum CondCodes {
@@ -217,6 +221,73 @@ enum AddressSpace : AddressSpaceUnderlyingType {
DeviceParam
};
+// Eviction and prefetch hint enums for !mem.cache_hint metadata. These
+// correspond to PTX L1::evict_*, L2::evict_*, and L2::*B qualifiers.
+
+// L1 Eviction Policy - maps to PTX L1::evict_* qualifiers
+enum class L1Eviction : uint8_t {
+ Normal = 0, // Default behavior (no qualifier)
+ Unchanged = 1, // L1::evict_unchanged
+ First = 2, // L1::evict_first
+ Last = 3, // L1::evict_last
+ NoAllocate = 4, // L1::no_allocate
+};
+
+// L2 Eviction Policy - maps to PTX L2::evict_* qualifiers
+enum class L2Eviction : uint8_t {
+ Normal = 0, // Default behavior (no qualifier)
+ First = 1, // L2::evict_first
+ Last = 2, // L2::evict_last
+};
+
+// L2 Prefetch Size - maps to PTX L2::*B qualifiers
+enum class L2Prefetch : uint8_t {
+ None = 0, // No prefetch hint
+ Bytes64 = 1, // L2::64B
+ Bytes128 = 2, // L2::128B
+ Bytes256 = 3, // L2::256B
+};
+
+// Bitfield layout for encoded eviction/prefetch hints (stored in unsigned):
+// Bits 0-2: L1 Eviction (3 bits, 5 values)
+// Bits 3-4: L2 Eviction (2 bits, 3 values)
+// Bits 5-6: L2 Prefetch (2 bits, 4 values)
+// Bit 7: L2::cache_hint mode flag (set when using CachePolicy)
+// Bits 8-31: Reserved
+//
+// Using llvm::Bitfield for type-safe access with compile-time validation.
+using L1EvictionBits =
+ Bitfield::Element<L1Eviction, 0, 3, L1Eviction::NoAllocate>;
+using L2EvictionBits = Bitfield::Element<L2Eviction, 3, 2, L2Eviction::Last>;
+using L2PrefetchBits =
+ Bitfield::Element<L2Prefetch, 5, 2, L2Prefetch::Bytes256>;
+using L2CacheHintBit = Bitfield::Element<bool, 7, 1>;
+
+inline unsigned encodeEvictionAndPrefetchHint(L1Eviction L1, L2Eviction L2,
+ L2Prefetch P) {
+ unsigned Hint = 0;
+ Bitfield::set<L1EvictionBits>(Hint, L1);
+ Bitfield::set<L2EvictionBits>(Hint, L2);
+ Bitfield::set<L2PrefetchBits>(Hint, P);
+ return Hint;
+}
+
+inline L1Eviction decodeL1Eviction(unsigned Hint) {
+ return Bitfield::get<L1EvictionBits>(Hint);
+}
+
+inline L2Eviction decodeL2Eviction(unsigned Hint) {
+ return Bitfield::get<L2EvictionBits>(Hint);
+}
+
+inline L2Prefetch decodeL2Prefetch(unsigned Hint) {
+ return Bitfield::get<L2PrefetchBits>(Hint);
+}
+
+inline bool isL2CacheHintMode(unsigned Hint) {
+ return Bitfield::get<L2CacheHintBit>(Hint);
+}
+
namespace PTXLdStInstCode {
enum FromType { Unsigned = 0, Signed, Float, Untyped };
} // namespace PTXLdStInstCode
@@ -293,7 +364,8 @@ void initializeNVPTXDAGToDAGISelLegacyPass(PassRegistry &);
#define GET_REGINFO_ENUM
#include "NVPTXGenRegisterInfo.inc"
-// Defines symbolic names for the NVPTX instructions.
+// Defines symbolic names for NVPTX instructions, MC helper declarations,
+// and named operand helpers generated from UseNamedOperandTable=1.
#define GET_INSTRINFO_ENUM
#define GET_INSTRINFO_MC_HELPER_DECLS
#define GET_INSTRINFO_OPERAND_ENUM
diff --git a/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp b/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
index 2b59286295d2b..eeeee076b3874 100644
--- a/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
@@ -98,11 +98,18 @@ static bool eliminateMove(MachineInstr &Mov, const MachineRegisterInfo &MRI,
constexpr unsigned LDInstBasePtrOpIdx = 6;
constexpr unsigned LDInstAddrSpaceOpIdx = 2;
+ constexpr unsigned LDInstEvictionAndPrefetchHintOpIdx = 8;
+ constexpr unsigned LDInstCachePolicyOpIdx = 9;
for (auto *LI : LoadInsts) {
(LI->uses().begin() + LDInstBasePtrOpIdx)
->ChangeToES(ParamSymbol->getSymbolName());
(LI->uses().begin() + LDInstAddrSpaceOpIdx)
->ChangeToImmediate(NVPTX::AddressSpace::DeviceParam);
+ // PTX cache modifiers are not allowed on ld.param.
+ (LI->uses().begin() + LDInstEvictionAndPrefetchHintOpIdx)
+ ->ChangeToImmediate(0);
+ (LI->uses().begin() + LDInstCachePolicyOpIdx)
+ ->ChangeToRegister(NVPTX::NoRegister, false);
}
return true;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ede1deb5400b0..d0ad28d1cd393 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -14,14 +14,19 @@
#include "NVPTX.h"
#include "NVPTXUtilities.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/CommandLine.h"
@@ -1108,6 +1113,145 @@ bool NVPTXDAGToDAGISel::SelectADDR(SDValue Addr, SDValue &Base,
return true;
}
+static void emitInvalidMemCacheHint(LLVMContext &Ctx, const Twine &Msg) {
+ Ctx.emitError(Twine("invalid NVPTX !mem.cache_hint metadata: ") + Msg);
+}
+
+static std::optional<NVPTX::L1Eviction> parseL1Eviction(StringRef Str) {
+ return StringSwitch<std::optional<NVPTX::L1Eviction>>(Str)
+ .Case("normal", NVPTX::L1Eviction::Normal)
+ .Case("unchanged", NVPTX::L1Eviction::Unchanged)
+ .Case("first", NVPTX::L1Eviction::First)
+ .Case("last", NVPTX::L1Eviction::Last)
+ .Case("no_allocate", NVPTX::L1Eviction::NoAllocate)
+ .Default(std::nullopt);
+}
+
+static std::optional<NVPTX::L2Eviction> parseL2Eviction(StringRef Str) {
+ return StringSwitch<std::optional<NVPTX::L2Eviction>>(Str)
+ .Case("normal", NVPTX::L2Eviction::Normal)
+ .Case("first", NVPTX::L2Eviction::First)
+ .Case("last", NVPTX::L2Eviction::Last)
+ .Default(std::nullopt);
+}
+
+static std::optional<NVPTX::L2Prefetch> parseL2Prefetch(StringRef Str) {
+ return StringSwitch<std::optional<NVPTX::L2Prefetch>>(Str)
+ .Case("64B", NVPTX::L2Prefetch::Bytes64)
+ .Case("128B", NVPTX::L2Prefetch::Bytes128)
+ .Case("256B", NVPTX::L2Prefetch::Bytes256)
+ .Default(std::nullopt);
+}
+
+template <typename T, typename ParseFn>
+static void parseMemCacheHintStringValue(LLVMContext &Ctx, StringRef Key,
+ const Metadata *Value, T &Result,
+ ParseFn Parse) {
+ const auto *Val = dyn_cast<MDString>(Value);
+ if (!Val) {
+ emitInvalidMemCacheHint(Ctx,
+ Twine("'") + Key + "' expects a string value");
+ return;
+ }
+
+ StringRef ValStr = Val->getString();
+ if (auto Parsed = Parse(ValStr))
+ Result = *Parsed;
+ else
+ emitInvalidMemCacheHint(Ctx, Twine("unknown value '") + ValStr +
+ "' for '" + Key + "'");
+}
+
+static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
+ NVPTX::L2Prefetch Prefetch) {
+ switch (Prefetch) {
+ case NVPTX::L2Prefetch::None:
+ return true;
+ case NVPTX::L2Prefetch::Bytes64:
+ return Subtarget.hasL2Prefetch64B();
+ case NVPTX::L2Prefetch::Bytes128:
+ return Subtarget.hasL2Prefetch128B();
+ case NVPTX::L2Prefetch::Bytes256:
+ return Subtarget.hasL2Prefetch256B();
+ }
+ llvm_unreachable("Unexpected L2 prefetch hint");
+}
+
+std::pair<unsigned, SDValue>
+NVPTXDAGToDAGISel::getMemCacheHintOperands(const MemSDNode *N,
+ unsigned CodeAddrSpace,
+ const SDLoc &DL) {
+ LLVMContext &Ctx = *CurDAG->getContext();
+ const MDNode *Node = N->getMemCacheHint();
+ SDValue PolicyReg = CurDAG->getRegister(NVPTX::NoRegister, MVT::i64);
+ if (!Node)
+ return {0, PolicyReg};
+ if (Node->getNumOperands() == 0) {
+ emitInvalidMemCacheHint(Ctx, "empty hint node");
+ return {0, PolicyReg};
+ }
+
+ NVPTX::L1Eviction L1 = NVPTX::L1Eviction::Normal;
+ NVPTX::L2Eviction L2 = NVPTX::L2Eviction::Normal;
+ NVPTX::L2Prefetch Prefetch = NVPTX::L2Prefetch::None;
+ std::optional<uint64_t> CachePolicy;
+
+ for (unsigned I = 0; I + 1 < Node->getNumOperands(); I += 2) {
+ const auto *Key = cast<MDString>(Node->getOperand(I));
+ StringRef KeyStr = Key->getString();
+ const Metadata *Value = Node->getOperand(I + 1).get();
+
+ if (KeyStr == "nvvm.l1_eviction") {
+ if (Subtarget->hasL1EvictionHint())
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, L1,
+ parseL1Eviction);
+ continue;
+ }
+
+ if (KeyStr == "nvvm.l2_eviction") {
+ if (Subtarget->hasL2EvictionHint())
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, L2,
+ parseL2Eviction);
+ continue;
+ }
+
+ if (KeyStr == "nvvm.l2_prefetch_size") {
+ NVPTX::L2Prefetch ParsedPrefetch = NVPTX::L2Prefetch::None;
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, ParsedPrefetch,
+ parseL2Prefetch);
+ if (isL2PrefetchSupported(*Subtarget, ParsedPrefetch))
+ Prefetch = ParsedPrefetch;
+ continue;
+ }
+
+ if (KeyStr == "nvvm.l2_cache_hint") {
+ if (CodeAddrSpace == NVPTX::AddressSpace::Global &&
+ Subtarget->hasL2CacheHint()) {
+ if (auto *ValCI = mdconst::dyn_extract<ConstantInt>(Value))
+ CachePolicy = ValCI->getZExtValue();
+ else
+ emitInvalidMemCacheHint(
+ Ctx, "'nvvm.l2_cache_hint' expects an integer value");
+ }
+ continue;
+ }
+
+ emitInvalidMemCacheHint(Ctx, Twine("unknown key '") + KeyStr + "'");
+ }
+
+ unsigned EvictionAndPrefetchHint =
+ NVPTX::encodeEvictionAndPrefetchHint(L1, L2, Prefetch);
+ if (CachePolicy) {
+ SDValue PolicyConst = CurDAG->getTargetConstant(*CachePolicy, DL, MVT::i64);
+ PolicyReg = SDValue(
+ CurDAG->getMachineNode(NVPTX::MOV_B64_i, DL, MVT::i64, PolicyConst),
+ 0);
+ Bitfield::set<NVPTX::L2CacheHintBit>(EvictionAndPrefetchHint, true);
+ }
+
+ return {EvictionAndPrefetchHint, PolicyReg};
+}
+
bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
MemSDNode *LD = cast<MemSDNode>(N);
assert(LD->readMem() && "Expected load");
@@ -1150,8 +1294,11 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
assert(isPowerOf2_32(FromTypeWidth) && FromTypeWidth >= 8 &&
FromTypeWidth <= 128 && "Invalid width for load");
- // Create the machine instruction DAG
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
+ const auto [EvictionAndPrefetchHint, PolicyReg] =
+ getMemCacheHintOperands(LD, CodeAddrSpace, DL);
+
+ // Create the machine instruction DAG
SDValue Ops[] = {getI32Imm(Ordering, DL),
getI32Imm(Scope, DL),
getI32Imm(CodeAddrSpace, DL),
@@ -1160,6 +1307,8 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
getI32Imm(UsedBytesMask, DL),
Base,
Offset,
+ getI32Imm(EvictionAndPrefetchHint, DL),
+ PolicyReg,
Chain};
const MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
@@ -1223,6 +1372,8 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
assert(!(EltVT.isVector() && ExtensionType != ISD::NON_EXTLOAD));
+ const auto [EvictionAndPrefetchHint, PolicyReg] =
+ getMemCacheHintOperands(LD, CodeAddrSpace, DL);
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
SDValue Ops[] = {getI32Imm(Ordering, DL),
getI32Imm(Scope, DL),
@@ -1232,6 +1383,8 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
getI32Imm(UsedBytesMask, DL),
Base,
Offset,
+ getI32Imm(EvictionAndPrefetchHint, DL),
+ PolicyReg,
Chain};
std::optional<unsigned> Opcode;
@@ -1285,11 +1438,15 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
ExtensionType != ISD::NON_EXTLOAD));
const auto [Base, Offset] = selectADDR(LD->getOperand(1), CurDAG);
+ const auto [EvictionAndPrefetchHint, PolicyReg] =
+ getMemCacheHintOperands(LD, NVPTX::AddressSpace::Global, DL);
SDValue Ops[] = {getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL),
getI32Imm(UsedBytesMask, DL),
Base,
Offset,
+ getI32Imm(EvictionAndPrefetchHint, DL),
+ PolicyReg,
LD->getChain()};
const MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
@@ -1408,6 +1565,11 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
"Invalid width for store");
const auto [Base, Offset] = selectADDR(ST->getBasePtr(), CurDAG);
+
+ // Extract eviction/prefetch hint and cache policy register.
+ const auto [EvictionAndPrefetchHint, PolicyReg] =
+ getMemCacheHintOperands(ST, CodeAddrSpace, DL);
+
SDValue Ops[] = {selectPossiblyImm(Value),
getI32Imm(Ordering, DL),
getI32Imm(Scope, DL),
@@ -1415,6 +1577,8 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
getI32Imm(ToTypeWidth, DL),
Base,
Offset,
+ getI32Imm(EvictionAndPrefetchHint, DL),
+ PolicyReg,
Chain};
const std::optional<unsigned> Opcode =
@@ -1460,10 +1624,15 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
assert(isPowerOf2_32(ToTypeWidth) && ToTypeWidth >= 8 && ToTypeWidth <= 128 &&
TotalWidth <= 256 && "Invalid width for store");
+ // Extract eviction/prefetch hint and cache policy register.
+ const auto [EvictionAndPrefetchHint, PolicyReg] =
+ getMemCacheHintOperands(ST, CodeAddrSpace, DL);
+
const auto [Base, Offset] = selectADDR(Addr, CurDAG);
Ops.append({getI32Imm(Ordering, DL), getI32Imm(Scope, DL),
getI32Imm(CodeAddrSpace, DL), getI32Imm(ToTypeWidth, DL), Base,
- Offset, Chain});
+ Offset, getI32Imm(EvictionAndPrefetchHint, DL), PolicyReg,
+ Chain});
const MVT::SimpleValueType EltVT =
ST->getOperand(1).getSimpleValueType().SimpleTy;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index fcb5700dcb6d4..931f7e4d86df2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -105,6 +105,14 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
SDValue getPTXCmpMode(const CondCodeSDNode &CondCode);
SDValue selectPossiblyImm(SDValue V);
+ // Returns the encoded eviction/prefetch hint and cache policy register for a
+ // memory operation. Hints unsupported by the subtarget or address space are
+ // dropped. If L2::cache_hint is active, returns the hint with
+ // L2CacheHintBit set and a register containing the 64-bit cache policy
+ // value. Otherwise returns NOREG for the policy operand.
+ std::pair<unsigned, SDValue> getMemCacheHintOperands(
+ const MemSDNode *N, unsigned CodeAddrSpace, const SDLoc &DL);
+
// Returns the Memory Order and Scope that the PTX memory instruction should
// use, and inserts appropriate fence instruction before the memory
// instruction, if needed to implement the instructions memory order. Required
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 65dd146926f8b..fc044e6b52e8c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -1699,6 +1699,10 @@ def AtomicCode : Operand<i32> {
let PrintMethod = "printAtomicCode";
}
+def EvictionAndPrefetchHint : Operand<i32> {
+ let PrintMethod = "printEvictionAndPrefetchHint";
+}
+
def MmaCode : Operand<i32> {
let PrintMethod = "printMmaCode";
}
@@ -1924,15 +1928,22 @@ def Callseq_End :
//
// Load / Store Handling
//
+// CachePolicy operand for L2::cache_hint support.
+// When present, prints the cache policy register.
+def CachePolicy : Operand<i64> { let PrintMethod = "printCachePolicy"; }
+
class LD<NVPTXRegClass regclass>
- : NVPTXInst<
- (outs regclass:$dst),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
- AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$addr),
- "${usedBytes}"
- "ld${sem:sem}${scope:scope}${addsp:addsp}.${Sign:sign}$fromWidth "
- "\t$dst, [$addr];">;
+ : NVPTXInst<
+ (outs regclass:$dst),
+ (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
+ ADDR:$addr, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "${usedBytes}"
+ "ld${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.${Sign:sign}$"
+ "fromWidth "
+ "\t$dst, [$addr]${policy};">;
let mayLoad=1, hasSideEffects=0 in {
def LD_i16 : LD<B16>;
@@ -1941,13 +1952,15 @@ let mayLoad=1, hasSideEffects=0 in {
}
class ST<DAGOperand O>
- : NVPTXInst<
- (outs),
- (ins O:$src,
- AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$toWidth,
- ADDR:$addr),
- "st${sem:sem}${scope:scope}${addsp:addsp}.b$toWidth"
- " \t[$addr], $src;">;
+ : NVPTXInst<
+ (outs),
+ (ins O:$src, AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ i32imm:$toWidth, ADDR:$addr,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "st${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.b$toWidth"
+ " \t[$addr], $src${policy};">;
let mayStore=1, hasSideEffects=0 in {
def ST_i16 : ST<RI16>;
@@ -1959,33 +1972,48 @@ let mayStore=1, hasSideEffects=0 in {
// elementization happens at the machine instruction level, so the following
// instructions never appear in the DAG.
multiclass LD_VEC<NVPTXRegClass regclass, bit support_v8 = false> {
- def _v2 : NVPTXInst<
- (outs regclass:$dst1, regclass:$dst2),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
- AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$addr),
- "${usedBytes}"
- "ld${sem:sem}${scope:scope}${addsp:addsp}.v2.${Sign:sign}$fromWidth "
- "\t{{$dst1, $dst2}}, [$addr];">;
- def _v4 : NVPTXInst<
- (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
- AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$addr),
- "${usedBytes}"
- "ld${sem:sem}${scope:scope}${addsp:addsp}.v4.${Sign:sign}$fromWidth "
- "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];">;
+ def _v2
+ : NVPTXInst<
+ (outs regclass:$dst1, regclass:$dst2),
+ (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
+ ADDR:$addr, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "${usedBytes}"
+ "ld${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v2.${Sign:sign}"
+ "$fromWidth "
+ "\t{{$dst1, $dst2}}, [$addr]${policy};">;
+ def _v4
+ : NVPTXInst<
+ (outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
+ regclass:$dst4),
+ (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
+ ADDR:$addr, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "${usedBytes}"
+ "ld${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v4.${Sign:sign}"
+ "$fromWidth "
+ "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr]${policy};">;
if support_v8 then
- def _v8 : NVPTXInst<
- (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4,
- regclass:$dst5, regclass:$dst6, regclass:$dst7, regclass:$dst8),
- (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
- AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$addr),
- "${usedBytes}"
- "ld${sem:sem}${scope:scope}${addsp:addsp}.v8.${Sign:sign}$fromWidth "
- "\t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, "
- "[$addr];">;
+ def _v8
+ : NVPTXInst<
+ (outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
+ regclass:$dst4, regclass:$dst5, regclass:$dst6,
+ regclass:$dst7, regclass:$dst8),
+ (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
+ AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
+ ADDR:$addr,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "${usedBytes}"
+ "ld${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v8.${Sign:sign}"
+ "$fromWidth "
+ "\t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, "
+ "[$addr]${policy};">;
}
let mayLoad=1, hasSideEffects=0 in {
defm LDV_i16 : LD_VEC<B16>;
@@ -1994,30 +2022,43 @@ let mayLoad=1, hasSideEffects=0 in {
}
multiclass ST_VEC<DAGOperand O, bit support_v8 = false> {
- def _v2 : NVPTXInst<
- (outs),
- (ins O:$src1, O:$src2,
- AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
- ADDR:$addr),
- "st${sem:sem}${scope:scope}${addsp:addsp}.v2.b$fromWidth "
- "\t[$addr], {{$src1, $src2}};">;
- def _v4 : NVPTXInst<
- (outs),
- (ins RegOrSink:$src1, RegOrSink:$src2, RegOrSink:$src3, RegOrSink:$src4,
- AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
- ADDR:$addr),
- "st${sem:sem}${scope:scope}${addsp:addsp}.v4.b$fromWidth "
- "\t[$addr], {{$src1, $src2, $src3, $src4}};">;
+ def _v2
+ : NVPTXInst<
+ (outs),
+ (ins O:$src1, O:$src2, AtomicCode:$sem, AtomicCode:$scope,
+ AtomicCode:$addsp, i32imm:$fromWidth, ADDR:$addr,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "st${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v2.b$fromWidth "
+ "\t[$addr], {{$src1, $src2}}${policy};">;
+ def _v4
+ : NVPTXInst<
+ (outs),
+ (ins RegOrSink:$src1, RegOrSink:$src2, RegOrSink:$src3,
+ RegOrSink:$src4, AtomicCode:$sem, AtomicCode:$scope,
+ AtomicCode:$addsp, i32imm:$fromWidth, ADDR:$addr,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "st${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v4.b$fromWidth "
+ "\t[$addr], {{$src1, $src2, $src3, $src4}}${policy};">;
if support_v8 then
- def _v8 : NVPTXInst<
- (outs),
- (ins RegOrSink:$src1, RegOrSink:$src2, RegOrSink:$src3, RegOrSink:$src4,
- RegOrSink:$src5, RegOrSink:$src6, RegOrSink:$src7, RegOrSink:$src8,
- AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
- ADDR:$addr),
- "st${sem:sem}${scope:scope}${addsp:addsp}.v8.b$fromWidth "
- "\t[$addr], "
- "{{$src1, $src2, $src3, $src4, $src5, $src6, $src7, $src8}};">;
+ def _v8
+ : NVPTXInst<
+ (outs),
+ (ins RegOrSink:$src1, RegOrSink:$src2, RegOrSink:$src3,
+ RegOrSink:$src4, RegOrSink:$src5, RegOrSink:$src6,
+ RegOrSink:$src7, RegOrSink:$src8, AtomicCode:$sem,
+ AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
+ ADDR:$addr,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
+ "st${sem:sem}${scope:scope}${addsp:addsp}${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v8.b$fromWidth "
+ "\t[$addr], "
+ "{{$src1, $src2, $src3, $src4, $src5, $src6, $src7, "
+ "$src8}}${policy};">;
}
let mayStore=1, hasSideEffects=0 in {
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 75e7e4cbbf0eb..ebd28580cf93f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2704,9 +2704,13 @@ def LDU_GLOBAL_v4i32 : VLDU_G_ELE_V4<B32>;
class LDG_G<NVPTXRegClass regclass>
: NVPTXInst<(outs regclass:$result),
(ins AtomicCode:$Sign, i32imm:$fromWidth,
- UsedBytesMask:$usedBytes, ADDR:$src),
+ UsedBytesMask:$usedBytes, ADDR:$src,
+ EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
"${usedBytes}"
- "ld.global.nc.${Sign:sign}$fromWidth \t$result, [$src];">;
+ "ld.global.nc${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.${"
+ "Sign:sign}$fromWidth $result, [$src]${policy};">;
def LD_GLOBAL_NC_i16 : LDG_G<B16>;
def LD_GLOBAL_NC_i32 : LDG_G<B32>;
@@ -2718,25 +2722,35 @@ def LD_GLOBAL_NC_i64 : LDG_G<B64>;
class VLDG_G_ELE_V2<NVPTXRegClass regclass> :
NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$src),
+ ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
"${usedBytes}"
- "ld.global.nc.v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src];">;
-
+ "ld.global.nc${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v2.${"
+ "Sign:sign}$fromWidth {{$dst1, $dst2}}, [$src]${policy};">;
class VLDG_G_ELE_V4<NVPTXRegClass regclass> :
- NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
+ NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
+ regclass:$dst4),
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$src),
+ ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
"${usedBytes}"
- "ld.global.nc.v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];">;
+ "ld.global.nc${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v4.${"
+ "Sign:sign}$fromWidth {{$dst1, $dst2, $dst3, $dst4}}, [$src]${policy};">;
class VLDG_G_ELE_V8<NVPTXRegClass regclass> :
- NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4,
- regclass:$dst5, regclass:$dst6, regclass:$dst7, regclass:$dst8),
+ NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
+ regclass:$dst4, regclass:$dst5, regclass:$dst6,
+ regclass:$dst7, regclass:$dst8),
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
- ADDR:$src),
+ ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
+ CachePolicy:$policy),
"${usedBytes}"
- "ld.global.nc.v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src];">;
+ "ld.global.nc${evictionAndPrefetchHint:l1}${"
+ "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v8.${"
+ "Sign:sign}$fromWidth {{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src]${policy};">;
// FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.
def LD_GLOBAL_NC_v2i16 : VLDG_G_ELE_V2<B16>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 1df5d326f63a6..add11e4eea961 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -125,6 +125,23 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
bool hasDotInstructions() const {
return SmVersion >= 61 && PTXVersion >= 50;
}
+ // Cache hint SM version requirements
+ //
+ // | Cache Hint | SM Requirement | PTX Requirement |
+ // |-----------------|----------------|-----------------|
+ // | L1::evict_* | SM 70+ | - |
+ // | L2::evict_* | SM 70+ | - |
+ // | L2::64B | SM 75+ | - |
+ // | L2::128B | SM 75+ | - |
+ // | L2::256B | SM 80+ | - |
+ // | L2::cache_hint | SM 80+ | PTX 7.4+ |
+ //
+ bool hasL1EvictionHint() const { return SmVersion >= 70; }
+ bool hasL2EvictionHint() const { return SmVersion >= 70; }
+ bool hasL2Prefetch64B() const { return SmVersion >= 75; }
+ bool hasL2Prefetch128B() const { return SmVersion >= 75; }
+ bool hasL2Prefetch256B() const { return SmVersion >= 80; }
+ bool hasL2CacheHint() const { return SmVersion >= 80 && PTXVersion >= 74; }
// Checks following instructions support:
// - tcgen05.ld/st
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 76cc06f2b4ed9..39d6c5bf1bfe7 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -15956,7 +15956,7 @@ SDValue convertTwoLoadsAndCmpToVCMPEQUB(SelectionDAG &DAG, SDNode *N,
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *NewMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), MMO->getSize(), MMO->getAlign(),
- MMO->getAAInfo(), nullptr, MMO->getSyncScopeID(),
+ MMO->getAAInfo(), nullptr, nullptr, MMO->getSyncScopeID(),
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
SDValue NewLoad = DAG.getLoad(MVT::v16i8, DL, LoadNode->getChain(),
LoadNode->getBasePtr(), NewMMO);
diff --git a/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir b/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir
index b057b51c99a00..6e6821b16b045 100644
--- a/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir
+++ b/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir
@@ -40,9 +40,9 @@ registers:
- { id: 7, class: b32 }
body: |
bb.0.entry:
- %0 = LD_i32 0, 0, 4, 2, 32, -1, &test_param_0, 0
+ %0 = LD_i32 0, 0, 4, 2, 32, -1, 0, &test_param_0, 0, $noreg
%1 = CVT_f64_f32 %0, 0
- %2 = LD_i32 0, 0, 4, 0, 32, -1, &test_param_1, 0
+ %2 = LD_i32 0, 0, 4, 0, 32, -1, 0, &test_param_1, 0, $noreg
; CHECK: %3:b64 = FADD_rnf64ri %1, double 3.250000e+00
%3 = FADD_rnf64ri %1, double 3.250000e+00
%4 = CVT_f32_f64 %3, 5
@@ -50,7 +50,7 @@ body: |
; CHECK: %6:b32 = FADD_rnf32ri %5, float 6.250000e+00
%6 = FADD_rnf32ri %5, float 6.250000e+00, 0
%7 = FMUL_rnf32rr %6, %4, 0
- ST_i32 %7, 0, 0, 101, 32, &func_retval0, 0 :: (store (s32), addrspace 101)
+ ST_i32 %7, 0, 0, 101, 32, 0, &func_retval0, 0, $noreg :: (store (s32), addrspace 101)
Return
...
---
@@ -66,9 +66,9 @@ registers:
- { id: 7, class: b32 }
body: |
bb.0.entry:
- %0 = LD_i32 0, 0, 4, 2, 32, -1, &test2_param_0, 0
+ %0 = LD_i32 0, 0, 4, 2, 32, -1, 0, &test2_param_0, 0, $noreg
%1 = CVT_f64_f32 %0, 0
- %2 = LD_i32 0, 0, 4, 0, 32, -1, &test2_param_1, 0
+ %2 = LD_i32 0, 0, 4, 0, 32, -1, 0, &test2_param_1, 0, $noreg
; CHECK: %3:b64 = FADD_rnf64ri %1, double +qnan
%3 = FADD_rnf64ri %1, double 0x7FF8000000000000
%4 = CVT_f32_f64 %3, 5
@@ -76,6 +76,6 @@ body: |
; CHECK: %6:b32 = FADD_rnf32ri %5, float +qnan
%6 = FADD_rnf32ri %5, float 0x7FF8000000000000, 0
%7 = FMUL_rnf32rr %6, %4, 0
- ST_i32 %7, 0, 0, 101, 32, &func_retval0, 0 :: (store (s32), addrspace 101)
+ ST_i32 %7, 0, 0, 101, 32, 0, &func_retval0, 0, $noreg :: (store (s32), addrspace 101)
Return
...
diff --git a/llvm/test/CodeGen/MIR/X86/memory-operands.mir b/llvm/test/CodeGen/MIR/X86/memory-operands.mir
index 6c2c7e36f1fb6..60a5a4b192aed 100644
--- a/llvm/test/CodeGen/MIR/X86/memory-operands.mir
+++ b/llvm/test/CodeGen/MIR/X86/memory-operands.mir
@@ -169,6 +169,8 @@
}
!12 = !{i8 0, i8 2}
+ !13 = !{i32 0, !14}
+ !14 = !{!"nvvm.l1_eviction", !"first"}
%st = type { i32, i32 }
@@ -490,8 +492,8 @@ body: |
bb.0.entry:
liveins: $rdi
; CHECK-LABEL: name: range_metadata
- ; CHECK: $al = MOV8rm killed $rdi, 1, $noreg, 0, $noreg :: (load (s8) from %ir.x, !range !11)
- $al = MOV8rm killed $rdi, 1, _, 0, _ :: (load (s8) from %ir.x, !range !11)
+ ; CHECK: $al = MOV8rm killed $rdi, 1, $noreg, 0, $noreg :: (load (s8) from %ir.x, !range !11, !mem.cache_hint !12)
+ $al = MOV8rm killed $rdi, 1, _, 0, _ :: (load (s8) from %ir.x, !range !11, !mem.cache_hint !12)
RET64 $al
...
---
diff --git a/llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll b/llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll
new file mode 100644
index 0000000000000..be331571b3108
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll
@@ -0,0 +1,54 @@
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; !mem.cache_hint is legal on atomic IR memory instructions, but
+; SelectionDAGBuilder currently drops it for atomic loads, stores, RMWs, and
+; cmpxchg.
+
+; TODO: This should eventually lower to ld.acquire.sys.global.L1::evict_first.b32.
+; CHECK-LABEL: atomic_load_l1_hint(
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK: ld.acquire.sys.global.b32
+define i32 @atomic_load_l1_hint(ptr addrspace(1) %p) {
+ %v = load atomic i32, ptr addrspace(1) %p acquire, align 4, !mem.cache_hint !0
+ ret i32 %v
+}
+
+; TODO: This should eventually lower to st.release.sys.global.L2::cache_hint.b32.
+; CHECK-LABEL: atomic_store_l2_cache_policy(
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK: st.release.sys.global.b32
+define void @atomic_store_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+ store atomic i32 %v, ptr addrspace(1) %p release, align 4, !mem.cache_hint !2
+ ret void
+}
+
+; TODO: This should eventually lower to atom.relaxed.sys.global.add.L2::cache_hint.u32.
+; CHECK-LABEL: atomicrmw_add_l2_cache_policy(
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK: atom.relaxed.sys.global.add.u32
+define i32 @atomicrmw_add_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+ %old = atomicrmw add ptr addrspace(1) %p, i32 %v monotonic, align 4, !mem.cache_hint !1
+ ret i32 %old
+}
+
+; PTX atom.cas does not have a .level::cache_hint operand.
+; CHECK-LABEL: cmpxchg_l2_cache_policy(
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK: atom.relaxed.sys.global.cas.b32
+define i32 @cmpxchg_l2_cache_policy(ptr addrspace(1) %p, i32 %cmp, i32 %new) {
+ %pair = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new monotonic monotonic, align 4, !mem.cache_hint !1
+ %old = extractvalue { i32, i1 } %pair, 0
+ ret i32 %old
+}
+
+!0 = !{i32 0, !10}
+!1 = !{i32 0, !11}
+!2 = !{i32 1, !11}
+
+!10 = !{!"nvvm.l1_eviction", !"first"}
+!11 = !{!"nvvm.l2_cache_hint", i64 12345}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
new file mode 100644
index 0000000000000..dcd3188fa1270
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
@@ -0,0 +1,346 @@
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefix=SM60
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefix=SM70
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefix=SM75
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefix=SM80
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s --check-prefix=SM80-PTX70
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefix=SM86
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefix=SM90
+
+; Test SM version requirements for cache hints (from PTX ISA documentation):
+; - L1::evict_* requires SM 70+
+; - L2::evict_* requires SM 70+
+; - L2::64B and L2::128B require SM 75+
+; - L2::256B requires SM 80+
+; - L2::cache_hint requires SM 80+ and PTX 7.4+
+
+;-----------------------------------------------------------------------------
+; L1 eviction - requires SM 70+
+; SM60 should NOT emit L1::evict_first (fall back to plain load)
+; SM70+ should emit L1::evict_first
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_l1_first
+; SM60: ld.global.b32
+; SM60-NOT: L1::evict_first
+
+; SM70-LABEL: test_load_l1_first
+; SM70: ld.global.L1::evict_first.b32
+
+; SM75-LABEL: test_load_l1_first
+; SM75: ld.global.L1::evict_first.b32
+
+; SM80-LABEL: test_load_l1_first
+; SM80: ld.global.L1::evict_first.b32
+
+; SM86-LABEL: test_load_l1_first
+; SM86: ld.global.L1::evict_first.b32
+
+; SM90-LABEL: test_load_l1_first
+; SM90: ld.global.L1::evict_first.b32
+define i32 @test_load_l1_first(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2 eviction - requires SM 70+
+; SM60 should NOT emit L2::evict_last (fall back to plain load)
+; SM70+ should emit L2::evict_last
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_l2_last
+; SM60: ld.global.b32
+; SM60-NOT: L2::evict_last
+
+; SM70-LABEL: test_load_l2_last
+; SM70: ld.global.L2::evict_last.b32
+
+; SM75-LABEL: test_load_l2_last
+; SM75: ld.global.L2::evict_last.b32
+
+; SM80-LABEL: test_load_l2_last
+; SM80: ld.global.L2::evict_last.b32
+
+; SM86-LABEL: test_load_l2_last
+; SM86: ld.global.L2::evict_last.b32
+
+; SM90-LABEL: test_load_l2_last
+; SM90: ld.global.L2::evict_last.b32
+define i32 @test_load_l2_last(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2::64B prefetch - requires SM 75+
+; SM60/SM70 should NOT emit L2::64B (fall back to plain load)
+; SM75+ should emit L2::64B
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_prefetch_64
+; SM60: ld.global.b32
+; SM60-NOT: L2::64B
+
+; SM70-LABEL: test_load_prefetch_64
+; SM70: ld.global.b32
+; SM70-NOT: L2::64B
+
+; SM75-LABEL: test_load_prefetch_64
+; SM75: ld.global.L2::64B.b32
+
+; SM80-LABEL: test_load_prefetch_64
+; SM80: ld.global.L2::64B.b32
+
+; SM86-LABEL: test_load_prefetch_64
+; SM86: ld.global.L2::64B.b32
+
+; SM90-LABEL: test_load_prefetch_64
+; SM90: ld.global.L2::64B.b32
+define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2::128B prefetch - requires SM 75+
+; SM60/SM70 should NOT emit L2::128B (fall back to plain load)
+; SM75+ should emit L2::128B
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_prefetch_128
+; SM60: ld.global.b32
+; SM60-NOT: L2::128B
+
+; SM70-LABEL: test_load_prefetch_128
+; SM70: ld.global.b32
+; SM70-NOT: L2::128B
+
+; SM75-LABEL: test_load_prefetch_128
+; SM75: ld.global.L2::128B.b32
+
+; SM80-LABEL: test_load_prefetch_128
+; SM80: ld.global.L2::128B.b32
+
+; SM86-LABEL: test_load_prefetch_128
+; SM86: ld.global.L2::128B.b32
+
+; SM90-LABEL: test_load_prefetch_128
+; SM90: ld.global.L2::128B.b32
+define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2::256B prefetch - requires SM 80+
+; SM60/SM70/SM75 should NOT emit L2::256B (fall back to plain load)
+; SM80+ should emit L2::256B
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_prefetch_256
+; SM60: ld.global.b32
+; SM60-NOT: L2::256B
+
+; SM70-LABEL: test_load_prefetch_256
+; SM70: ld.global.b32
+; SM70-NOT: L2::256B
+
+; SM75-LABEL: test_load_prefetch_256
+; SM75: ld.global.b32
+; SM75-NOT: L2::256B
+
+; SM80-LABEL: test_load_prefetch_256
+; SM80: ld.global.L2::256B.b32
+
+; SM86-LABEL: test_load_prefetch_256
+; SM86: ld.global.L2::256B.b32
+
+; SM90-LABEL: test_load_prefetch_256
+; SM90: ld.global.L2::256B.b32
+define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint - requires SM 80+ and PTX 7.4+
+; SM60/SM70/SM75 should NOT emit L2::cache_hint (fall back to plain load)
+; SM80 with PTX < 7.4 should NOT emit L2::cache_hint
+; SM80+ with PTX 7.4+ should emit L2::cache_hint
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_cache_hint
+; SM60: ld.global.b32
+; SM60-NOT: L2::cache_hint
+
+; SM70-LABEL: test_load_cache_hint
+; SM70: ld.global.b32
+; SM70-NOT: L2::cache_hint
+
+; SM75-LABEL: test_load_cache_hint
+; SM75: ld.global.b32
+; SM75-NOT: L2::cache_hint
+
+; SM80-LABEL: test_load_cache_hint
+; SM80: mov.b64 [[POLICY:%rd[0-9]+]], 12345
+; SM80: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+
+; SM80-PTX70-LABEL: test_load_cache_hint
+; SM80-PTX70: ld.global.b32
+; SM80-PTX70-NOT: L2::cache_hint
+
+; SM86-LABEL: test_load_cache_hint
+; SM86: mov.b64 [[POLICY:%rd[0-9]+]], 12345
+; SM86: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+
+; SM90-LABEL: test_load_cache_hint
+; SM90: mov.b64 [[POLICY:%rd[0-9]+]], 12345
+; SM90: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint combined with L1 eviction on older SM
+; Both hints should be dropped on SM60
+; L1 hint emitted but L2::cache_hint dropped on SM70/SM75
+; Both emitted on SM80+
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_cache_hint_with_l1
+; SM60: ld.global.b32
+; SM60-NOT: L1::evict_first
+; SM60-NOT: L2::cache_hint
+
+; SM70-LABEL: test_load_cache_hint_with_l1
+; SM70: ld.global.L1::evict_first.b32
+; SM70-NOT: L2::cache_hint
+
+; SM75-LABEL: test_load_cache_hint_with_l1
+; SM75: ld.global.L1::evict_first.b32
+; SM75-NOT: L2::cache_hint
+
+; SM80-LABEL: test_load_cache_hint_with_l1
+; SM80: mov.b64 [[POLICY:%rd[0-9]+]], 44445
+; SM80: ld.global.L1::evict_first.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+
+; SM80-PTX70-LABEL: test_load_cache_hint_with_l1
+; SM80-PTX70: ld.global.L1::evict_first.b32
+; SM80-PTX70-NOT: L2::cache_hint
+define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2::128B combined with L1 eviction on older SM
+; Both hints dropped on SM60
+; L1 hint emitted but L2::128B dropped on SM70
+; Both emitted on SM75+
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_load_prefetch_with_l1
+; SM60: ld.global.b32
+; SM60-NOT: L1::evict_first
+; SM60-NOT: L2::128B
+
+; SM70-LABEL: test_load_prefetch_with_l1
+; SM70: ld.global.L1::evict_first.b32
+; SM70-NOT: L2::128B
+
+; SM75-LABEL: test_load_prefetch_with_l1
+; SM75: ld.global.L1::evict_first.L2::128B.b32
+
+; SM80-LABEL: test_load_prefetch_with_l1
+; SM80: ld.global.L1::evict_first.L2::128B.b32
+define i32 @test_load_prefetch_with_l1(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Store with L2::cache_hint
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_store_cache_hint
+; SM60: st.global.b32
+; SM60-NOT: L2::cache_hint
+
+; SM70-LABEL: test_store_cache_hint
+; SM70: st.global.b32
+; SM70-NOT: L2::cache_hint
+
+; SM80-LABEL: test_store_cache_hint
+; SM80: mov.b64 [[POLICY:%rd[0-9]+]], 67890
+; SM80: st.global.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
+
+; SM80-PTX70-LABEL: test_store_cache_hint
+; SM80-PTX70: st.global.b32
+; SM80-PTX70-NOT: L2::cache_hint
+define void @test_store_cache_hint(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Store with L1 eviction hint
+;-----------------------------------------------------------------------------
+
+; SM60-LABEL: test_store_l1_no_allocate
+; SM60: st.global.b32
+; SM60-NOT: L1::no_allocate
+
+; SM70-LABEL: test_store_l1_no_allocate
+; SM70: st.global.L1::no_allocate.b32
+
+; SM80-LABEL: test_store_l1_no_allocate
+; SM80: st.global.L1::no_allocate.b32
+define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !9
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+; L1 eviction: first
+!0 = !{i32 0, !100}
+!100 = !{!"nvvm.l1_eviction", !"first"}
+
+; L2 eviction: last
+!1 = !{i32 0, !101}
+!101 = !{!"nvvm.l2_eviction", !"last"}
+
+; L2 prefetch: 128B
+!2 = !{i32 0, !102}
+!102 = !{!"nvvm.l2_prefetch_size", !"128B"}
+
+; L2::cache_hint only
+!3 = !{i32 0, !103}
+!103 = !{!"nvvm.l2_cache_hint", i64 12345}
+
+; L2::cache_hint + L1 eviction
+!4 = !{i32 0, !104}
+!104 = !{!"nvvm.l2_cache_hint", i64 44445, !"nvvm.l1_eviction", !"first"}
+
+; L2::cache_hint for store
+!5 = !{i32 1, !105}
+!105 = !{!"nvvm.l2_cache_hint", i64 67890}
+
+; L2 prefetch: 64B
+!6 = !{i32 0, !106}
+!106 = !{!"nvvm.l2_prefetch_size", !"64B"}
+
+; L2 prefetch: 256B
+!7 = !{i32 0, !107}
+!107 = !{!"nvvm.l2_prefetch_size", !"256B"}
+
+; L2 prefetch: 128B + L1 eviction
+!8 = !{i32 0, !108}
+!108 = !{!"nvvm.l2_prefetch_size", !"128B", !"nvvm.l1_eviction", !"first"}
+
+; L1 eviction: no_allocate (for store)
+!9 = !{i32 1, !109}
+!109 = !{!"nvvm.l1_eviction", !"no_allocate"}
diff --git a/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll b/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
new file mode 100644
index 0000000000000..246a819141275
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
@@ -0,0 +1,131 @@
+; RUN: split-file %s %t
+; RUN: not llc < %t/bad-empty.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-EMPTY
+; RUN: not llc < %t/bad-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-KEY
+; RUN: not llc < %t/bad-other-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-OTHER-KEY
+; RUN: not llc < %t/bad-l1-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-TYPE
+; RUN: not llc < %t/bad-l1-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-VALUE
+; RUN: not llc < %t/bad-l2-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-TYPE
+; RUN: not llc < %t/bad-l2-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-VALUE
+; RUN: not llc < %t/bad-prefetch-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-TYPE
+; RUN: not llc < %t/bad-prefetch-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-VALUE
+; RUN: not llc < %t/bad-policy.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-POLICY
+
+;--- bad-empty.ll
+
+; Test with empty hint node - should produce an empty-node diagnostic.
+; BAD-EMPTY: invalid NVPTX !mem.cache_hint metadata: empty hint node
+define i32 @bad_empty_hint_node(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{}
+
+;--- bad-key.ll
+
+; Test with misspelled NVPTX key - should produce an unknown-key diagnostic.
+; BAD-KEY: invalid NVPTX !mem.cache_hint metadata: unknown key 'nvvm.l1_evict'
+define i32 @bad_nvvm_key(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l1_evict", !"first"}
+
+;--- bad-other-key.ll
+
+; Test with a non-NVPTX key - should produce an unknown-key diagnostic.
+; BAD-OTHER-KEY: invalid NVPTX !mem.cache_hint metadata: unknown key 'some.target_hint'
+define i32 @bad_other_key(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"some.target_hint", !"value"}
+
+;--- bad-l1-type.ll
+
+; nvvm.l1_eviction expects a string value.
+; BAD-L1-TYPE: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l1_eviction' expects a string value
+define i32 @bad_l1_type(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l1_eviction", i32 0}
+
+;--- bad-l1-value.ll
+
+; nvvm.l1_eviction accepts only known PTX eviction values.
+; BAD-L1-VALUE: invalid NVPTX !mem.cache_hint metadata: unknown value 'middle' for 'nvvm.l1_eviction'
+define i32 @bad_l1_value(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l1_eviction", !"middle"}
+
+;--- bad-l2-type.ll
+
+; nvvm.l2_eviction expects a string value.
+; BAD-L2-TYPE: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_eviction' expects a string value
+define i32 @bad_l2_type(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l2_eviction", i32 0}
+
+;--- bad-l2-value.ll
+
+; nvvm.l2_eviction accepts only known PTX eviction values.
+; BAD-L2-VALUE: invalid NVPTX !mem.cache_hint metadata: unknown value 'middle' for 'nvvm.l2_eviction'
+define i32 @bad_l2_value(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l2_eviction", !"middle"}
+
+;--- bad-prefetch-type.ll
+
+; nvvm.l2_prefetch_size expects a string value.
+; BAD-PREFETCH-TYPE: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_prefetch_size' expects a string value
+define i32 @bad_prefetch_type(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l2_prefetch_size", i32 64}
+
+;--- bad-prefetch-value.ll
+
+; nvvm.l2_prefetch_size accepts only supported PTX prefetch sizes.
+; BAD-PREFETCH-VALUE: invalid NVPTX !mem.cache_hint metadata: unknown value '32B' for 'nvvm.l2_prefetch_size'
+define i32 @bad_prefetch_value(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l2_prefetch_size", !"32B"}
+
+;--- bad-policy.ll
+
+; nvvm.l2_cache_hint expects an integer cache-policy value.
+; BAD-POLICY: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_cache_hint' expects an integer value
+define i32 @bad_cache_policy_value(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l2_cache_hint", !"not_an_integer"}
diff --git a/llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll b/llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll
new file mode 100644
index 0000000000000..2a6ac808d2d2e
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll
@@ -0,0 +1,935 @@
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; Test !mem.cache_hint metadata lowering to PTX cache qualifiers
+; PTX supports the following cache qualifiers:
+; L1 eviction: L1::evict_first, L1::evict_last, L1::evict_unchanged, L1::no_allocate
+; L2 eviction: L2::evict_first, L2::evict_last
+; L2 prefetch: L2::64B, L2::128B, L2::256B
+
+;-----------------------------------------------------------------------------
+; Basic L1 eviction policies for loads
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_l1_first
+; CHECK: ld.global.L1::evict_first.b32
+define i32 @test_load_l1_first(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l1_last
+; CHECK: ld.global.L1::evict_last.b32
+define i32 @test_load_l1_last(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l1_unchanged
+; CHECK: ld.global.L1::evict_unchanged.b32
+define i32 @test_load_l1_unchanged(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l1_no_allocate
+; CHECK: ld.global.L1::no_allocate.b32
+define i32 @test_load_l1_no_allocate(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Basic L2 eviction policies for loads
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_l2_first
+; CHECK: ld.global.L2::evict_first.b32
+define i32 @test_load_l2_first(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l2_last
+; CHECK: ld.global.L2::evict_last.b32
+define i32 @test_load_l2_last(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2 prefetch sizes for loads
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_prefetch_64
+; CHECK: ld.global.L2::64B.b32
+define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_prefetch_128
+; CHECK: ld.global.L2::128B.b32
+define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_prefetch_256
+; CHECK: ld.global.L2::256B.b32
+define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; All L1 + L2 combinations for loads
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_l1_first_l2_first
+; CHECK: ld.global.L1::evict_first.L2::evict_first.b32
+define i32 @test_load_l1_first_l2_first(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !20
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l1_first_l2_last
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b32
+define i32 @test_load_l1_first_l2_last(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !21
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l1_last_l2_first
+; CHECK: ld.global.L1::evict_last.L2::evict_first.b32
+define i32 @test_load_l1_last_l2_first(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !22
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l1_last_l2_last
+; CHECK: ld.global.L1::evict_last.L2::evict_last.b32
+define i32 @test_load_l1_last_l2_last(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !23
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L1 + L2 + Prefetch combination for loads
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_l1_first_l2_last_prefetch_128
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::128B.b32
+define i32 @test_load_l1_first_l2_last_prefetch_128(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !24
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Basic L1 eviction policies for stores
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_store_l1_first
+; CHECK: st.global.L1::evict_first.b32
+define void @test_store_l1_first(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1000
+ ret void
+}
+
+; CHECK-LABEL: test_store_l1_last
+; CHECK: st.global.L1::evict_last.b32
+define void @test_store_l1_last(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1001
+ ret void
+}
+
+; CHECK-LABEL: test_store_l1_unchanged
+; CHECK: st.global.L1::evict_unchanged.b32
+define void @test_store_l1_unchanged(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1002
+ ret void
+}
+
+; CHECK-LABEL: test_store_l1_no_allocate
+; CHECK: st.global.L1::no_allocate.b32
+define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1003
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Basic L2 eviction policies for stores
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_store_l2_first
+; CHECK: st.global.L2::evict_first.b32
+define void @test_store_l2_first(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1004
+ ret void
+}
+
+; CHECK-LABEL: test_store_l2_last
+; CHECK: st.global.L2::evict_last.b32
+define void @test_store_l2_last(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1005
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; All L1 + L2 combinations for stores
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_store_l1_first_l2_first
+; CHECK: st.global.L1::evict_first.L2::evict_first.b32
+define void @test_store_l1_first_l2_first(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1020
+ ret void
+}
+
+; CHECK-LABEL: test_store_l1_first_l2_last
+; CHECK: st.global.L1::evict_first.L2::evict_last.b32
+define void @test_store_l1_first_l2_last(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1021
+ ret void
+}
+
+; CHECK-LABEL: test_store_l1_last_l2_first
+; CHECK: st.global.L1::evict_last.L2::evict_first.b32
+define void @test_store_l1_last_l2_first(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1022
+ ret void
+}
+
+; CHECK-LABEL: test_store_l1_last_l2_last
+; CHECK: st.global.L1::evict_last.L2::evict_last.b32
+define void @test_store_l1_last_l2_last(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1023
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Different data types - loads
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_i16_l1_first
+; CHECK: ld.global.L1::evict_first.b16
+define i16 @test_load_i16_l1_first(ptr addrspace(1) %p) {
+ %v = load i16, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i16 %v
+}
+
+; CHECK-LABEL: test_load_i64_l1_last
+; CHECK: ld.global.L1::evict_last.b64
+define i64 @test_load_i64_l1_last(ptr addrspace(1) %p) {
+ %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !1
+ ret i64 %v
+}
+
+; CHECK-LABEL: test_load_f32_l2_first
+; CHECK: ld.global.L2::evict_first.b32
+define float @test_load_f32_l2_first(ptr addrspace(1) %p) {
+ %v = load float, ptr addrspace(1) %p, !mem.cache_hint !4
+ ret float %v
+}
+
+; CHECK-LABEL: test_load_f64_l2_last
+; CHECK: ld.global.L2::evict_last.b64
+define double @test_load_f64_l2_last(ptr addrspace(1) %p) {
+ %v = load double, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret double %v
+}
+
+;-----------------------------------------------------------------------------
+; Different data types - stores
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_store_i16_l1_first
+; CHECK: st.global.L1::evict_first.b16
+define void @test_store_i16_l1_first(ptr addrspace(1) %p, i16 %v) {
+ store i16 %v, ptr addrspace(1) %p, !mem.cache_hint !1000
+ ret void
+}
+
+; CHECK-LABEL: test_store_i64_l2_last
+; CHECK: st.global.L2::evict_last.b64
+define void @test_store_i64_l2_last(ptr addrspace(1) %p, i64 %v) {
+ store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !1005
+ ret void
+}
+
+; CHECK-LABEL: test_store_f32_l1_no_allocate
+; CHECK: st.global.L1::no_allocate.b32
+define void @test_store_f32_l1_no_allocate(ptr addrspace(1) %p, float %v) {
+ store float %v, ptr addrspace(1) %p, !mem.cache_hint !1003
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Vector loads with cache hints
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_v2i32_l1_first
+; CHECK: ld.global.L1::evict_first.v2.b32
+define <2 x i32> @test_load_v2i32_l1_first(ptr addrspace(1) %p) {
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret <2 x i32> %v
+}
+
+; CHECK-LABEL: test_load_v4i32_l2_last
+; CHECK: ld.global.L2::evict_last.v4.b32
+define <4 x i32> @test_load_v4i32_l2_last(ptr addrspace(1) %p) {
+ %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret <4 x i32> %v
+}
+
+; CHECK-LABEL: test_load_v2f32_l1_unchanged
+; CHECK: ld.global.L1::evict_unchanged.v2.b32
+define <2 x float> @test_load_v2f32_l1_unchanged(ptr addrspace(1) %p) {
+ %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !2
+ ret <2 x float> %v
+}
+
+; CHECK-LABEL: test_load_v2f64_prefetch_128
+; CHECK: ld.global.L2::128B.v2.b64
+define <2 x double> @test_load_v2f64_prefetch_128(ptr addrspace(1) %p) {
+ %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret <2 x double> %v
+}
+
+;-----------------------------------------------------------------------------
+; Vector stores with cache hints
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_store_v2i32_l1_last
+; CHECK: st.global.L1::evict_last.v2.b32
+define void @test_store_v2i32_l1_last(ptr addrspace(1) %p, <2 x i32> %v) {
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1001
+ ret void
+}
+
+; CHECK-LABEL: test_store_v4i32_l2_first
+; CHECK: st.global.L2::evict_first.v4.b32
+define void @test_store_v4i32_l2_first(ptr addrspace(1) %p, <4 x i32> %v) {
+ store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1004
+ ret void
+}
+
+; CHECK-LABEL: test_store_v2f64_l1_no_allocate
+; CHECK: st.global.L1::no_allocate.v2.b64
+define void @test_store_v2f64_l1_no_allocate(ptr addrspace(1) %p, <2 x double> %v) {
+ store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !1003
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Invariant loads with cache hints may still use LDG (ld.global.nc)
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_invariant_load_with_hint
+; CHECK: ld.global.nc.L1::evict_first.b32
+define i32 @test_invariant_load_with_hint(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !0
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_invariant_load_with_cache_policy
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 12345
+; CHECK: ld.global.nc.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_invariant_load_with_cache_policy(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !30
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_invariant_load_v2i32_with_hint
+; CHECK: ld.global.nc.L1::evict_last.L2::evict_first.v2.b32
+define <2 x i32> @test_invariant_load_v2i32_with_hint(ptr addrspace(1) %p) {
+ %v = load <2 x i32>, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !22
+ ret <2 x i32> %v
+}
+
+;-----------------------------------------------------------------------------
+; No hint should produce plain load/store
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_no_hint
+; CHECK: ld.global.b32
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+define i32 @test_load_no_hint(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_store_no_hint
+; CHECK: st.global.b32
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+define void @test_store_no_hint(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint with constant cache-policy operand (metadata-based)
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_cache_hint_i32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 12345
+; CHECK: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint_i32(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !30
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_cache_hint_i64
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 98765
+; CHECK: ld.global.L2::cache_hint.b64 {{%rd[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i64 @test_load_cache_hint_i64(ptr addrspace(1) %p) {
+ %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !31
+ ret i64 %v
+}
+
+; CHECK-LABEL: test_load_cache_hint_f32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 55555
+; CHECK: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define float @test_load_cache_hint_f32(ptr addrspace(1) %p) {
+ %v = load float, ptr addrspace(1) %p, !mem.cache_hint !32
+ ret float %v
+}
+
+; CHECK-LABEL: test_store_cache_hint_i32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 67890
+; CHECK: st.global.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
+define void @test_store_cache_hint_i32(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1033
+ ret void
+}
+
+; CHECK-LABEL: test_store_cache_hint_i64
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 11111
+; CHECK: st.global.L2::cache_hint.b64 [{{%rd[0-9]+}}], {{%rd[0-9]+}}, [[POLICY]]
+define void @test_store_cache_hint_i64(ptr addrspace(1) %p, i64 %v) {
+ store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !1034
+ ret void
+}
+
+; CHECK-LABEL: test_store_cache_hint_f32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 22222
+; CHECK: st.global.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
+define void @test_store_cache_hint_f32(ptr addrspace(1) %p, float %v) {
+ store float %v, ptr addrspace(1) %p, !mem.cache_hint !1035
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint with vector types
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_load_cache_hint_v2i32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 33333
+; CHECK: ld.global.L2::cache_hint.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
+define <2 x i32> @test_load_cache_hint_v2i32(ptr addrspace(1) %p) {
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !40
+ ret <2 x i32> %v
+}
+
+; CHECK-LABEL: test_load_cache_hint_v4i32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 44444
+; CHECK: ld.global.L2::cache_hint.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
+define <4 x i32> @test_load_cache_hint_v4i32(ptr addrspace(1) %p) {
+ %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !41
+ ret <4 x i32> %v
+}
+
+; CHECK-LABEL: test_load_cache_hint_v2i64
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 55556
+; CHECK: ld.global.L2::cache_hint.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
+define <2 x i64> @test_load_cache_hint_v2i64(ptr addrspace(1) %p) {
+ %v = load <2 x i64>, ptr addrspace(1) %p, !mem.cache_hint !42
+ ret <2 x i64> %v
+}
+
+; CHECK-LABEL: test_load_cache_hint_v2f32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 66666
+; CHECK: ld.global.L2::cache_hint.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
+define <2 x float> @test_load_cache_hint_v2f32(ptr addrspace(1) %p) {
+ %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !43
+ ret <2 x float> %v
+}
+
+; CHECK-LABEL: test_load_cache_hint_v2f64
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 77777
+; CHECK: ld.global.L2::cache_hint.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
+define <2 x double> @test_load_cache_hint_v2f64(ptr addrspace(1) %p) {
+ %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !44
+ ret <2 x double> %v
+}
+
+; CHECK-LABEL: test_store_cache_hint_v2i32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 88888
+; CHECK: st.global.L2::cache_hint.v2.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
+define void @test_store_cache_hint_v2i32(ptr addrspace(1) %p, <2 x i32> %v) {
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1045
+ ret void
+}
+
+; CHECK-LABEL: test_store_cache_hint_v4i32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 99999
+; CHECK: st.global.L2::cache_hint.v4.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
+define void @test_store_cache_hint_v4i32(ptr addrspace(1) %p, <4 x i32> %v) {
+ store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1046
+ ret void
+}
+
+; CHECK-LABEL: test_store_cache_hint_v2i64
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 11112
+; CHECK: st.global.L2::cache_hint.v2.b64 [{{%rd[0-9]+}}], {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [[POLICY]]
+define void @test_store_cache_hint_v2i64(ptr addrspace(1) %p, <2 x i64> %v) {
+ store <2 x i64> %v, ptr addrspace(1) %p, !mem.cache_hint !1047
+ ret void
+}
+
+; CHECK-LABEL: test_store_cache_hint_v2f32
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 22223
+; CHECK: st.global.L2::cache_hint.v2.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
+define void @test_store_cache_hint_v2f32(ptr addrspace(1) %p, <2 x float> %v) {
+ store <2 x float> %v, ptr addrspace(1) %p, !mem.cache_hint !1048
+ ret void
+}
+
+; CHECK-LABEL: test_store_cache_hint_v2f64
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 33334
+; CHECK: st.global.L2::cache_hint.v2.b64 [{{%rd[0-9]+}}], {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [[POLICY]]
+define void @test_store_cache_hint_v2f64(ptr addrspace(1) %p, <2 x double> %v) {
+ store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !1049
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint combined with other hints (L2::cache_hint takes precedence)
+;-----------------------------------------------------------------------------
+
+; L2::cache_hint + L1 eviction: both qualifiers should be emitted
+; CHECK-LABEL: test_load_cache_hint_with_l1
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 44445
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !50
+ ret i32 %v
+}
+
+; L2::cache_hint + L2 eviction: both qualifiers should be emitted
+; CHECK-LABEL: test_load_cache_hint_with_l2_eviction
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 55557
+; CHECK: ld.global.L2::evict_last.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint_with_l2_eviction(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !51
+ ret i32 %v
+}
+
+; L2::cache_hint + L2 prefetch: both qualifiers should be emitted
+; CHECK-LABEL: test_load_cache_hint_with_prefetch
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 66667
+; CHECK: ld.global.L2::cache_hint.L2::128B.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint_with_prefetch(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !52
+ ret i32 %v
+}
+
+; L2::cache_hint + all other hints: all qualifiers emitted
+; CHECK-LABEL: test_load_cache_hint_with_all
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 77778
+; CHECK: ld.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::256B.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint_with_all(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !53
+ ret i32 %v
+}
+
+; Store: L2::cache_hint + L1 eviction
+; CHECK-LABEL: test_store_cache_hint_with_l1
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 88889
+; CHECK: st.global.L1::evict_unchanged.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
+define void @test_store_cache_hint_with_l1(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1054
+ ret void
+}
+
+; Store: L2::cache_hint + L1 + L2 eviction (all qualifiers emitted)
+; CHECK-LABEL: test_store_cache_hint_with_all
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 99990
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
+define void @test_store_cache_hint_with_all(ptr addrspace(1) %p, i32 %v) {
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1055
+ ret void
+}
+
+; Vector load: L2::cache_hint + L1 eviction
+; CHECK-LABEL: test_load_cache_hint_v2i32_with_l1
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 11113
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
+define <2 x i32> @test_load_cache_hint_v2i32_with_l1(ptr addrspace(1) %p) {
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !56
+ ret <2 x i32> %v
+}
+
+; Vector store: L2::cache_hint + L1 + L2 eviction + prefetch (all qualifiers emitted)
+; CHECK-LABEL: test_store_cache_hint_v2i32_with_all
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 22224
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::64B.v2.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
+define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32> %v) {
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1057
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Multiple loads sharing same pointer - each gets its own policy
+;-----------------------------------------------------------------------------
+
+; Two volatile loads from the same pointer - each load gets its own cache policy.
+; The per-MMO storage ensures no policy collisions when multiple memops share
+; the same pointer operand but have different cache policies.
+; CHECK-LABEL: test_multiple_loads_same_ptr
+; CHECK-DAG: mov.b64 [[POLICY1:%rd[0-9]+]], 11111
+; CHECK-DAG: mov.b64 [[POLICY2:%rd[0-9]+]], 22222
+; CHECK-DAG: ld.volatile.global.L1::evict_last.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY1]]
+; CHECK-DAG: ld.volatile.global.L1::evict_first.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY2]]
+define i32 @test_multiple_loads_same_ptr(ptr addrspace(1) %p) {
+ %v1 = load volatile i32, ptr addrspace(1) %p, !mem.cache_hint !60
+ %v2 = load volatile i32, ptr addrspace(1) %p, !mem.cache_hint !61
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Non-volatile loads can CSE. Matching cache hints are preserved on the merged
+; DAG node, but conflicting hints are dropped.
+; CHECK-LABEL: test_cse_loads_same_cache_hint
+; CHECK: ld.global.L1::evict_first.b32
+; CHECK-NOT: ld.global
+define i32 @test_cse_loads_same_cache_hint(ptr addrspace(1) %p) {
+ %v1 = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ %v2 = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; CHECK-LABEL: test_cse_loads_conflicting_cache_hints
+; CHECK: ld.global.b32
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK-NOT: ld.global
+define i32 @test_cse_loads_conflicting_cache_hints(ptr addrspace(1) %p) {
+ %v1 = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ %v2 = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; NVPTXForwardParams rewrites eligible byval loads to ld.param. Since cache
+; hints are not allowed on ld.param, we must drop them.
+; CHECK-LABEL: test_forward_param_drops_l1_hint
+; CHECK-NOT: L1::evict_first
+; CHECK: ld.param.b32
+; CHECK-NOT: L1::evict_first
+; CHECK-NOT: ld.local
+define i32 @test_forward_param_drops_l1_hint(ptr byval(i32) %a) {
+ %v = load i32, ptr %a, !mem.cache_hint !90
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_forward_param_drops_l2_cache_hint
+; CHECK-NOT: mov.b64
+; CHECK-NOT: L2::cache_hint
+; CHECK: ld.param.b32
+; CHECK-NOT: mov.b64
+; CHECK-NOT: L2::cache_hint
+; CHECK-NOT: ld.local
+define i32 @test_forward_param_drops_l2_cache_hint(ptr byval(i32) %a) {
+ %v = load i32, ptr %a, !mem.cache_hint !91
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Valid edge cases
+;-----------------------------------------------------------------------------
+
+; Test with custom hint key order - should still work
+; CHECK-LABEL: test_load_reordered_metadata
+; CHECK: ld.global.L1::evict_last.L2::evict_first.b32
+define i32 @test_load_reordered_metadata(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !11
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; nvvm.l2_cache_hint with alternate integer value width
+;-----------------------------------------------------------------------------
+
+; nvvm.l2_cache_hint with i32 instead of i64 - should still work
+; as mdconst::dyn_extract<ConstantInt> accepts any integer type
+; CHECK-LABEL: test_load_cache_hint_i32_value
+; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 99999
+; CHECK: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
+define i32 @test_load_cache_hint_i32_value(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !12
+ ret i32 %v
+}
+
+; Test "normal" eviction - should not emit any qualifier (default behavior)
+; CHECK-LABEL: test_load_l1_normal
+; CHECK: ld.global.b32
+; CHECK-NOT: L1::evict_normal
+define i32 @test_load_l1_normal(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !13
+ ret i32 %v
+}
+
+; CHECK-LABEL: test_load_l2_normal
+; CHECK: ld.global.b32
+; CHECK-NOT: L2::evict_normal
+define i32 @test_load_l2_normal(ptr addrspace(1) %p) {
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !14
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; TODO: Preserve cache hints across DAGCombiner-created memory rewrites.
+; This documents the current store-of-concat-trunc behavior: copied MMOs for
+; the split stores do not retain !mem.cache_hint metadata yet.
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_dagcombine_store_concat_trunc_v8i32
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK-COUNT-2: st.global.v4.b32
+define void @test_dagcombine_store_concat_trunc_v8i32(ptr addrspace(1) %p, <4 x i64> %a, <4 x i64> %b) {
+ %ta = trunc <4 x i64> %a to <4 x i32>
+ %tb = trunc <4 x i64> %b to <4 x i32>
+ %c = shufflevector <4 x i32> %ta, <4 x i32> %tb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ store <8 x i32> %c, ptr addrspace(1) %p, align 16, !mem.cache_hint !2004
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; TODO: Preserve cache hints across one-to-N DAG memory rewrites.
+; These tests document the current split/scalarized behavior: newly-created
+; memory ops do not retain !mem.cache_hint metadata yet.
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_legalize_split_load_v16i32
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK-COUNT-4: ld.global.v4.b32
+define <16 x i32> @test_legalize_split_load_v16i32(ptr addrspace(1) %p) {
+ %v = load <16 x i32>, ptr addrspace(1) %p, align 16, !mem.cache_hint !2000
+ ret <16 x i32> %v
+}
+
+; CHECK-LABEL: test_legalize_split_store_v16i32
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK-COUNT-4: st.global.v4.b32
+define void @test_legalize_split_store_v16i32(ptr addrspace(1) %p, <16 x i32> %v) {
+ store <16 x i32> %v, ptr addrspace(1) %p, align 16, !mem.cache_hint !2001
+ ret void
+}
+
+; CHECK-LABEL: test_legalize_scalarize_load_v3i64
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK-COUNT-3: ld.global.{{[bu]}}64
+define <3 x i64> @test_legalize_scalarize_load_v3i64(ptr addrspace(1) %p) {
+ %v = load <3 x i64>, ptr addrspace(1) %p, align 8, !mem.cache_hint !2002
+ ret <3 x i64> %v
+}
+
+; CHECK-LABEL: test_legalize_scalarize_store_v3i64
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK-COUNT-3: st.global.{{[bu]}}64
+define void @test_legalize_scalarize_store_v3i64(ptr addrspace(1) %p, <3 x i64> %v) {
+ store <3 x i64> %v, ptr addrspace(1) %p, align 8, !mem.cache_hint !2003
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+; L1 eviction policies
+!0 = !{i32 0, !100}
+!100 = !{!"nvvm.l1_eviction", !"first"}
+
+!1 = !{i32 0, !101}
+!101 = !{!"nvvm.l1_eviction", !"last"}
+
+!2 = !{i32 0, !102}
+!102 = !{!"nvvm.l1_eviction", !"unchanged"}
+
+!3 = !{i32 0, !103}
+!103 = !{!"nvvm.l1_eviction", !"no_allocate"}
+
+; L2 eviction policies
+!4 = !{i32 0, !104}
+!104 = !{!"nvvm.l2_eviction", !"first"}
+
+!5 = !{i32 0, !105}
+!105 = !{!"nvvm.l2_eviction", !"last"}
+
+; L2 prefetch sizes
+!6 = !{i32 0, !106}
+!106 = !{!"nvvm.l2_prefetch_size", !"64B"}
+
+!7 = !{i32 0, !107}
+!107 = !{!"nvvm.l2_prefetch_size", !"128B"}
+
+!8 = !{i32 0, !108}
+!108 = !{!"nvvm.l2_prefetch_size", !"256B"}
+
+; Custom key order in hint node
+!11 = !{i32 0, !111}
+!111 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+
+; i32 instead of i64 - still valid, ConstantInt accepts any integer type
+!12 = !{i32 0, !112}
+!112 = !{!"nvvm.l2_cache_hint", i32 99999}
+
+; "normal" eviction (default, should not emit qualifier)
+!13 = !{i32 0, !113}
+!113 = !{!"nvvm.l1_eviction", !"normal"}
+
+!14 = !{i32 0, !114}
+!114 = !{!"nvvm.l2_eviction", !"normal"}
+
+; All L1 + L2 combinations
+!20 = !{i32 0, !120}
+!120 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first"}
+
+!21 = !{i32 0, !121}
+!121 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last"}
+
+!22 = !{i32 0, !122}
+!122 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+
+!23 = !{i32 0, !123}
+!123 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last"}
+
+; L1 + L2 + Prefetch combination
+!24 = !{i32 0, !124}
+!124 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
+
+; L2::cache_hint with constant cache-policy
+!30 = !{i32 0, !130}
+!130 = !{!"nvvm.l2_cache_hint", i64 12345}
+
+!31 = !{i32 0, !131}
+!131 = !{!"nvvm.l2_cache_hint", i64 98765}
+
+!32 = !{i32 0, !132}
+!132 = !{!"nvvm.l2_cache_hint", i64 55555}
+
+!133 = !{!"nvvm.l2_cache_hint", i64 67890}
+
+!134 = !{!"nvvm.l2_cache_hint", i64 11111}
+
+!135 = !{!"nvvm.l2_cache_hint", i64 22222}
+
+; L2::cache_hint for vector types
+!40 = !{i32 0, !140}
+!140 = !{!"nvvm.l2_cache_hint", i64 33333}
+
+!41 = !{i32 0, !141}
+!141 = !{!"nvvm.l2_cache_hint", i64 44444}
+
+!42 = !{i32 0, !142}
+!142 = !{!"nvvm.l2_cache_hint", i64 55556}
+
+!43 = !{i32 0, !143}
+!143 = !{!"nvvm.l2_cache_hint", i64 66666}
+
+!44 = !{i32 0, !144}
+!144 = !{!"nvvm.l2_cache_hint", i64 77777}
+
+!145 = !{!"nvvm.l2_cache_hint", i64 88888}
+
+!146 = !{!"nvvm.l2_cache_hint", i64 99999}
+
+!147 = !{!"nvvm.l2_cache_hint", i64 11112}
+
+!148 = !{!"nvvm.l2_cache_hint", i64 22223}
+
+!149 = !{!"nvvm.l2_cache_hint", i64 33334}
+
+; L2::cache_hint combined with other hints (L2::cache_hint takes precedence)
+!50 = !{i32 0, !150}
+!150 = !{!"nvvm.l2_cache_hint", i64 44445, !"nvvm.l1_eviction", !"first"}
+
+!51 = !{i32 0, !151}
+!151 = !{!"nvvm.l2_cache_hint", i64 55557, !"nvvm.l2_eviction", !"last"}
+
+!52 = !{i32 0, !152}
+!152 = !{!"nvvm.l2_cache_hint", i64 66667, !"nvvm.l2_prefetch_size", !"128B"}
+
+!53 = !{i32 0, !153}
+!153 = !{!"nvvm.l2_cache_hint", i64 77778, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
+
+!154 = !{!"nvvm.l2_cache_hint", i64 88889, !"nvvm.l1_eviction", !"unchanged"}
+
+!155 = !{!"nvvm.l2_cache_hint", i64 99990, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
+
+!56 = !{i32 0, !156}
+!156 = !{!"nvvm.l2_cache_hint", i64 11113, !"nvvm.l1_eviction", !"first"}
+
+!157 = !{!"nvvm.l2_cache_hint", i64 22224, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"64B"}
+
+; Multiple loads same pointer test (different policies)
+!60 = !{i32 0, !160}
+!160 = !{!"nvvm.l2_cache_hint", i64 11111, !"nvvm.l1_eviction", !"last"}
+
+!61 = !{i32 0, !161}
+!161 = !{!"nvvm.l2_cache_hint", i64 22222, !"nvvm.l1_eviction", !"first"}
+
+; ForwardParams byval tests
+!90 = !{i32 0, !190}
+!190 = !{!"nvvm.l1_eviction", !"first"}
+
+!91 = !{i32 0, !191}
+!191 = !{!"nvvm.l2_cache_hint", i64 12345}
+
+; Store-side metadata uses operand 1 (the store pointer operand).
+!1000 = !{i32 1, !100}
+!1001 = !{i32 1, !101}
+!1002 = !{i32 1, !102}
+!1003 = !{i32 1, !103}
+!1004 = !{i32 1, !104}
+!1005 = !{i32 1, !105}
+!1020 = !{i32 1, !120}
+!1021 = !{i32 1, !121}
+!1022 = !{i32 1, !122}
+!1023 = !{i32 1, !123}
+!1033 = !{i32 1, !133}
+!1034 = !{i32 1, !134}
+!1035 = !{i32 1, !135}
+!1045 = !{i32 1, !145}
+!1046 = !{i32 1, !146}
+!1047 = !{i32 1, !147}
+!1048 = !{i32 1, !148}
+!1049 = !{i32 1, !149}
+!1054 = !{i32 1, !154}
+!1055 = !{i32 1, !155}
+!1057 = !{i32 1, !157}
+
+; Legalization tests
+!2000 = !{i32 0, !2100}
+!2100 = !{!"nvvm.l2_cache_hint", i64 424242}
+
+!2001 = !{i32 1, !2101}
+!2101 = !{!"nvvm.l2_cache_hint", i64 515151}
+
+!2002 = !{i32 0, !2102}
+!2102 = !{!"nvvm.l1_eviction", !"last"}
+
+!2003 = !{i32 1, !2103}
+!2103 = !{!"nvvm.l2_eviction", !"first"}
+
+!2004 = !{i32 1, !2104}
+!2104 = !{!"nvvm.l2_cache_hint", i64 616161}
diff --git a/llvm/test/CodeGen/NVPTX/machinelicm-no-preheader.mir b/llvm/test/CodeGen/NVPTX/machinelicm-no-preheader.mir
index 186e97088039f..f7ad6ed209f0c 100644
--- a/llvm/test/CodeGen/NVPTX/machinelicm-no-preheader.mir
+++ b/llvm/test/CodeGen/NVPTX/machinelicm-no-preheader.mir
@@ -26,10 +26,10 @@ body: |
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.2(0x30000000), %bb.3(0x50000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[LD_i32_:%[0-9]+]]:b32 = LD_i32 0, 0, 101, 3, 32, -1, &test_hoist_param_1, 0 :: (dereferenceable invariant load (s32), addrspace 101)
- ; CHECK-NEXT: [[LD_i64_:%[0-9]+]]:b64 = LD_i64 0, 0, 101, 3, 64, -1, &test_hoist_param_0, 0 :: (dereferenceable invariant load (s64), addrspace 101)
+ ; CHECK-NEXT: [[LD_i32_:%[0-9]+]]:b32 = LD_i32 0, 0, 101, 3, 32, -1, 0, 0, &test_hoist_param_1, 0 :: (dereferenceable invariant load (s32), addrspace 101)
+ ; CHECK-NEXT: [[LD_i64_:%[0-9]+]]:b64 = LD_i64 0, 0, 101, 3, 64, -1, 0, 0, &test_hoist_param_0, 0 :: (dereferenceable invariant load (s64), addrspace 101)
; CHECK-NEXT: [[ADD64ri:%[0-9]+]]:b64 = nuw ADD64ri killed [[LD_i64_]], 2
- ; CHECK-NEXT: [[LD_i32_1:%[0-9]+]]:b32 = LD_i32 0, 0, 1, 3, 32, -1, [[ADD64ri]], 0
+ ; CHECK-NEXT: [[LD_i32_1:%[0-9]+]]:b32 = LD_i32 0, 0, 1, 3, 32, -1, 0, 0, [[ADD64ri]], 0
; CHECK-NEXT: [[SETP_i32ri:%[0-9]+]]:b1 = SETP_i32ri [[LD_i32_]], 0, 0
; CHECK-NEXT: CBranch killed [[SETP_i32ri]], %bb.2, 0
; CHECK-NEXT: {{ $}}
@@ -49,15 +49,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: [[PHI1:%[0-9]+]]:b32 = PHI [[LD_i32_1]], %bb.0, [[SREM32rr]], %bb.1
- ; CHECK-NEXT: ST_i32 [[PHI1]], 0, 0, 1, 32, [[ADD64ri]], 0
+ ; CHECK-NEXT: ST_i32 [[PHI1]], 0, 0, 1, 32, 0, [[ADD64ri]], 0, 0
; CHECK-NEXT: Return
bb.0.entry:
successors: %bb.2(0x30000000), %bb.1(0x50000000)
- %5:b32 = LD_i32 0, 0, 101, 3, 32, -1, &test_hoist_param_1, 0 :: (dereferenceable invariant load (s32), addrspace 101)
- %6:b64 = LD_i64 0, 0, 101, 3, 64, -1, &test_hoist_param_0, 0 :: (dereferenceable invariant load (s64), addrspace 101)
+ %5:b32 = LD_i32 0, 0, 101, 3, 32, -1, 0, 0, &test_hoist_param_1, 0 :: (dereferenceable invariant load (s32), addrspace 101)
+ %6:b64 = LD_i64 0, 0, 101, 3, 64, -1, 0, 0, &test_hoist_param_0, 0 :: (dereferenceable invariant load (s64), addrspace 101)
%0:b64 = nuw ADD64ri killed %6, 2
- %1:b32 = LD_i32 0, 0, 1, 3, 32, -1, %0, 0
+ %1:b32 = LD_i32 0, 0, 1, 3, 32, -1, 0, 0, %0, 0
%7:b1 = SETP_i32ri %5, 0, 0
CBranch killed %7, %bb.2, 0
GOTO %bb.1
@@ -75,6 +75,6 @@ body: |
bb.2:
%4:b32 = PHI %1, %bb.0, %3, %bb.1
- ST_i32 %4, 0, 0, 1, 32, %0, 0
+ ST_i32 %4, 0, 0, 1, 32, 0, %0, 0, 0
Return
...
diff --git a/llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll b/llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll
new file mode 100644
index 0000000000000..a6699a779107c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll
@@ -0,0 +1,406 @@
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; Test !mem.cache_hint metadata on llvm.memcpy intrinsic
+; For memcpy:
+; operand_no = 0 applies to destination (store side)
+; operand_no = 1 applies to source (load side)
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1), ptr addrspace(1), i64, i1)
+
+;-----------------------------------------------------------------------------
+; Test memcpy with cache hints on both source and destination
+; Source (operand 1): L1::evict_first, L2::evict_first, L2::128B
+; Dest (operand 0): L1::evict_last, L2::evict_last
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_memcpy_both_hints
+; CHECK-DAG: ld.global.L1::evict_first.L2::evict_first.L2::128B.b
+; CHECK-DAG: st.global.L1::evict_last.L2::evict_last.b
+define void @test_memcpy_both_hints(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !80
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy with cache hint only on source (load side)
+; Source (operand 1): L1::evict_first
+; Dest (operand 0): no hint
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_memcpy_src_hint_only
+; CHECK: ld.global.L1::evict_first.b
+; CHECK: st.global.b
+; CHECK-NOT: st.global.L1
+define void @test_memcpy_src_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !81
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy with cache hint only on destination (store side)
+; Source (operand 1): no hint
+; Dest (operand 0): L2::evict_last
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_memcpy_dest_hint_only
+; CHECK: ld.global.b
+; CHECK-NOT: ld.global.L2
+; CHECK: st.global.L2::evict_last.b
+define void @test_memcpy_dest_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !82
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy with L2::cache_hint on both operands
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_memcpy_l2_cache_hint
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 34343
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 12121
+; CHECK-DAG: ld.global.L2::cache_hint.b
+; CHECK-DAG: st.global.L2::cache_hint.b
+define void @test_memcpy_l2_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !83
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy without cache hints produces plain load/store
+;-----------------------------------------------------------------------------
+
+; CHECK-LABEL: test_memcpy_no_hint
+; CHECK: ld.global.b
+; CHECK: st.global.b
+; CHECK-NOT: L1::evict
+; CHECK-NOT: L2::evict
+; CHECK-NOT: L2::cache_hint
+define void @test_memcpy_no_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false)
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Combined L1 + L2 eviction policies
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_first + L2::evict_last
+; Dest: no hint
+; CHECK-LABEL: test_memcpy_src_l1_l2_combined
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b
+; CHECK: st.global.b
+; CHECK-NOT: st.global.L1
+; CHECK-NOT: st.global.L2
+define void @test_memcpy_src_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !84
+ ret void
+}
+
+; Source: no hint
+; Dest: L1::evict_unchanged + L2::evict_first
+; CHECK-LABEL: test_memcpy_dest_l1_l2_combined
+; CHECK: ld.global.b
+; CHECK-NOT: ld.global.L1
+; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b
+define void @test_memcpy_dest_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !85
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L1 + prefetch combinations
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_last + L2::256B prefetch
+; Dest: L1::no_allocate
+; CHECK-LABEL: test_memcpy_l1_prefetch
+; CHECK-DAG: ld.global.L1::evict_last.L2::256B.b
+; CHECK-DAG: st.global.L1::no_allocate.b
+define void @test_memcpy_l1_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !86
+ ret void
+}
+
+; Source: L2::64B prefetch only
+; Dest: L2::evict_last only
+; CHECK-LABEL: test_memcpy_prefetch_vs_eviction
+; CHECK: ld.global.L2::64B.b
+; CHECK: st.global.L2::evict_last.b
+define void @test_memcpy_prefetch_vs_eviction(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !87
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint combined with other hints
+;-----------------------------------------------------------------------------
+
+; Source: L2::cache_hint + L1::evict_first
+; Dest: no hint
+; CHECK-LABEL: test_memcpy_src_cache_hint_l1
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 55555
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b
+; CHECK: st.global.b
+; CHECK-NOT: st.global.L2::cache_hint
+define void @test_memcpy_src_cache_hint_l1(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !88
+ ret void
+}
+
+; Source: no hint
+; Dest: L2::cache_hint + L1::evict_last + L2::evict_first
+; CHECK-LABEL: test_memcpy_dest_cache_hint_combined
+; CHECK: ld.global.b
+; CHECK-NOT: ld.global.L2::cache_hint
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 66666
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b
+define void @test_memcpy_dest_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !89
+ ret void
+}
+
+; Both operands: L2::cache_hint + L1 eviction + L2 eviction
+; Source: L2::cache_hint(77777) + L1::evict_unchanged + L2::evict_last
+; Dest: L2::cache_hint(88888) + L1::evict_first + L2::evict_first
+; CHECK-LABEL: test_memcpy_both_cache_hint_combined
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 77777
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 88888
+; CHECK-DAG: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b
+; CHECK-DAG: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b
+define void @test_memcpy_both_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !90
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint + prefetch combinations
+;-----------------------------------------------------------------------------
+
+; Source: L2::cache_hint + L2::128B prefetch
+; Dest: L2::cache_hint + L1::evict_last
+; CHECK-LABEL: test_memcpy_cache_hint_prefetch
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 11111
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 22222
+; CHECK-DAG: ld.global.L2::cache_hint.L2::128B.b
+; CHECK-DAG: st.global.L1::evict_last.L2::cache_hint.b
+define void @test_memcpy_cache_hint_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !91
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Asymmetric hint combinations (complex vs simple)
+;-----------------------------------------------------------------------------
+
+; Source: all hints (L1 + L2 eviction + prefetch)
+; Dest: simple L1 hint only
+; CHECK-LABEL: test_memcpy_complex_src_simple_dest
+; CHECK-DAG: ld.global.L1::evict_first.L2::evict_last.L2::64B.b
+; CHECK-DAG: st.global.L1::evict_last.b
+define void @test_memcpy_complex_src_simple_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !92
+ ret void
+}
+
+; Source: simple L2 prefetch only
+; Dest: all hints (L1 + L2 eviction + L2::cache_hint)
+; CHECK-LABEL: test_memcpy_simple_src_complex_dest
+; CHECK: ld.global.L2::256B.b
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 99999
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b
+define void @test_memcpy_simple_src_complex_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !93
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Different L1 eviction policies on src vs dest
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_unchanged
+; Dest: L1::evict_first
+; CHECK-LABEL: test_memcpy_different_l1_policies
+; CHECK-DAG: ld.global.L1::evict_unchanged.b
+; CHECK-DAG: st.global.L1::evict_first.b
+define void @test_memcpy_different_l1_policies(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !94
+ ret void
+}
+
+; Source: L1::no_allocate
+; Dest: L1::evict_unchanged
+; CHECK-LABEL: test_memcpy_no_allocate_vs_unchanged
+; CHECK-DAG: ld.global.L1::no_allocate.b
+; CHECK-DAG: st.global.L1::evict_unchanged.b
+define void @test_memcpy_no_allocate_vs_unchanged(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !95
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; All hints maxed out on both operands
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_first + L2::evict_first + L2::256B + L2::cache_hint
+; Dest: L1::evict_last + L2::evict_last + L2::128B + L2::cache_hint
+; CHECK-LABEL: test_memcpy_all_hints_both
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 12345
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 67890
+; CHECK-DAG: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b
+; CHECK-DAG: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b
+define void @test_memcpy_all_hints_both(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !96
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+; memcpy with both dest and src hints
+!80 = !{i32 0, !180, i32 1, !181}
+; operand 0 (dest/store): L1::evict_last, L2::evict_last
+!180 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last"}
+; operand 1 (src/load): L1::evict_first, L2::evict_first, L2::128B prefetch
+!181 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"128B"}
+
+; memcpy with only source hint (load side)
+!81 = !{i32 1, !182}
+!182 = !{!"nvvm.l1_eviction", !"first"}
+
+; memcpy with only dest hint (store side)
+!82 = !{i32 0, !183}
+!183 = !{!"nvvm.l2_eviction", !"last"}
+
+; memcpy with L2::cache_hint on both operands
+!83 = !{i32 0, !184, i32 1, !185}
+!184 = !{!"nvvm.l2_cache_hint", i64 12121}
+!185 = !{!"nvvm.l2_cache_hint", i64 34343}
+
+; Combined L1 + L2 eviction on source only
+!84 = !{i32 1, !186}
+!186 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last"}
+
+; Combined L1 + L2 eviction on dest only
+!85 = !{i32 0, !187}
+!187 = !{!"nvvm.l1_eviction", !"unchanged", !"nvvm.l2_eviction", !"first"}
+
+; L1 + prefetch on source, L1 on dest
+!86 = !{i32 0, !189, i32 1, !188}
+!188 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
+!189 = !{!"nvvm.l1_eviction", !"no_allocate"}
+
+; Prefetch on source, L2 eviction on dest
+!87 = !{i32 0, !191, i32 1, !190}
+!190 = !{!"nvvm.l2_prefetch_size", !"64B"}
+!191 = !{!"nvvm.l2_eviction", !"last"}
+
+; L2::cache_hint + L1 eviction on source only
+!88 = !{i32 1, !192}
+!192 = !{!"nvvm.l2_cache_hint", i64 55555, !"nvvm.l1_eviction", !"first"}
+
+; L2::cache_hint + L1 + L2 eviction on dest only
+!89 = !{i32 0, !193}
+!193 = !{!"nvvm.l2_cache_hint", i64 66666, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+
+; Both operands: L2::cache_hint + L1 + L2 eviction
+!90 = !{i32 0, !195, i32 1, !194}
+!194 = !{!"nvvm.l2_cache_hint", i64 77777, !"nvvm.l1_eviction", !"unchanged", !"nvvm.l2_eviction", !"last"}
+!195 = !{!"nvvm.l2_cache_hint", i64 88888, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first"}
+
+; L2::cache_hint + prefetch on source, L2::cache_hint + L1 on dest
+!91 = !{i32 0, !197, i32 1, !196}
+!196 = !{!"nvvm.l2_cache_hint", i64 11111, !"nvvm.l2_prefetch_size", !"128B"}
+!197 = !{!"nvvm.l2_cache_hint", i64 22222, !"nvvm.l1_eviction", !"last"}
+
+; Complex source (all non-cache_hint), simple dest
+!92 = !{i32 0, !199, i32 1, !198}
+!198 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"64B"}
+!199 = !{!"nvvm.l1_eviction", !"last"}
+
+; Simple source, complex dest (with cache_hint)
+!93 = !{i32 0, !201, i32 1, !200}
+!200 = !{!"nvvm.l2_prefetch_size", !"256B"}
+!201 = !{!"nvvm.l2_cache_hint", i64 99999, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
+
+; Different L1 policies: unchanged vs first
+!94 = !{i32 0, !203, i32 1, !202}
+!202 = !{!"nvvm.l1_eviction", !"unchanged"}
+!203 = !{!"nvvm.l1_eviction", !"first"}
+
+; Different L1 policies: no_allocate vs unchanged
+!95 = !{i32 0, !205, i32 1, !204}
+!204 = !{!"nvvm.l1_eviction", !"no_allocate"}
+!205 = !{!"nvvm.l1_eviction", !"unchanged"}
+
+; All hints maxed out on both operands
+!96 = !{i32 0, !207, i32 1, !206}
+!206 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
+!207 = !{!"nvvm.l2_cache_hint", i64 67890, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
+
+;-----------------------------------------------------------------------------
+; Large memcpy tests - verify hints propagate to all expanded load/stores
+; LLVM expands memcpy to multiple load/store pairs. Each pair should
+; get the appropriate cache hints from the original memcpy metadata.
+; The expansion may use various sizes (b8, b16, b32, v2, v4, etc.)
+;-----------------------------------------------------------------------------
+
+; 16-byte memcpy: verify hints are applied to expanded loads/stores
+; CHECK-LABEL: test_memcpy_16bytes
+; CHECK: ld.global.L1::evict_first.b
+; CHECK: st.global.L1::evict_last.b
+define void @test_memcpy_16bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 16, i1 false), !mem.cache_hint !97
+ ret void
+}
+
+; 32-byte memcpy: all loads should have L1::evict_unchanged, all stores L2::evict_first
+; CHECK-LABEL: test_memcpy_32bytes
+; CHECK: ld.global.L1::evict_unchanged.b
+; CHECK: st.global.L2::evict_first.b
+define void @test_memcpy_32bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 32, i1 false), !mem.cache_hint !98
+ ret void
+}
+
+; 64-byte memcpy with L2::cache_hint
+; All loads and stores should get the L2::cache_hint with their respective policies
+; CHECK-LABEL: test_memcpy_64bytes_cache_hint
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 11111
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 22222
+; CHECK: ld.global.L2::cache_hint.b
+; CHECK: st.global.L2::cache_hint.b
+define void @test_memcpy_64bytes_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 64, i1 false), !mem.cache_hint !99
+ ret void
+}
+
+; 128-byte memcpy with combined hints
+; Note: Large memcpy (>64 bytes) is expanded to a loop in the backend.
+; Cache hints are not preserved for loop-based expansion.
+; This test verifies the code compiles correctly.
+; CHECK-LABEL: test_memcpy_128bytes_combined
+; CHECK: ld.global.b
+; CHECK: st.global.b
+define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 128, i1 false), !mem.cache_hint !100
+ ret void
+}
+
+; Large memcpy metadata
+!97 = !{i32 0, !209, i32 1, !208}
+!208 = !{!"nvvm.l1_eviction", !"first"}
+!209 = !{!"nvvm.l1_eviction", !"last"}
+
+!98 = !{i32 0, !211, i32 1, !210}
+!210 = !{!"nvvm.l1_eviction", !"unchanged"}
+!211 = !{!"nvvm.l2_eviction", !"first"}
+
+!99 = !{i32 0, !213, i32 1, !212}
+!212 = !{!"nvvm.l2_cache_hint", i64 11111}
+!213 = !{!"nvvm.l2_cache_hint", i64 22222}
+
+!100 = !{i32 0, !215, i32 1, !214}
+!214 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
+!215 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
diff --git a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
index a84b7fcd33836..8e1c7975335c9 100644
--- a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
+++ b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
@@ -77,22 +77,22 @@ constants: []
machineFunctionInfo: {}
body: |
bb.0:
- %0:b32, %1:b32, %2:b32, %3:b32 = LDV_i32_v4 0, 0, 101, 3, 32, -1, &retval0, 0 :: (load (s128), addrspace 101)
+ %0:b32, %1:b32, %2:b32, %3:b32 = LDV_i32_v4 0, 0, 101, 3, 32, -1, 0, 0, &retval0, 0 :: (load (s128), addrspace 101)
; CHECK-NOT: ProxyReg
%4:b32 = ProxyRegB32 killed %0
%5:b32 = ProxyRegB32 killed %1
%6:b32 = ProxyRegB32 killed %2
%7:b32 = ProxyRegB32 killed %3
; CHECK: STV_i32_v4 killed %0, killed %1, killed %2, killed %3
- STV_i32_v4 killed %4, killed %5, killed %6, killed %7, 0, 0, 101, 32, &func_retval0, 0 :: (store (s128), addrspace 101)
+ STV_i32_v4 killed %4, killed %5, killed %6, killed %7, 0, 0, 101, 32, 0, &func_retval0, 0, 0 :: (store (s128), addrspace 101)
- %8:b32 = LD_i32 0, 0, 101, 3, 32, -1, &retval0, 0 :: (load (s32), addrspace 101)
+ %8:b32 = LD_i32 0, 0, 101, 3, 32, -1, 0, 0, &retval0, 0 :: (load (s32), addrspace 101)
; CHECK-NOT: ProxyReg
%9:b32 = ProxyRegB32 killed %8
%10:b32 = ProxyRegB32 killed %9
%11:b32 = ProxyRegB32 killed %10
; CHECK: ST_i32 killed %8
- ST_i32 killed %11, 0, 0, 101, 32, &func_retval0, 0 :: (store (s32), addrspace 101)
+ ST_i32 killed %11, 0, 0, 101, 32, 0, &func_retval0, 0, 0 :: (store (s32), addrspace 101)
Return
...
diff --git a/llvm/test/DebugInfo/NVPTX/inlinedAt_2.mir b/llvm/test/DebugInfo/NVPTX/inlinedAt_2.mir
index 98c8083322747..930dba5044bae 100644
--- a/llvm/test/DebugInfo/NVPTX/inlinedAt_2.mir
+++ b/llvm/test/DebugInfo/NVPTX/inlinedAt_2.mir
@@ -113,7 +113,7 @@ body: |
bb.0.entry:
successors: %bb.1(0x40000000), %bb.2(0x40000000)
- %1:b32 = LD_i32 0, 0, 1, 3, 32, -1, @gg, 0, debug-location !6 :: (dereferenceable load (s32) from @gg, addrspace 1); t2.cu:9:3 @[ t2.cu:18:3 ]
+ %1:b32 = LD_i32 0, 0, 1, 3, 32, -1, 0, @gg, 0, 0, debug-location !6 :: (dereferenceable load (s32) from @gg, addrspace 1); t2.cu:9:3 @[ t2.cu:18:3 ]
%2:b1 = SETP_i32ri %1, 8, 2, debug-location !6; t2.cu:9:3 @[ t2.cu:18:3 ]
CBranch %2, %bb.2, 0, debug-location !6; t2.cu:9:3 @[ t2.cu:18:3 ]
@@ -122,7 +122,7 @@ body: |
successors: %bb.2(0x80000000)
%0:b32 = nuw nsw ADD32ri %1, 1
- ST_i32 %0, 0, 0, 1, 32, @gg, 0, debug-location !11 :: (store (s32) into @gg, addrspace 1); t2.cu:14:3 @[ t2.cu:10:5 @[ t2.cu:18:3 ] ]
+ ST_i32 %0, 0, 0, 1, 32, 0, @gg, 0, 0, debug-location !11 :: (store (s32) into @gg, addrspace 1); t2.cu:14:3 @[ t2.cu:10:5 @[ t2.cu:18:3 ] ]
bb.2._Z3foov.exit:
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index 2a16c749dc567..d984dcfc91aaa 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -239,7 +239,7 @@ static void cloneMemOperands(MachineInstr &DstMI, MachineInstr &SrcMI,
MachineMemOperand *NewMMO = DstMF.getMachineMemOperand(
NewPtrInfo, OldMMO->getFlags(), OldMMO->getMemoryType(),
OldMMO->getBaseAlign(), OldMMO->getAAInfo(), OldMMO->getRanges(),
- OldMMO->getSyncScopeID(), OldMMO->getSuccessOrdering(),
+ nullptr, OldMMO->getSyncScopeID(), OldMMO->getSuccessOrdering(),
OldMMO->getFailureOrdering());
NewMMOs.push_back(NewMMO);
}
diff --git a/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp b/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp
index 992866f48a9ae..8eefc77a90f61 100644
--- a/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp
@@ -53,7 +53,7 @@ TEST_F(AArch64GISelMITest, SimpleAlias) {
// Same for atomics.
auto *LoadAtomicMMO = MF->getMachineMemOperand(
PtrInfo, MachineMemOperand::Flags::MOLoad, S64, Align(8), AAMDNodes(),
- nullptr, SyncScope::System, AtomicOrdering::Acquire);
+ nullptr, nullptr, SyncScope::System, AtomicOrdering::Acquire);
auto AtomicLd1 = B.buildLoad(S64, Addr, *LoadAtomicMMO);
auto AtomicLd2 = B.buildLoad(S64, Base2, *LoadAtomicMMO);
EXPECT_TRUE(
diff --git a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
index 67e5c0964cba4..94e4abdb10e40 100644
--- a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -305,7 +305,7 @@ TEST_F(AArch64GISelMITest, BuildAtomicRMW) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo(),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore, 8, Align(8),
- AAMDNodes(), nullptr, SyncScope::System, AtomicOrdering::Unordered);
+ AAMDNodes(), nullptr, nullptr, SyncScope::System, AtomicOrdering::Unordered);
auto Ptr = B.buildUndef(P0);
B.buildAtomicRMWFAdd(S64, Ptr, Copies[0], *MMO);
>From 94b0bf845e5e94d55803a0a163137c57a1bbb3ff Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 16 Jun 2026 06:28:41 +0000
Subject: [PATCH 02/33] format
---
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 4 +-
llvm/include/llvm/CodeGen/SelectionDAG.h | 45 ++++++++++---------
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 11 +++--
llvm/lib/CodeGen/MachineFunction.cpp | 38 ++++++++--------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 28 ++++++------
.../SelectionDAG/SelectionDAGBuilder.cpp | 8 ++--
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.h | 4 +-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 24 ++++------
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 5 ++-
.../GlobalISel/MachineIRBuilderTest.cpp | 3 +-
10 files changed, 81 insertions(+), 89 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 2616c011d7997..3be9d31bc57c9 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -195,9 +195,7 @@ class GAnyLoad : public GLoadStore {
}
/// Returns the cache hint metadata for this load.
- const MDNode *getMemCacheHint() const {
- return getMMO().getMemCacheHint();
- }
+ const MDNode *getMemCacheHint() const { return getMMO().getMemCacheHint(); }
static bool classof(const MachineInstr *MI) {
switch (MI->getOpcode()) {
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 5e9f00d997dad..2b866d0e5def1 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1522,12 +1522,12 @@ class SelectionDAG {
///
/// This function will set the MOLoad flag on MMOFlags, but you can set it if
/// you want. The MOStore flag must not be set.
- LLVM_ABI SDValue getLoad(
- EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
- MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
- MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr);
+ LLVM_ABI SDValue
+ getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
+ MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
+ MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+ const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr);
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
MachineMemOperand *MMO);
LLVM_ABI SDValue
@@ -1543,20 +1543,21 @@ class SelectionDAG {
LLVM_ABI SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
SDValue Base, SDValue Offset,
ISD::MemIndexedMode AM);
- LLVM_ABI SDValue getLoad(
- ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
- SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
- EVT MemVT, Align Alignment,
- MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr);
- inline SDValue getLoad(
- ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
- SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
- EVT MemVT, MaybeAlign Alignment = MaybeAlign(),
- MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr) {
+ LLVM_ABI SDValue
+ getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
+ const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
+ MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
+ MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+ const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr);
+ inline SDValue
+ getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
+ const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
+ MachinePointerInfo PtrInfo, EVT MemVT,
+ MaybeAlign Alignment = MaybeAlign(),
+ MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
+ const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr) {
// Ensures that codegen never sees a None Alignment.
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo,
@@ -1603,8 +1604,8 @@ class SelectionDAG {
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *MemCacheHint = nullptr) {
return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
- Alignment.value_or(getEVTAlign(SVT)), MMOFlags,
- AAInfo, MemCacheHint);
+ Alignment.value_or(getEVTAlign(SVT)), MMOFlags, AAInfo,
+ MemCacheHint);
}
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 97295f81213fe..5aa2fe39a1bc0 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1436,8 +1436,8 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
Align BaseAlign = getMemOpAlign(LI);
auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
commonAlignment(BaseAlign, Offsets[i]),
- AAInfo, nullptr, nullptr, LI.getSyncScopeID(),
- LI.getOrdering());
+ AAInfo, nullptr, nullptr,
+ LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -3576,8 +3576,7 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
*MF->getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
getMemOpAlign(I), I.getAAMetadata(), nullptr, nullptr,
- I.getSyncScopeID(), I.getSuccessOrdering(),
- I.getFailureOrdering()));
+ I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
return true;
}
@@ -3672,8 +3671,8 @@ bool IRTranslator::translateAtomicRMW(const User &U,
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, MRI->getType(Val), getMemOpAlign(I),
- I.getAAMetadata(), nullptr, nullptr, I.getSyncScopeID(),
- I.getOrdering()));
+ I.getAAMetadata(), nullptr, nullptr,
+ I.getSyncScopeID(), I.getOrdering()));
return true;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index b0bed8e3a102c..837b7adc8165b 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -596,20 +596,18 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
"Unexpected an unknown size to be represented using "
"LocationSize::beforeOrAfter()");
- return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
- MMO->getMemCacheHint());
+ return new (Allocator) MachineMemOperand(
+ PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(), AAMDNodes(), nullptr,
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering(), MMO->getMemCacheHint());
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
- return new (Allocator)
- MachineMemOperand(PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
- AAMDNodes(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
- MMO->getMemCacheHint());
+ return new (Allocator) MachineMemOperand(
+ PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(), AAMDNodes(), nullptr,
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering(), MMO->getMemCacheHint());
}
MachineMemOperand *
@@ -625,11 +623,11 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// Do not preserve ranges, since we don't necessarily know what the high bits
// are anymore.
- return new (Allocator) MachineMemOperand(
- PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
- MMO->getAAInfo(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
- MMO->getMemCacheHint());
+ return new (Allocator)
+ MachineMemOperand(PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty,
+ Alignment, MMO->getAAInfo(), nullptr,
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering(), MMO->getMemCacheHint());
}
MachineMemOperand *
@@ -648,11 +646,11 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand::Flags Flags) {
- return new (Allocator) MachineMemOperand(
- MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
- MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering(),
- MMO->getMemCacheHint());
+ return new (Allocator)
+ MachineMemOperand(MMO->getPointerInfo(), Flags, MMO->getSize(),
+ MMO->getBaseAlign(), MMO->getAAInfo(), MMO->getRanges(),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering(), MMO->getMemCacheHint());
}
MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfo(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 30c6e1b53c921..fa9ab3c6939b5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9461,7 +9461,8 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
Store = DAG.getStore(
Chain, dl, Value,
DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
- DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags, NewAAInfo, DstMemCacheHint);
+ DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags, NewAAInfo,
+ DstMemCacheHint);
OutChains.push_back(Store);
}
}
@@ -9487,13 +9488,15 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
ISD::EXTLOAD, dl, NVT, Chain,
DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),
SrcPtrInfo.getWithOffset(SrcOff), VT,
- commonAlignment(SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo, SrcMemCacheHint);
+ commonAlignment(SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo,
+ SrcMemCacheHint);
OutLoadChains.push_back(Value.getValue(1));
Store = DAG.getTruncStore(
Chain, dl, Value,
DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
- DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags, NewAAInfo, DstMemCacheHint);
+ DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags, NewAAInfo,
+ DstMemCacheHint);
OutStoreChains.push_back(Store);
}
SrcOff += VTSize;
@@ -10579,14 +10582,11 @@ static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
return Info;
}
-SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
- EVT VT, const SDLoc &dl, SDValue Chain,
- SDValue Ptr, SDValue Offset,
- MachinePointerInfo PtrInfo, EVT MemVT,
- Align Alignment,
- MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo, const MDNode *Ranges,
- const MDNode *MemCacheHint) {
+SDValue SelectionDAG::getLoad(
+ ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
+ SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
+ EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags,
+ const AAMDNodes &AAInfo, const MDNode *Ranges, const MDNode *MemCacheHint) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
@@ -10809,9 +10809,9 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo, nullptr,
- MemCacheHint);
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment,
+ AAInfo, nullptr, MemCacheHint);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7f41890d58584..934def97381d6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5224,10 +5224,10 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
MachineFunction &MF = DAG.getMachineFunction();
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
- MemVT.getStoreSize(), I.getAlign(), AAMDNodes(),
- nullptr, nullptr, SSID, SuccessOrdering, FailureOrdering);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
+ I.getAlign(), AAMDNodes(), nullptr, nullptr, SSID, SuccessOrdering,
+ FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
dl, MemVT, VTs, InChain,
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index f977927b4d5ca..43b54eb421ef9 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -47,8 +47,8 @@ class NVPTXInstPrinter : public MCInstPrinter {
void printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
const MCSubtargetInfo &STI, raw_ostream &O,
StringRef Modifier = {});
- void printCachePolicy(const MCInst *MI, int OpNum,
- const MCSubtargetInfo &STI, raw_ostream &O);
+ void printCachePolicy(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
+ raw_ostream &O);
void printMmaCode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
void printMemOperand(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index d0ad28d1cd393..b9acdc1952501 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1149,8 +1149,7 @@ static void parseMemCacheHintStringValue(LLVMContext &Ctx, StringRef Key,
ParseFn Parse) {
const auto *Val = dyn_cast<MDString>(Value);
if (!Val) {
- emitInvalidMemCacheHint(Ctx,
- Twine("'") + Key + "' expects a string value");
+ emitInvalidMemCacheHint(Ctx, Twine("'") + Key + "' expects a string value");
return;
}
@@ -1158,8 +1157,8 @@ static void parseMemCacheHintStringValue(LLVMContext &Ctx, StringRef Key,
if (auto Parsed = Parse(ValStr))
Result = *Parsed;
else
- emitInvalidMemCacheHint(Ctx, Twine("unknown value '") + ValStr +
- "' for '" + Key + "'");
+ emitInvalidMemCacheHint(Ctx, Twine("unknown value '") + ValStr + "' for '" +
+ Key + "'");
}
static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
@@ -1177,10 +1176,8 @@ static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
llvm_unreachable("Unexpected L2 prefetch hint");
}
-std::pair<unsigned, SDValue>
-NVPTXDAGToDAGISel::getMemCacheHintOperands(const MemSDNode *N,
- unsigned CodeAddrSpace,
- const SDLoc &DL) {
+std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
+ const MemSDNode *N, unsigned CodeAddrSpace, const SDLoc &DL) {
LLVMContext &Ctx = *CurDAG->getContext();
const MDNode *Node = N->getMemCacheHint();
SDValue PolicyReg = CurDAG->getRegister(NVPTX::NoRegister, MVT::i64);
@@ -1203,22 +1200,20 @@ NVPTXDAGToDAGISel::getMemCacheHintOperands(const MemSDNode *N,
if (KeyStr == "nvvm.l1_eviction") {
if (Subtarget->hasL1EvictionHint())
- parseMemCacheHintStringValue(Ctx, KeyStr, Value, L1,
- parseL1Eviction);
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, L1, parseL1Eviction);
continue;
}
if (KeyStr == "nvvm.l2_eviction") {
if (Subtarget->hasL2EvictionHint())
- parseMemCacheHintStringValue(Ctx, KeyStr, Value, L2,
- parseL2Eviction);
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, L2, parseL2Eviction);
continue;
}
if (KeyStr == "nvvm.l2_prefetch_size") {
NVPTX::L2Prefetch ParsedPrefetch = NVPTX::L2Prefetch::None;
parseMemCacheHintStringValue(Ctx, KeyStr, Value, ParsedPrefetch,
- parseL2Prefetch);
+ parseL2Prefetch);
if (isL2PrefetchSupported(*Subtarget, ParsedPrefetch))
Prefetch = ParsedPrefetch;
continue;
@@ -1244,8 +1239,7 @@ NVPTXDAGToDAGISel::getMemCacheHintOperands(const MemSDNode *N,
if (CachePolicy) {
SDValue PolicyConst = CurDAG->getTargetConstant(*CachePolicy, DL, MVT::i64);
PolicyReg = SDValue(
- CurDAG->getMachineNode(NVPTX::MOV_B64_i, DL, MVT::i64, PolicyConst),
- 0);
+ CurDAG->getMachineNode(NVPTX::MOV_B64_i, DL, MVT::i64, PolicyConst), 0);
Bitfield::set<NVPTX::L2CacheHintBit>(EvictionAndPrefetchHint, true);
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index 931f7e4d86df2..fef115fcea6b0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -110,8 +110,9 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
// dropped. If L2::cache_hint is active, returns the hint with
// L2CacheHintBit set and a register containing the 64-bit cache policy
// value. Otherwise returns NOREG for the policy operand.
- std::pair<unsigned, SDValue> getMemCacheHintOperands(
- const MemSDNode *N, unsigned CodeAddrSpace, const SDLoc &DL);
+ std::pair<unsigned, SDValue> getMemCacheHintOperands(const MemSDNode *N,
+ unsigned CodeAddrSpace,
+ const SDLoc &DL);
// Returns the Memory Order and Scope that the PTX memory instruction should
// use, and inserts appropriate fence instruction before the memory
diff --git a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
index 94e4abdb10e40..1fc20f6f238b4 100644
--- a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -305,7 +305,8 @@ TEST_F(AArch64GISelMITest, BuildAtomicRMW) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo(),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore, 8, Align(8),
- AAMDNodes(), nullptr, nullptr, SyncScope::System, AtomicOrdering::Unordered);
+ AAMDNodes(), nullptr, nullptr, SyncScope::System,
+ AtomicOrdering::Unordered);
auto Ptr = B.buildUndef(P0);
B.buildAtomicRMWFAdd(S64, Ptr, Copies[0], *MMO);
>From 150a5b094a92bd56bccf097c982b8ca1daef404c Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 16 Jun 2026 06:58:31 +0000
Subject: [PATCH 03/33] add masked load tests
---
...cache-hint.ll => intrinsics-cache-hint.ll} | 138 +++++++++++-------
1 file changed, 88 insertions(+), 50 deletions(-)
rename llvm/test/CodeGen/NVPTX/{memcpy-cache-hint.ll => intrinsics-cache-hint.ll} (91%)
diff --git a/llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll b/llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll
similarity index 91%
rename from llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll
rename to llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll
index a6699a779107c..9f05c0d1ce9d8 100644
--- a/llvm/test/CodeGen/NVPTX/memcpy-cache-hint.ll
+++ b/llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll
@@ -1,12 +1,15 @@
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
-; Test !mem.cache_hint metadata on llvm.memcpy intrinsic
+; Test !mem.cache_hint metadata on LLVM memory intrinsics.
+;
; For memcpy:
; operand_no = 0 applies to destination (store side)
; operand_no = 1 applies to source (load side)
declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1), ptr addrspace(1), i64, i1)
+declare <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1), <2 x i1>, <2 x i16>)
+declare <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1), <4 x i1>, <4 x i32>)
;-----------------------------------------------------------------------------
; Test memcpy with cache hints on both source and destination
@@ -254,6 +257,83 @@ define void @test_memcpy_all_hints_both(ptr addrspace(1) %dest, ptr addrspace(1)
ret void
}
+;-----------------------------------------------------------------------------
+; TODO: Preserve cache hints on llvm.masked.load.
+; Masked load pointer operand is operand 0. SelectionDAGBuilder::visitMaskedLoad
+; does not currently thread !mem.cache_hint into the MachineMemOperand, so the
+; generated loads are plain today.
+;-----------------------------------------------------------------------------
+
+; TODO: This should eventually lower to ld.global.L1::evict_first.b32.
+; CHECK-LABEL: test_masked_load_l1_hint_v2i16
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK: ld.global.b32
+define <2 x i16> @test_masked_load_l1_hint_v2i16(ptr addrspace(1) %p) {
+ %v = call <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1) align 4 %p, <2 x i1> <i1 true, i1 false>, <2 x i16> poison), !mem.cache_hint !101
+ ret <2 x i16> %v
+}
+
+; TODO: This should eventually lower to ld.global.L2::cache_hint.v4.b32.
+; CHECK-LABEL: test_masked_load_l2_cache_policy_v4i32
+; CHECK-NOT: mov.b64
+; CHECK-NOT: L1::
+; CHECK-NOT: L2::
+; CHECK: ld.global.v4.b32
+define <4 x i32> @test_masked_load_l2_cache_policy_v4i32(ptr addrspace(1) %p) {
+ %v = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 %p, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> poison), !mem.cache_hint !102
+ ret <4 x i32> %v
+}
+
+;-----------------------------------------------------------------------------
+; Large memcpy tests - verify hints propagate to all expanded load/stores
+; LLVM expands memcpy to multiple load/store pairs. Each pair should
+; get the appropriate cache hints from the original memcpy metadata.
+; The expansion may use various sizes (b8, b16, b32, v2, v4, etc.)
+;-----------------------------------------------------------------------------
+
+; 16-byte memcpy: verify hints are applied to expanded loads/stores
+; CHECK-LABEL: test_memcpy_16bytes
+; CHECK: ld.global.L1::evict_first.b
+; CHECK: st.global.L1::evict_last.b
+define void @test_memcpy_16bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 16, i1 false), !mem.cache_hint !97
+ ret void
+}
+
+; 32-byte memcpy: all loads should have L1::evict_unchanged, all stores L2::evict_first
+; CHECK-LABEL: test_memcpy_32bytes
+; CHECK: ld.global.L1::evict_unchanged.b
+; CHECK: st.global.L2::evict_first.b
+define void @test_memcpy_32bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 32, i1 false), !mem.cache_hint !98
+ ret void
+}
+
+; 64-byte memcpy with L2::cache_hint
+; All loads and stores should get the L2::cache_hint with their respective policies
+; CHECK-LABEL: test_memcpy_64bytes_cache_hint
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 11111
+; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 22222
+; CHECK: ld.global.L2::cache_hint.b
+; CHECK: st.global.L2::cache_hint.b
+define void @test_memcpy_64bytes_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 64, i1 false), !mem.cache_hint !99
+ ret void
+}
+
+; 128-byte memcpy with combined hints
+; Note: Large memcpy (>64 bytes) is expanded to a loop in the backend.
+; Cache hints are not preserved for loop-based expansion.
+; This test verifies the code compiles correctly.
+; CHECK-LABEL: test_memcpy_128bytes_combined
+; CHECK: ld.global.b
+; CHECK: st.global.b
+define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 128, i1 false), !mem.cache_hint !100
+ ret void
+}
+
;-----------------------------------------------------------------------------
; Metadata definitions
;-----------------------------------------------------------------------------
@@ -339,55 +419,6 @@ define void @test_memcpy_all_hints_both(ptr addrspace(1) %dest, ptr addrspace(1)
!206 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
!207 = !{!"nvvm.l2_cache_hint", i64 67890, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
-;-----------------------------------------------------------------------------
-; Large memcpy tests - verify hints propagate to all expanded load/stores
-; LLVM expands memcpy to multiple load/store pairs. Each pair should
-; get the appropriate cache hints from the original memcpy metadata.
-; The expansion may use various sizes (b8, b16, b32, v2, v4, etc.)
-;-----------------------------------------------------------------------------
-
-; 16-byte memcpy: verify hints are applied to expanded loads/stores
-; CHECK-LABEL: test_memcpy_16bytes
-; CHECK: ld.global.L1::evict_first.b
-; CHECK: st.global.L1::evict_last.b
-define void @test_memcpy_16bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 16, i1 false), !mem.cache_hint !97
- ret void
-}
-
-; 32-byte memcpy: all loads should have L1::evict_unchanged, all stores L2::evict_first
-; CHECK-LABEL: test_memcpy_32bytes
-; CHECK: ld.global.L1::evict_unchanged.b
-; CHECK: st.global.L2::evict_first.b
-define void @test_memcpy_32bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 32, i1 false), !mem.cache_hint !98
- ret void
-}
-
-; 64-byte memcpy with L2::cache_hint
-; All loads and stores should get the L2::cache_hint with their respective policies
-; CHECK-LABEL: test_memcpy_64bytes_cache_hint
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 11111
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 22222
-; CHECK: ld.global.L2::cache_hint.b
-; CHECK: st.global.L2::cache_hint.b
-define void @test_memcpy_64bytes_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 64, i1 false), !mem.cache_hint !99
- ret void
-}
-
-; 128-byte memcpy with combined hints
-; Note: Large memcpy (>64 bytes) is expanded to a loop in the backend.
-; Cache hints are not preserved for loop-based expansion.
-; This test verifies the code compiles correctly.
-; CHECK-LABEL: test_memcpy_128bytes_combined
-; CHECK: ld.global.b
-; CHECK: st.global.b
-define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 128, i1 false), !mem.cache_hint !100
- ret void
-}
-
; Large memcpy metadata
!97 = !{i32 0, !209, i32 1, !208}
!208 = !{!"nvvm.l1_eviction", !"first"}
@@ -404,3 +435,10 @@ define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace
!100 = !{i32 0, !215, i32 1, !214}
!214 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
!215 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+
+; Masked load metadata
+!101 = !{i32 0, !216}
+!216 = !{!"nvvm.l1_eviction", !"first"}
+
+!102 = !{i32 0, !217}
+!217 = !{!"nvvm.l2_cache_hint", i64 24680}
>From 9061f04dfcc9979807ed97e7d4d0fb3f42f493a3 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 16 Jun 2026 17:02:22 +0000
Subject: [PATCH 04/33] error -> warning
---
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 4 +-
llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll | 46 +++++++++----------
2 files changed, 26 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index b9acdc1952501..d74e273e22cf1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -22,6 +22,7 @@
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
@@ -1114,7 +1115,8 @@ bool NVPTXDAGToDAGISel::SelectADDR(SDValue Addr, SDValue &Base,
}
static void emitInvalidMemCacheHint(LLVMContext &Ctx, const Twine &Msg) {
- Ctx.emitError(Twine("invalid NVPTX !mem.cache_hint metadata: ") + Msg);
+ Ctx.diagnose(DiagnosticInfoGeneric(
+ Twine("invalid NVPTX !mem.cache_hint metadata: ") + Msg, DS_Warning));
}
static std::optional<NVPTX::L1Eviction> parseL1Eviction(StringRef Str) {
diff --git a/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll b/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
index 246a819141275..e0bfe79b190c4 100644
--- a/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
+++ b/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
@@ -1,19 +1,19 @@
; RUN: split-file %s %t
-; RUN: not llc < %t/bad-empty.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-EMPTY
-; RUN: not llc < %t/bad-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-KEY
-; RUN: not llc < %t/bad-other-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-OTHER-KEY
-; RUN: not llc < %t/bad-l1-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-TYPE
-; RUN: not llc < %t/bad-l1-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-VALUE
-; RUN: not llc < %t/bad-l2-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-TYPE
-; RUN: not llc < %t/bad-l2-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-VALUE
-; RUN: not llc < %t/bad-prefetch-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-TYPE
-; RUN: not llc < %t/bad-prefetch-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-VALUE
-; RUN: not llc < %t/bad-policy.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-POLICY
+; RUN: llc < %t/bad-empty.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-EMPTY
+; RUN: llc < %t/bad-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-KEY
+; RUN: llc < %t/bad-other-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-OTHER-KEY
+; RUN: llc < %t/bad-l1-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-TYPE
+; RUN: llc < %t/bad-l1-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-VALUE
+; RUN: llc < %t/bad-l2-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-TYPE
+; RUN: llc < %t/bad-l2-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-VALUE
+; RUN: llc < %t/bad-prefetch-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-TYPE
+; RUN: llc < %t/bad-prefetch-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-VALUE
+; RUN: llc < %t/bad-policy.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-POLICY
;--- bad-empty.ll
-; Test with empty hint node - should produce an empty-node diagnostic.
-; BAD-EMPTY: invalid NVPTX !mem.cache_hint metadata: empty hint node
+; Test with empty hint node - should produce an empty-node warning.
+; BAD-EMPTY: warning: invalid NVPTX !mem.cache_hint metadata: empty hint node
define i32 @bad_empty_hint_node(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -24,8 +24,8 @@ define i32 @bad_empty_hint_node(ptr addrspace(1) %p) {
;--- bad-key.ll
-; Test with misspelled NVPTX key - should produce an unknown-key diagnostic.
-; BAD-KEY: invalid NVPTX !mem.cache_hint metadata: unknown key 'nvvm.l1_evict'
+; Test with misspelled NVPTX key - should produce an unknown-key warning.
+; BAD-KEY: warning: invalid NVPTX !mem.cache_hint metadata: unknown key 'nvvm.l1_evict'
define i32 @bad_nvvm_key(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -36,8 +36,8 @@ define i32 @bad_nvvm_key(ptr addrspace(1) %p) {
;--- bad-other-key.ll
-; Test with a non-NVPTX key - should produce an unknown-key diagnostic.
-; BAD-OTHER-KEY: invalid NVPTX !mem.cache_hint metadata: unknown key 'some.target_hint'
+; Test with a non-NVPTX key - should produce an unknown-key warning.
+; BAD-OTHER-KEY: warning: invalid NVPTX !mem.cache_hint metadata: unknown key 'some.target_hint'
define i32 @bad_other_key(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -49,7 +49,7 @@ define i32 @bad_other_key(ptr addrspace(1) %p) {
;--- bad-l1-type.ll
; nvvm.l1_eviction expects a string value.
-; BAD-L1-TYPE: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l1_eviction' expects a string value
+; BAD-L1-TYPE: warning: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l1_eviction' expects a string value
define i32 @bad_l1_type(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -61,7 +61,7 @@ define i32 @bad_l1_type(ptr addrspace(1) %p) {
;--- bad-l1-value.ll
; nvvm.l1_eviction accepts only known PTX eviction values.
-; BAD-L1-VALUE: invalid NVPTX !mem.cache_hint metadata: unknown value 'middle' for 'nvvm.l1_eviction'
+; BAD-L1-VALUE: warning: invalid NVPTX !mem.cache_hint metadata: unknown value 'middle' for 'nvvm.l1_eviction'
define i32 @bad_l1_value(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -73,7 +73,7 @@ define i32 @bad_l1_value(ptr addrspace(1) %p) {
;--- bad-l2-type.ll
; nvvm.l2_eviction expects a string value.
-; BAD-L2-TYPE: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_eviction' expects a string value
+; BAD-L2-TYPE: warning: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_eviction' expects a string value
define i32 @bad_l2_type(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -85,7 +85,7 @@ define i32 @bad_l2_type(ptr addrspace(1) %p) {
;--- bad-l2-value.ll
; nvvm.l2_eviction accepts only known PTX eviction values.
-; BAD-L2-VALUE: invalid NVPTX !mem.cache_hint metadata: unknown value 'middle' for 'nvvm.l2_eviction'
+; BAD-L2-VALUE: warning: invalid NVPTX !mem.cache_hint metadata: unknown value 'middle' for 'nvvm.l2_eviction'
define i32 @bad_l2_value(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -97,7 +97,7 @@ define i32 @bad_l2_value(ptr addrspace(1) %p) {
;--- bad-prefetch-type.ll
; nvvm.l2_prefetch_size expects a string value.
-; BAD-PREFETCH-TYPE: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_prefetch_size' expects a string value
+; BAD-PREFETCH-TYPE: warning: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_prefetch_size' expects a string value
define i32 @bad_prefetch_type(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -109,7 +109,7 @@ define i32 @bad_prefetch_type(ptr addrspace(1) %p) {
;--- bad-prefetch-value.ll
; nvvm.l2_prefetch_size accepts only supported PTX prefetch sizes.
-; BAD-PREFETCH-VALUE: invalid NVPTX !mem.cache_hint metadata: unknown value '32B' for 'nvvm.l2_prefetch_size'
+; BAD-PREFETCH-VALUE: warning: invalid NVPTX !mem.cache_hint metadata: unknown value '32B' for 'nvvm.l2_prefetch_size'
define i32 @bad_prefetch_value(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
@@ -121,7 +121,7 @@ define i32 @bad_prefetch_value(ptr addrspace(1) %p) {
;--- bad-policy.ll
; nvvm.l2_cache_hint expects an integer cache-policy value.
-; BAD-POLICY: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_cache_hint' expects an integer value
+; BAD-POLICY: warning: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_cache_hint' expects an integer value
define i32 @bad_cache_policy_value(ptr addrspace(1) %p) {
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
>From 6e7b3e9e9fca8e3ebff4bcded4608c20298208d6 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 16 Jun 2026 17:13:28 +0000
Subject: [PATCH 05/33] move mem cache hint metadata to go after ranges in
parameter order
---
llvm/include/llvm/CodeGen/MachineMemOperand.h | 8 +++---
llvm/lib/CodeGen/MachineFunction.cpp | 28 +++++++++----------
llvm/lib/CodeGen/MachineOperand.cpp | 18 ++++++------
3 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 809fa5a4bc28e..d548e5248c1ff 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -194,18 +194,18 @@ class MachineMemOperand {
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LocationSize TS,
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr,
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic,
- const MDNode *MemCacheHint = nullptr);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
LLVM_ABI
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
const AAMDNodes &AAInfo = AAMDNodes(),
const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr,
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic,
- const MDNode *MemCacheHint = nullptr);
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 837b7adc8165b..376b565da77ac 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -574,8 +574,8 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
"Unexpected an unknown size to be represented using "
"LocationSize::beforeOrAfter()");
return new (Allocator)
- MachineMemOperand(PtrInfo, F, Size, BaseAlignment, AAInfo, Ranges, SSID,
- Ordering, FailureOrdering, MemCacheHint);
+ MachineMemOperand(PtrInfo, F, Size, BaseAlignment, AAInfo, Ranges,
+ MemCacheHint, SSID, Ordering, FailureOrdering);
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
@@ -584,8 +584,8 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
const MDNode *MemCacheHint, SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges, SSID,
- Ordering, FailureOrdering, MemCacheHint);
+ MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges,
+ MemCacheHint, SSID, Ordering, FailureOrdering);
}
MachineMemOperand *
@@ -598,16 +598,16 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(), AAMDNodes(), nullptr,
- MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering(), MMO->getMemCacheHint());
+ MMO->getMemCacheHint(), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(), AAMDNodes(), nullptr,
- MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering(), MMO->getMemCacheHint());
+ MMO->getMemCacheHint(), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *
@@ -626,8 +626,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator)
MachineMemOperand(PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty,
Alignment, MMO->getAAInfo(), nullptr,
- MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering(), MMO->getMemCacheHint());
+ MMO->getMemCacheHint(), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *
@@ -639,8 +639,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(), AAInfo,
- MMO->getRanges(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering(), MMO->getMemCacheHint());
+ MMO->getRanges(), MMO->getMemCacheHint(), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *
@@ -649,8 +649,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator)
MachineMemOperand(MMO->getPointerInfo(), Flags, MMO->getSize(),
MMO->getBaseAlign(), MMO->getAAInfo(), MMO->getRanges(),
- MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering(), MMO->getMemCacheHint());
+ MMO->getMemCacheHint(), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfo(
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 712e2745b4131..e04afe18ee262 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1171,10 +1171,11 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
LLT type, Align a, const AAMDNodes &AAInfo,
- const MDNode *Ranges, SyncScope::ID SSID,
+ const MDNode *Ranges,
+ const MDNode *MemCacheHint,
+ SyncScope::ID SSID,
AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering,
- const MDNode *MemCacheHint)
+ AtomicOrdering FailureOrdering)
: PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a),
AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {
assert((PtrInfo.V.isNull() || isa<const PseudoSourceValue *>(PtrInfo.V) ||
@@ -1193,18 +1194,19 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
LocationSize TS, Align BaseAlignment,
const AAMDNodes &AAInfo,
- const MDNode *Ranges, SyncScope::ID SSID,
+ const MDNode *Ranges,
+ const MDNode *MemCacheHint,
+ SyncScope::ID SSID,
AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering,
- const MDNode *MemCacheHint)
+ AtomicOrdering FailureOrdering)
: MachineMemOperand(
ptrinfo, F,
!TS.isPrecise() ? LLT()
: TS.isScalable()
? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
: LLT::scalar(8 * TS.getValue().getKnownMinValue()),
- BaseAlignment, AAInfo, Ranges, SSID, Ordering, FailureOrdering,
- MemCacheHint) {}
+ BaseAlignment, AAInfo, Ranges, MemCacheHint, SSID, Ordering,
+ FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
>From 933b7d8d9a11db6d55eda9f386d63d111d54d56d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 16 Jun 2026 17:13:47 +0000
Subject: [PATCH 06/33] format
---
llvm/lib/CodeGen/MachineFunction.cpp | 17 ++++++++---------
llvm/lib/CodeGen/MachineOperand.cpp | 23 ++++++++---------------
2 files changed, 16 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 376b565da77ac..e68481e548b9d 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -598,16 +598,16 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(), AAMDNodes(), nullptr,
- MMO->getMemCacheHint(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getMemCacheHint(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(), AAMDNodes(), nullptr,
- MMO->getMemCacheHint(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getMemCacheHint(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
}
MachineMemOperand *
@@ -623,11 +623,10 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// Do not preserve ranges, since we don't necessarily know what the high bits
// are anymore.
- return new (Allocator)
- MachineMemOperand(PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty,
- Alignment, MMO->getAAInfo(), nullptr,
- MMO->getMemCacheHint(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ return new (Allocator) MachineMemOperand(
+ PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
+ MMO->getAAInfo(), nullptr, MMO->getMemCacheHint(), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
MachineMemOperand *
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index e04afe18ee262..3ea77d36a89ec 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1169,13 +1169,10 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
return MachinePointerInfo(MF.getDataLayout().getAllocaAddrSpace());
}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
- LLT type, Align a, const AAMDNodes &AAInfo,
- const MDNode *Ranges,
- const MDNode *MemCacheHint,
- SyncScope::ID SSID,
- AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
+MachineMemOperand::MachineMemOperand(
+ MachinePointerInfo ptrinfo, Flags f, LLT type, Align a,
+ const AAMDNodes &AAInfo, const MDNode *Ranges, const MDNode *MemCacheHint,
+ SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering)
: PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a),
AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {
assert((PtrInfo.V.isNull() || isa<const PseudoSourceValue *>(PtrInfo.V) ||
@@ -1191,14 +1188,10 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
-MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags F,
- LocationSize TS, Align BaseAlignment,
- const AAMDNodes &AAInfo,
- const MDNode *Ranges,
- const MDNode *MemCacheHint,
- SyncScope::ID SSID,
- AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering)
+MachineMemOperand::MachineMemOperand(
+ MachinePointerInfo ptrinfo, Flags F, LocationSize TS, Align BaseAlignment,
+ const AAMDNodes &AAInfo, const MDNode *Ranges, const MDNode *MemCacheHint,
+ SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering)
: MachineMemOperand(
ptrinfo, F,
!TS.isPrecise() ? LLT()
>From 206709249ba6d7db08fdd388a22381134b298ed3 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 15:30:11 +0000
Subject: [PATCH 07/33] use getNamedOperandIdx
---
llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp | 31 +++++++++++---------
llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp | 1 +
2 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp b/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
index eeeee076b3874..447a080739e8c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
@@ -96,20 +96,23 @@ static bool eliminateMove(MachineInstr &Mov, const MachineRegisterInfo &MRI,
const MachineOperand *ParamSymbol = Mov.uses().begin();
assert(ParamSymbol->isSymbol());
- constexpr unsigned LDInstBasePtrOpIdx = 6;
- constexpr unsigned LDInstAddrSpaceOpIdx = 2;
- constexpr unsigned LDInstEvictionAndPrefetchHintOpIdx = 8;
- constexpr unsigned LDInstCachePolicyOpIdx = 9;
- for (auto *LI : LoadInsts) {
- (LI->uses().begin() + LDInstBasePtrOpIdx)
- ->ChangeToES(ParamSymbol->getSymbolName());
- (LI->uses().begin() + LDInstAddrSpaceOpIdx)
- ->ChangeToImmediate(NVPTX::AddressSpace::DeviceParam);
- // PTX cache modifiers are not allowed on ld.param.
- (LI->uses().begin() + LDInstEvictionAndPrefetchHintOpIdx)
- ->ChangeToImmediate(0);
- (LI->uses().begin() + LDInstCachePolicyOpIdx)
- ->ChangeToRegister(NVPTX::NoRegister, false);
+ for (MachineInstr *LI : LoadInsts) {
+ unsigned Opc = LI->getOpcode();
+ int Idx = getNamedOperandIdx(Opc, NVPTX::OpName::addr);
+ assert(Idx != -1 && "no addr operand");
+ LI->getOperand(Idx).ChangeToES(ParamSymbol->getSymbolName());
+
+ Idx = getNamedOperandIdx(Opc, NVPTX::OpName::addsp);
+ assert(Idx != -1 && "no addsp operand");
+ LI->getOperand(Idx).ChangeToImmediate(NVPTX::AddressSpace::DeviceParam);
+ // PTX cache hints and policy are not allowed on ld.param
+ Idx = getNamedOperandIdx(Opc, NVPTX::OpName::evictionAndPrefetchHint);
+ assert(Idx != -1 && "no evictionAndPrefetchHint operand");
+ LI->getOperand(Idx).ChangeToImmediate(0);
+
+ Idx = getNamedOperandIdx(Opc, NVPTX::OpName::policy);
+ assert(Idx != -1 && "no policy operand");
+ LI->getOperand(Idx).ChangeToRegister(NVPTX::NoRegister, false);
}
return true;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index d9900e98fe694..2c3f27c40ab37 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -20,6 +20,7 @@
using namespace llvm;
#define GET_INSTRINFO_CTOR_DTOR
+#define GET_INSTRINFO_NAMED_OPS
#include "NVPTXGenInstrInfo.inc"
// Pin the vtable to this file.
>From bebc6705386736545977a1cd661fc17eab34e5b9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 16:16:35 +0000
Subject: [PATCH 08/33] remove auto and forward RHS
---
llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index fe1becb121bb1..014c8bc286e51 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -401,8 +401,7 @@ void NVPTXInstPrinter::printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
bool IsCacheHintMode = NVPTX::isL2CacheHintMode(Hint);
if (Modifier == "l1") {
- auto L1 = NVPTX::decodeL1Eviction(Hint);
- switch (L1) {
+ switch (NVPTX::decodeL1Eviction(Hint)) {
case NVPTX::L1Eviction::Normal:
return;
case NVPTX::L1Eviction::Unchanged:
@@ -419,8 +418,7 @@ void NVPTXInstPrinter::printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
return;
}
} else if (Modifier == "l2") {
- auto L2 = NVPTX::decodeL2Eviction(Hint);
- switch (L2) {
+ switch (NVPTX::decodeL2Eviction(Hint)) {
case NVPTX::L2Eviction::Normal:
break;
case NVPTX::L2Eviction::First:
@@ -434,8 +432,7 @@ void NVPTXInstPrinter::printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
O << ".L2::cache_hint";
return;
} else if (Modifier == "prefetch") {
- auto Prefetch = NVPTX::decodeL2Prefetch(Hint);
- switch (Prefetch) {
+ switch (NVPTX::decodeL2Prefetch(Hint)) {
case NVPTX::L2Prefetch::None:
return;
case NVPTX::L2Prefetch::Bytes64:
>From 7319180ff59d51270cd302aa55f6c36840b18add Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 18:43:28 +0000
Subject: [PATCH 09/33] group MMO metadata
---
llvm/include/llvm/CodeGen/MachineFunction.h | 26 ++++-----
llvm/include/llvm/CodeGen/MachineMemOperand.h | 38 ++++++++-----
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 34 ++++++------
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 7 +--
llvm/lib/CodeGen/MachineFunction.cpp | 53 +++++++++++--------
llvm/lib/CodeGen/MachineOperand.cpp | 30 ++++++-----
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 5 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 37 +++++++------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 17 +++---
.../SelectionDAG/SelectionDAGBuilder.cpp | 34 ++++++------
.../Target/Hexagon/HexagonISelLowering.cpp | 5 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 34 ++++++------
12 files changed, 177 insertions(+), 143 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 9ce8bb242fb42..fbb9235c04090 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1113,40 +1113,40 @@ class LLVM_ABI MachineFunction {
/// MachineMemOperands are owned by the MachineFunction and need not be
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
- Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
+ Align BaseAlignment,
+ MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ Align BaseAlignment,
+ MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
- Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ Align BaseAlignment,
+ MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
- BaseAlignment, AAInfo, Ranges, MemCacheHint,
- SSID, Ordering, FailureOrdering);
+ BaseAlignment, MMOMetadata, SSID, Ordering,
+ FailureOrdering);
}
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize Size,
- Align BaseAlignment, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr, const MDNode *MemCacheHint = nullptr,
+ Align BaseAlignment,
+ MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
- BaseAlignment, AAInfo, Ranges, MemCacheHint,
- SSID, Ordering, FailureOrdering);
+ BaseAlignment, MMOMetadata, SSID, Ordering,
+ FailureOrdering);
}
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index d548e5248c1ff..e85159346a7be 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -129,6 +129,18 @@ struct MachinePointerInfo {
///
class MachineMemOperand {
public:
+ /// LLVM IR metadata carried by a MachineMemOperand.
+ struct Metadata {
+ AAMDNodes AAInfo;
+ const MDNode *Ranges;
+ const MDNode *MemCacheHint;
+
+ Metadata() : Ranges(nullptr), MemCacheHint(nullptr) {}
+ Metadata(const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr)
+ : AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {}
+ };
+
/// Flags values. These may be or'd together.
enum Flags : uint16_t {
// No flags set.
@@ -191,21 +203,19 @@ class MachineMemOperand {
/// atomic operations the atomic ordering requirements when store does not
/// occur must also be specified.
LLVM_ABI
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LocationSize TS,
- Align a, const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr,
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(
+ MachinePointerInfo PtrInfo, Flags Flags, LocationSize TS, Align A,
+ MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
LLVM_ABI
- MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
- const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr,
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(
+ MachinePointerInfo PtrInfo, Flags Flags, LLT Type, Align A,
+ MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 5aa2fe39a1bc0..3a6e894c45f5c 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1418,9 +1418,10 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
if (Regs.size() == 1) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
- MRI->getType(Regs[0]), getMemOpAlign(LI), AAInfo,
- LI.getMetadata(LLVMContext::MD_range), nullptr, LI.getSyncScopeID(),
- LI.getOrdering());
+ MRI->getType(Regs[0]), getMemOpAlign(LI),
+ MachineMemOperand::Metadata(AAInfo,
+ LI.getMetadata(LLVMContext::MD_range)),
+ LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
return true;
}
@@ -1434,10 +1435,10 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(LI);
- auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
- commonAlignment(BaseAlign, Offsets[i]),
- AAInfo, nullptr, nullptr,
- LI.getSyncScopeID(), LI.getOrdering());
+ auto *MMO =
+ MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
+ commonAlignment(BaseAlign, Offsets[i]), AAInfo,
+ LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -1466,8 +1467,8 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
if (Vals.size() == 1) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(SI.getPointerOperand()), Flags,
- MRI->getType(Vals[0]), getMemOpAlign(SI), SI.getAAMetadata(), nullptr,
- nullptr, SI.getSyncScopeID(), SI.getOrdering());
+ MRI->getType(Vals[0]), getMemOpAlign(SI), SI.getAAMetadata(),
+ SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[0], Base, *MMO);
return true;
}
@@ -1483,7 +1484,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
Align BaseAlign = getMemOpAlign(SI);
auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),
commonAlignment(BaseAlign, Offsets[i]),
- SI.getAAMetadata(), nullptr, nullptr,
+ SI.getAAMetadata(),
SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
}
@@ -2948,9 +2949,8 @@ bool IRTranslator::translateIntrinsic(
MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
}
MIB.addMemOperand(MF->getMachineMemOperand(
- MPI, Info.flags, MemTy, Alignment, CB.getAAMetadata(),
- /*Ranges=*/nullptr, /*MemCacheHint=*/nullptr, Info.ssid, Info.order,
- Info.failureOrder));
+ MPI, Info.flags, MemTy, Alignment, CB.getAAMetadata(), Info.ssid,
+ Info.order, Info.failureOrder));
}
if (CB.isConvergent()) {
@@ -3575,8 +3575,8 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
OldValRes, SuccessRes, Addr, Cmp, NewVal,
*MF->getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
- getMemOpAlign(I), I.getAAMetadata(), nullptr, nullptr,
- I.getSyncScopeID(), I.getSuccessOrdering(), I.getFailureOrdering()));
+ getMemOpAlign(I), I.getAAMetadata(), I.getSyncScopeID(),
+ I.getSuccessOrdering(), I.getFailureOrdering()));
return true;
}
@@ -3671,8 +3671,8 @@ bool IRTranslator::translateAtomicRMW(const User &U,
Opcode, Res, Addr, Val,
*MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
Flags, MRI->getType(Val), getMemOpAlign(I),
- I.getAAMetadata(), nullptr, nullptr,
- I.getSyncScopeID(), I.getOrdering()));
+ I.getAAMetadata(), I.getSyncScopeID(),
+ I.getOrdering()));
return true;
}
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 194080f928da3..c6afbbd5b1942 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3860,9 +3860,10 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
}
if (expectAndConsume(MIToken::rparen))
return true;
- Dest = MF.getMachineMemOperand(Ptr, Flags, MemoryType, Align(BaseAlignment),
- AAInfo, Range, MemCacheHint, SSID, Order,
- FailureOrder);
+ Dest = MF.getMachineMemOperand(
+ Ptr, Flags, MemoryType, Align(BaseAlignment),
+ MachineMemOperand::Metadata(AAInfo, Range, MemCacheHint), SSID, Order,
+ FailureOrder);
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index e68481e548b9d..cbfc609657441 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -566,26 +566,26 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- const MDNode *MemCacheHint, SyncScope::ID SSID, AtomicOrdering Ordering,
+ Align BaseAlignment, MachineMemOperand::Metadata MMOMetadata,
+ SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
assert((!Size.hasValue() ||
Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
"Unexpected an unknown size to be represented using "
"LocationSize::beforeOrAfter()");
return new (Allocator)
- MachineMemOperand(PtrInfo, F, Size, BaseAlignment, AAInfo, Ranges,
- MemCacheHint, SSID, Ordering, FailureOrdering);
+ MachineMemOperand(PtrInfo, F, Size, BaseAlignment, MMOMetadata, SSID,
+ Ordering, FailureOrdering);
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
- MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
- Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
- const MDNode *MemCacheHint, SyncScope::ID SSID, AtomicOrdering Ordering,
+ MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
+ Align BaseAlignment, MachineMemOperand::Metadata MMOMetadata,
+ SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, f, MemTy, base_alignment, AAInfo, Ranges,
- MemCacheHint, SSID, Ordering, FailureOrdering);
+ MachineMemOperand(PtrInfo, F, MemTy, BaseAlignment, MMOMetadata, SSID,
+ Ordering, FailureOrdering);
}
MachineMemOperand *
@@ -597,16 +597,18 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"Unexpected an unknown size to be represented using "
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
- PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(), AAMDNodes(), nullptr,
- MMO->getMemCacheHint(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
+ MachineMemOperand::Metadata(AAMDNodes(), nullptr, MMO->getMemCacheHint()),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
- PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(), AAMDNodes(), nullptr,
- MMO->getMemCacheHint(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
+ MachineMemOperand::Metadata(AAMDNodes(), nullptr, MMO->getMemCacheHint()),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -625,8 +627,10 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// are anymore.
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
- MMO->getAAInfo(), nullptr, MMO->getMemCacheHint(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MachineMemOperand::Metadata(MMO->getAAInfo(), nullptr,
+ MMO->getMemCacheHint()),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
}
MachineMemOperand *
@@ -637,19 +641,22 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachinePointerInfo(MMO->getPseudoValue(), MMO->getOffset());
return new (Allocator) MachineMemOperand(
- MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(), AAInfo,
- MMO->getRanges(), MMO->getMemCacheHint(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(),
+ MachineMemOperand::Metadata(AAInfo, MMO->getRanges(),
+ MMO->getMemCacheHint()),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
}
MachineMemOperand *
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand::Flags Flags) {
- return new (Allocator)
- MachineMemOperand(MMO->getPointerInfo(), Flags, MMO->getSize(),
- MMO->getBaseAlign(), MMO->getAAInfo(), MMO->getRanges(),
- MMO->getMemCacheHint(), MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ return new (Allocator) MachineMemOperand(
+ MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
+ MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges(),
+ MMO->getMemCacheHint()),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
}
MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfo(
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 3ea77d36a89ec..03060808c7d8b 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1169,12 +1169,15 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
return MachinePointerInfo(MF.getDataLayout().getAllocaAddrSpace());
}
-MachineMemOperand::MachineMemOperand(
- MachinePointerInfo ptrinfo, Flags f, LLT type, Align a,
- const AAMDNodes &AAInfo, const MDNode *Ranges, const MDNode *MemCacheHint,
- SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering)
- : PtrInfo(ptrinfo), MemoryType(type), FlagVals(f), BaseAlign(a),
- AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {
+MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
+ LLT Type, Align A,
+ MachineMemOperand::Metadata MMOMetadata,
+ SyncScope::ID SSID,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
+ : PtrInfo(PtrInfo), MemoryType(Type), FlagVals(F), BaseAlign(A),
+ AAInfo(MMOMetadata.AAInfo), Ranges(MMOMetadata.Ranges),
+ MemCacheHint(MMOMetadata.MemCacheHint) {
assert((PtrInfo.V.isNull() || isa<const PseudoSourceValue *>(PtrInfo.V) ||
isa<PointerType>(cast<const Value *>(PtrInfo.V)->getType())) &&
"invalid pointer value");
@@ -1188,18 +1191,19 @@ MachineMemOperand::MachineMemOperand(
assert(getFailureOrdering() == FailureOrdering && "Value truncated");
}
-MachineMemOperand::MachineMemOperand(
- MachinePointerInfo ptrinfo, Flags F, LocationSize TS, Align BaseAlignment,
- const AAMDNodes &AAInfo, const MDNode *Ranges, const MDNode *MemCacheHint,
- SyncScope::ID SSID, AtomicOrdering Ordering, AtomicOrdering FailureOrdering)
+MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
+ LocationSize TS, Align BaseAlignment,
+ MachineMemOperand::Metadata MMOMetadata,
+ SyncScope::ID SSID,
+ AtomicOrdering Ordering,
+ AtomicOrdering FailureOrdering)
: MachineMemOperand(
- ptrinfo, F,
+ PtrInfo, F,
!TS.isPrecise() ? LLT()
: TS.isScalable()
? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
: LLT::scalar(8 * TS.getValue().getKnownMinValue()),
- BaseAlignment, AAInfo, Ranges, MemCacheHint, SSID, Ordering,
- FailureOrdering) {}
+ BaseAlignment, MMOMetadata, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 1b6d7b57c2c58..a2696fcde40af 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -2374,8 +2374,9 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
if (IsInvariant)
Flags |= MachineMemOperand::MOInvariant;
- return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
- *Alignment, AAInfo, Ranges);
+ return FuncInfo.MF->getMachineMemOperand(
+ MachinePointerInfo(Ptr), Flags, Size, *Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges));
}
CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 97aa765642ea7..537034ed28389 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2514,8 +2514,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
- LocationSize::beforeOrAfterPointer(), Alignment, LD->getAAInfo(),
- LD->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment,
+ MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
Lo =
DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
@@ -2539,7 +2539,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
- Alignment, LD->getAAInfo(), LD->getRanges());
+ Alignment,
+ MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2583,8 +2584,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
- LocationSize::beforeOrAfterPointer(), Alignment, LD->getAAInfo(),
- LD->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment,
+ MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
Lo = DAG.getLoadFFVP(LoVT, dl, Ch, Ptr, MaskLo, EVLLo, MMO);
@@ -2658,7 +2659,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
- Alignment, SLD->getAAInfo(), SLD->getRanges());
+ Alignment,
+ MachineMemOperand::Metadata(SLD->getAAInfo(), SLD->getRanges()));
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2718,7 +2720,8 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, MLD->getAAInfo(), MLD->getRanges());
+ Alignment,
+ MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
MMO, MLD->getAddressingMode(), ExtType,
@@ -2742,7 +2745,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
- MLD->getAAInfo(), MLD->getRanges());
+ MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
@@ -2805,7 +2808,7 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, N->getAAInfo(), N->getRanges());
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -4474,8 +4477,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
SDValue Lo, Hi;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
- N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment,
+ MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4499,7 +4502,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, N->getAAInfo(), N->getRanges());
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4572,7 +4575,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, N->getAAInfo(), N->getRanges());
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
SDValue Hi = DAG.getStridedStoreVP(
N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
@@ -4623,8 +4626,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
SDValue Lo, Hi, Res;
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
- LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
- N->getRanges());
+ LocationSize::beforeOrAfterPointer(), Alignment,
+ MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4650,7 +4653,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, N->getAAInfo(), N->getRanges());
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4715,7 +4718,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, N->getAAInfo(), N->getRanges());
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index fa9ab3c6939b5..63fbe21efa92d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -10600,7 +10600,8 @@ SDValue SelectionDAG::getLoad(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment, AAInfo, Ranges, MemCacheHint);
+ PtrInfo, MMOFlags, Size, Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges, MemCacheHint));
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
}
@@ -10732,7 +10733,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment, AAInfo, nullptr, MemCacheHint);
+ PtrInfo, MMOFlags, Size, Alignment,
+ MachineMemOperand::Metadata(AAInfo, nullptr, MemCacheHint));
return getStore(Chain, dl, Val, Ptr, MMO);
}
@@ -10809,9 +10811,9 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment,
- AAInfo, nullptr, MemCacheHint);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment,
+ MachineMemOperand::Metadata(AAInfo, nullptr, MemCacheHint));
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -10847,8 +10849,9 @@ SDValue SelectionDAG::getLoadVP(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
- Alignment, AAInfo, Ranges);
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges));
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
MMO, IsExpanding);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 934def97381d6..0839fcbe2d4e9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5139,7 +5139,8 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
- LocationSize::upperBound(VT.getStoreSize()), Alignment, AAInfo, Ranges);
+ LocationSize::upperBound(VT.getStoreSize()), Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges));
const auto &TLI = DAG.getTargetLoweringInfo();
@@ -5183,8 +5184,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
- LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
- Ranges);
+ LocationSize::beforeOrAfterPointer(), Alignment,
+ MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -5226,7 +5227,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, nullptr, SSID, SuccessOrdering,
+ I.getAlign(), MachineMemOperand::Metadata(), SSID, SuccessOrdering,
FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
@@ -5298,7 +5299,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, nullptr, SSID, Ordering);
+ I.getAlign(), MachineMemOperand::Metadata(), SSID, Ordering);
SDValue L =
DAG.getAtomic(NT, dl, MemVT, InChain,
@@ -5345,7 +5346,8 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const MDNode *Ranges = getRangeMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), Ranges, nullptr, SSID, Order);
+ I.getAlign(), MachineMemOperand::Metadata(AAMDNodes(), Ranges), SSID,
+ Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -5382,7 +5384,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), AAMDNodes(), nullptr, nullptr, SSID, Ordering);
+ I.getAlign(), MachineMemOperand::Metadata(), SSID, Ordering);
SDValue Val = getValue(I.getValueOperand());
if (Val.getValueType() != MemVT)
@@ -5572,9 +5574,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
Size = LocationSize::precise(MemVT.getStoreSize());
Align Alignment = Info.align.value_or(DAG.getEVTAlign(MemVT));
MachineMemOperand *MMO = MF.getMachineMemOperand(
- MPI, Info.flags, Size, Alignment, I.getAAMetadata(),
- /*Ranges=*/nullptr, /*MemCacheHint=*/nullptr, Info.ssid, Info.order,
- Info.failureOrder);
+ MPI, Info.flags, Size, Alignment, I.getAAMetadata(), Info.ssid,
+ Info.order, Info.failureOrder);
MMOs.push_back(MMO);
}
@@ -6584,7 +6585,8 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
- MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
+ MemoryLocation::UnknownSize, Alignment,
+ MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -8768,7 +8770,8 @@ void SelectionDAGBuilder::visitVPLoad(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
- LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges));
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
if (AddToChain)
@@ -8795,7 +8798,8 @@ void SelectionDAGBuilder::visitVPLoadFF(
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
- LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
+ LocationSize::beforeOrAfterPointer(), *Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges));
LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO);
SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
@@ -8822,7 +8826,7 @@ void SelectionDAGBuilder::visitVPGather(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment, AAInfo, Ranges);
+ *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
SDValue Base, Index, Scale;
bool UniformBase =
getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
@@ -8931,7 +8935,7 @@ void SelectionDAGBuilder::visitVPStridedLoad(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment, AAInfo, Ranges);
+ *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO,
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index a6e82b3e1b377..a93af05d151f6 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3169,8 +3169,9 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
MachineFunction &MF = DAG.getMachineFunction();
WideMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
- MMO->getAAInfo(), MMO->getRanges(), nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges()),
+ MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
}
SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 39d6c5bf1bfe7..1c2b883032b3f 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8925,9 +8925,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
} else if (Subtarget.hasLFIWAX() &&
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
- RLI.Alignment, RLI.AAInfo, RLI.Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8936,9 +8936,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
} else if (Subtarget.hasFPCVT() &&
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
- RLI.Alignment, RLI.AAInfo, RLI.Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8970,9 +8970,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
RLI.Alignment = Align(4);
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
- RLI.Alignment, RLI.AAInfo, RLI.Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ?
PPCISD::LFIWZX : PPCISD::LFIWAX,
@@ -9032,9 +9032,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
RLI.Alignment = Align(4);
}
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
- RLI.Alignment, RLI.AAInfo, RLI.Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other), Ops,
@@ -12074,9 +12074,9 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
Op0.getValueType() == MVT::i32 && Op0.hasOneUse() &&
canReuseLoadAddress(Op0, MVT::i32, RLI, DAG, ISD::NON_EXTLOAD)) {
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4,
- RLI.Alignment, RLI.AAInfo, RLI.Ranges);
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
SDValue Bits = DAG.getMemIntrinsicNode(
PPCISD::LD_SPLAT, dl, DAG.getVTList(MVT::v4i32, MVT::Other), Ops,
@@ -15956,8 +15956,8 @@ SDValue convertTwoLoadsAndCmpToVCMPEQUB(SelectionDAG &DAG, SDNode *N,
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *NewMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), MMO->getSize(), MMO->getAlign(),
- MMO->getAAInfo(), nullptr, nullptr, MMO->getSyncScopeID(),
- MMO->getSuccessOrdering(), MMO->getFailureOrdering());
+ MMO->getAAInfo(), MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
+ MMO->getFailureOrdering());
SDValue NewLoad = DAG.getLoad(MVT::v16i8, DL, LoadNode->getChain(),
LoadNode->getBasePtr(), NewMMO);
DAG.ReplaceAllUsesOfValueWith(SDValue(LoadNode, 1), NewLoad.getValue(1));
>From 5a38e3b8bb85fa14a47912959f3ff7094ca8fdd6 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 19:44:52 +0000
Subject: [PATCH 10/33] auto generated and comments
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +-
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 5 +-
llvm/lib/CodeGen/MachineFunction.cpp | 21 +-
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 2 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 44 ++-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 15 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 23 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 3 +-
llvm/lib/Target/NVPTX/NVPTX.h | 2 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 15 +-
.../CodeGen/NVPTX/cache-hint-sm-version.ll | 341 ++++++++----------
llvm/tools/llvm-reduce/ReducerWorkItem.cpp | 7 +-
12 files changed, 247 insertions(+), 236 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 3a6e894c45f5c..00df15a08f78d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1419,8 +1419,9 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI),
- MachineMemOperand::Metadata(AAInfo,
- LI.getMetadata(LLVMContext::MD_range)),
+ MachineMemOperand::Metadata(
+ /*AAInfo=*/AAInfo,
+ /*Ranges=*/LI.getMetadata(LLVMContext::MD_range)),
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
return true;
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index c6afbbd5b1942..0222301f550f7 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3862,8 +3862,9 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
return true;
Dest = MF.getMachineMemOperand(
Ptr, Flags, MemoryType, Align(BaseAlignment),
- MachineMemOperand::Metadata(AAInfo, Range, MemCacheHint), SSID, Order,
- FailureOrder);
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Range,
+ /*MemCacheHint=*/MemCacheHint),
+ SSID, Order, FailureOrder);
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index cbfc609657441..1ae81665f45f1 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -598,7 +598,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAMDNodes(), nullptr, MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(/*AAInfo=*/AAMDNodes(), /*Ranges=*/nullptr,
+ /*MemCacheHint=*/MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -607,7 +608,8 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAMDNodes(), nullptr, MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(/*AAInfo=*/AAMDNodes(), /*Ranges=*/nullptr,
+ /*MemCacheHint=*/MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -627,8 +629,9 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// are anymore.
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
- MachineMemOperand::Metadata(MMO->getAAInfo(), nullptr,
- MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(/*AAInfo=*/MMO->getAAInfo(),
+ /*Ranges=*/nullptr,
+ /*MemCacheHint=*/MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -642,8 +645,9 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAInfo, MMO->getRanges(),
- MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo,
+ /*Ranges=*/MMO->getRanges(),
+ /*MemCacheHint=*/MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -653,8 +657,9 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand::Flags Flags) {
return new (Allocator) MachineMemOperand(
MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
- MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges(),
- MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(/*AAInfo=*/MMO->getAAInfo(),
+ /*Ranges=*/MMO->getRanges(),
+ /*MemCacheHint=*/MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index a2696fcde40af..3ea38345f0f37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -2376,7 +2376,7 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
return FuncInfo.MF->getMachineMemOperand(
MachinePointerInfo(Ptr), Flags, Size, *Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
}
CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 537034ed28389..765fe644a9a47 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2515,7 +2515,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/LD->getAAInfo(),
+ /*Ranges=*/LD->getRanges()));
Lo =
DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
@@ -2540,7 +2541,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/LD->getAAInfo(),
+ /*Ranges=*/LD->getRanges()));
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2585,7 +2587,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/LD->getAAInfo(),
+ /*Ranges=*/LD->getRanges()));
Lo = DAG.getLoadFFVP(LoVT, dl, Ch, Ptr, MaskLo, EVLLo, MMO);
@@ -2660,7 +2663,8 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(SLD->getAAInfo(), SLD->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/SLD->getAAInfo(),
+ /*Ranges=*/SLD->getRanges()));
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2721,7 +2725,8 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/MLD->getAAInfo(),
+ /*Ranges=*/MLD->getRanges()));
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
MMO, MLD->getAddressingMode(), ExtType,
@@ -2745,7 +2750,8 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/MLD->getAAInfo(),
+ /*Ranges=*/MLD->getRanges()));
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
@@ -2808,7 +2814,9 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -4478,7 +4486,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4502,7 +4511,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4575,7 +4586,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
SDValue Hi = DAG.getStridedStoreVP(
N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
@@ -4627,7 +4640,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4653,7 +4667,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4718,7 +4734,9 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
+ /*Ranges=*/N->getRanges()));
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 63fbe21efa92d..f6a1c56473a29 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -10601,7 +10601,8 @@ SDValue SelectionDAG::getLoad(
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges, MemCacheHint));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges,
+ /*MemCacheHint=*/MemCacheHint));
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
}
@@ -10734,7 +10735,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(AAInfo, nullptr, MemCacheHint));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/nullptr,
+ /*MemCacheHint=*/MemCacheHint));
return getStore(Chain, dl, Val, Ptr, MMO);
}
@@ -10813,7 +10815,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment,
- MachineMemOperand::Metadata(AAInfo, nullptr, MemCacheHint));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/nullptr,
+ /*MemCacheHint=*/MemCacheHint));
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -10849,9 +10852,9 @@ SDValue SelectionDAG::getLoadVP(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MMOFlags, Size, Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
MMO, IsExpanding);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 0839fcbe2d4e9..504a02a6077fd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5140,7 +5140,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::upperBound(VT.getStoreSize()), Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
const auto &TLI = DAG.getTargetLoweringInfo();
@@ -5185,7 +5185,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/I.getAAMetadata(),
+ /*Ranges=*/Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -5346,8 +5347,9 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const MDNode *Ranges = getRangeMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MachineMemOperand::Metadata(AAMDNodes(), Ranges), SSID,
- Order);
+ I.getAlign(),
+ MachineMemOperand::Metadata(/*AAInfo=*/AAMDNodes(), /*Ranges=*/Ranges),
+ SSID, Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -6586,7 +6588,8 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
MachinePointerInfo(AS),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
MemoryLocation::UnknownSize, Alignment,
- MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/I.getAAMetadata(),
+ /*Ranges=*/Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -8771,7 +8774,7 @@ void SelectionDAGBuilder::visitVPLoad(
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::beforeOrAfterPointer(), *Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
if (AddToChain)
@@ -8799,7 +8802,7 @@ void SelectionDAGBuilder::visitVPLoadFF(
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), *Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO);
SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
@@ -8826,7 +8829,8 @@ void SelectionDAGBuilder::visitVPGather(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
+ *Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
SDValue Base, Index, Scale;
bool UniformBase =
getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
@@ -8935,7 +8939,8 @@ void SelectionDAGBuilder::visitVPStridedLoad(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
+ *Alignment,
+ MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO,
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index a93af05d151f6..2450ebeb4381b 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3169,7 +3169,8 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
MachineFunction &MF = DAG.getMachineFunction();
WideMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
- MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges()),
+ MachineMemOperand::Metadata(/*AAInfo=*/MMO->getAAInfo(),
+ /*Ranges=*/MMO->getRanges()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index d047a303fb96e..0853f7114a2d9 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -252,7 +252,7 @@ enum class L2Prefetch : uint8_t {
// Bits 0-2: L1 Eviction (3 bits, 5 values)
// Bits 3-4: L2 Eviction (2 bits, 3 values)
// Bits 5-6: L2 Prefetch (2 bits, 4 values)
-// Bit 7: L2::cache_hint mode flag (set when using CachePolicy)
+// Bit 7: L2::cache_hint mode flag (set when using CachePolicy)
// Bits 8-31: Reserved
//
// Using llvm::Bitfield for type-safe access with compile-time validation.
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 1c2b883032b3f..cbe6adb2c61b4 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8927,7 +8927,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
+ /*Ranges=*/RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8938,7 +8939,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
+ /*Ranges=*/RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8972,7 +8974,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
+ /*Ranges=*/RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ?
PPCISD::LFIWZX : PPCISD::LFIWAX,
@@ -9034,7 +9037,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
+ /*Ranges=*/RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other), Ops,
@@ -12076,7 +12080,8 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
+ /*Ranges=*/RLI.Ranges));
SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
SDValue Bits = DAG.getMemIntrinsicNode(
PPCISD::LD_SPLAT, dl, DAG.getVTList(MVT::v4i32, MVT::Other), Ops,
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
index dcd3188fa1270..7dbf67cf4ca13 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
@@ -1,10 +1,11 @@
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefix=SM60
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefix=SM70
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefix=SM75
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefix=SM80
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s --check-prefix=SM80-PTX70
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefix=SM86
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefix=SM90
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(ld\.global|st\.global|mov\.b64)" --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM60
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM70
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM75
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=COMMON,SM80
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s --check-prefixes=SM80-PTX70
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefixes=COMMON,SM86
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=COMMON,SM90
; Test SM version requirements for cache hints (from PTX ISA documentation):
; - L1::evict_* requires SM 70+
@@ -19,25 +20,21 @@
; SM70+ should emit L1::evict_first
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_l1_first
-; SM60: ld.global.b32
-; SM60-NOT: L1::evict_first
-
-; SM70-LABEL: test_load_l1_first
-; SM70: ld.global.L1::evict_first.b32
-
-; SM75-LABEL: test_load_l1_first
-; SM75: ld.global.L1::evict_first.b32
-
-; SM80-LABEL: test_load_l1_first
-; SM80: ld.global.L1::evict_first.b32
-
-; SM86-LABEL: test_load_l1_first
-; SM86: ld.global.L1::evict_first.b32
-
-; SM90-LABEL: test_load_l1_first
-; SM90: ld.global.L1::evict_first.b32
define i32 @test_load_l1_first(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_l1_first(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_l1_first(
+; SM70: ld.global.L1::evict_first.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_l1_first(
+; SM75: ld.global.L1::evict_first.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_l1_first(
+; COMMON: ld.global.L1::evict_first.b32 %r1, [%rd1];
+;
+; SM80-PTX70-LABEL: test_load_l1_first(
+; SM80-PTX70: ld.global.L1::evict_first.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
}
@@ -48,25 +45,21 @@ define i32 @test_load_l1_first(ptr addrspace(1) %p) {
; SM70+ should emit L2::evict_last
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_l2_last
-; SM60: ld.global.b32
-; SM60-NOT: L2::evict_last
-
-; SM70-LABEL: test_load_l2_last
-; SM70: ld.global.L2::evict_last.b32
-
-; SM75-LABEL: test_load_l2_last
-; SM75: ld.global.L2::evict_last.b32
-
-; SM80-LABEL: test_load_l2_last
-; SM80: ld.global.L2::evict_last.b32
-
-; SM86-LABEL: test_load_l2_last
-; SM86: ld.global.L2::evict_last.b32
-
-; SM90-LABEL: test_load_l2_last
-; SM90: ld.global.L2::evict_last.b32
define i32 @test_load_l2_last(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_l2_last(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_l2_last(
+; SM70: ld.global.L2::evict_last.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_l2_last(
+; SM75: ld.global.L2::evict_last.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_l2_last(
+; COMMON: ld.global.L2::evict_last.b32 %r1, [%rd1];
+;
+; SM80-PTX70-LABEL: test_load_l2_last(
+; SM80-PTX70: ld.global.L2::evict_last.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
ret i32 %v
}
@@ -77,26 +70,21 @@ define i32 @test_load_l2_last(ptr addrspace(1) %p) {
; SM75+ should emit L2::64B
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_prefetch_64
-; SM60: ld.global.b32
-; SM60-NOT: L2::64B
-
-; SM70-LABEL: test_load_prefetch_64
-; SM70: ld.global.b32
-; SM70-NOT: L2::64B
-
-; SM75-LABEL: test_load_prefetch_64
-; SM75: ld.global.L2::64B.b32
-
-; SM80-LABEL: test_load_prefetch_64
-; SM80: ld.global.L2::64B.b32
-
-; SM86-LABEL: test_load_prefetch_64
-; SM86: ld.global.L2::64B.b32
-
-; SM90-LABEL: test_load_prefetch_64
-; SM90: ld.global.L2::64B.b32
define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_prefetch_64(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_prefetch_64(
+; SM70: ld.global.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_prefetch_64(
+; SM75: ld.global.L2::64B.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_prefetch_64(
+; COMMON: ld.global.L2::64B.b32 %r1, [%rd1];
+;
+; SM80-PTX70-LABEL: test_load_prefetch_64(
+; SM80-PTX70: ld.global.L2::64B.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
ret i32 %v
}
@@ -107,26 +95,21 @@ define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
; SM75+ should emit L2::128B
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_prefetch_128
-; SM60: ld.global.b32
-; SM60-NOT: L2::128B
-
-; SM70-LABEL: test_load_prefetch_128
-; SM70: ld.global.b32
-; SM70-NOT: L2::128B
-
-; SM75-LABEL: test_load_prefetch_128
-; SM75: ld.global.L2::128B.b32
-
-; SM80-LABEL: test_load_prefetch_128
-; SM80: ld.global.L2::128B.b32
-
-; SM86-LABEL: test_load_prefetch_128
-; SM86: ld.global.L2::128B.b32
-
-; SM90-LABEL: test_load_prefetch_128
-; SM90: ld.global.L2::128B.b32
define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_prefetch_128(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_prefetch_128(
+; SM70: ld.global.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_prefetch_128(
+; SM75: ld.global.L2::128B.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_prefetch_128(
+; COMMON: ld.global.L2::128B.b32 %r1, [%rd1];
+;
+; SM80-PTX70-LABEL: test_load_prefetch_128(
+; SM80-PTX70: ld.global.L2::128B.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
ret i32 %v
}
@@ -137,27 +120,21 @@ define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
; SM80+ should emit L2::256B
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_prefetch_256
-; SM60: ld.global.b32
-; SM60-NOT: L2::256B
-
-; SM70-LABEL: test_load_prefetch_256
-; SM70: ld.global.b32
-; SM70-NOT: L2::256B
-
-; SM75-LABEL: test_load_prefetch_256
-; SM75: ld.global.b32
-; SM75-NOT: L2::256B
-
-; SM80-LABEL: test_load_prefetch_256
-; SM80: ld.global.L2::256B.b32
-
-; SM86-LABEL: test_load_prefetch_256
-; SM86: ld.global.L2::256B.b32
-
-; SM90-LABEL: test_load_prefetch_256
-; SM90: ld.global.L2::256B.b32
define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_prefetch_256(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_prefetch_256(
+; SM70: ld.global.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_prefetch_256(
+; SM75: ld.global.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_prefetch_256(
+; COMMON: ld.global.L2::256B.b32 %r1, [%rd1];
+;
+; SM80-PTX70-LABEL: test_load_prefetch_256(
+; SM80-PTX70: ld.global.L2::256B.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
ret i32 %v
}
@@ -169,34 +146,22 @@ define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
; SM80+ with PTX 7.4+ should emit L2::cache_hint
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_cache_hint
-; SM60: ld.global.b32
-; SM60-NOT: L2::cache_hint
-
-; SM70-LABEL: test_load_cache_hint
-; SM70: ld.global.b32
-; SM70-NOT: L2::cache_hint
-
-; SM75-LABEL: test_load_cache_hint
-; SM75: ld.global.b32
-; SM75-NOT: L2::cache_hint
-
-; SM80-LABEL: test_load_cache_hint
-; SM80: mov.b64 [[POLICY:%rd[0-9]+]], 12345
-; SM80: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-
-; SM80-PTX70-LABEL: test_load_cache_hint
-; SM80-PTX70: ld.global.b32
-; SM80-PTX70-NOT: L2::cache_hint
-
-; SM86-LABEL: test_load_cache_hint
-; SM86: mov.b64 [[POLICY:%rd[0-9]+]], 12345
-; SM86: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-
-; SM90-LABEL: test_load_cache_hint
-; SM90: mov.b64 [[POLICY:%rd[0-9]+]], 12345
-; SM90: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
define i32 @test_load_cache_hint(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_cache_hint(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_cache_hint(
+; SM70: ld.global.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_cache_hint(
+; SM75: ld.global.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_cache_hint(
+; COMMON: mov.b64 %rd2, 12345;
+; COMMON: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+;
+; SM80-PTX70-LABEL: test_load_cache_hint(
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
ret i32 %v
}
@@ -208,27 +173,22 @@ define i32 @test_load_cache_hint(ptr addrspace(1) %p) {
; Both emitted on SM80+
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_cache_hint_with_l1
-; SM60: ld.global.b32
-; SM60-NOT: L1::evict_first
-; SM60-NOT: L2::cache_hint
-
-; SM70-LABEL: test_load_cache_hint_with_l1
-; SM70: ld.global.L1::evict_first.b32
-; SM70-NOT: L2::cache_hint
-
-; SM75-LABEL: test_load_cache_hint_with_l1
-; SM75: ld.global.L1::evict_first.b32
-; SM75-NOT: L2::cache_hint
-
-; SM80-LABEL: test_load_cache_hint_with_l1
-; SM80: mov.b64 [[POLICY:%rd[0-9]+]], 44445
-; SM80: ld.global.L1::evict_first.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-
-; SM80-PTX70-LABEL: test_load_cache_hint_with_l1
-; SM80-PTX70: ld.global.L1::evict_first.b32
-; SM80-PTX70-NOT: L2::cache_hint
define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_cache_hint_with_l1(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_cache_hint_with_l1(
+; SM70: ld.global.L1::evict_first.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_cache_hint_with_l1(
+; SM75: ld.global.L1::evict_first.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_cache_hint_with_l1(
+; COMMON: mov.b64 %rd2, 44445;
+; COMMON: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+;
+; SM80-PTX70-LABEL: test_load_cache_hint_with_l1(
+; SM80-PTX70: ld.global.L1::evict_first.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
ret i32 %v
}
@@ -240,21 +200,21 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
; Both emitted on SM75+
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_load_prefetch_with_l1
-; SM60: ld.global.b32
-; SM60-NOT: L1::evict_first
-; SM60-NOT: L2::128B
-
-; SM70-LABEL: test_load_prefetch_with_l1
-; SM70: ld.global.L1::evict_first.b32
-; SM70-NOT: L2::128B
-
-; SM75-LABEL: test_load_prefetch_with_l1
-; SM75: ld.global.L1::evict_first.L2::128B.b32
-
-; SM80-LABEL: test_load_prefetch_with_l1
-; SM80: ld.global.L1::evict_first.L2::128B.b32
define i32 @test_load_prefetch_with_l1(ptr addrspace(1) %p) {
+; SM60-LABEL: test_load_prefetch_with_l1(
+; SM60: ld.global.b32 %r1, [%rd1];
+;
+; SM70-LABEL: test_load_prefetch_with_l1(
+; SM70: ld.global.L1::evict_first.b32 %r1, [%rd1];
+;
+; SM75-LABEL: test_load_prefetch_with_l1(
+; SM75: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
+;
+; COMMON-LABEL: test_load_prefetch_with_l1(
+; COMMON: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
+;
+; SM80-PTX70-LABEL: test_load_prefetch_with_l1(
+; SM80-PTX70: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
ret i32 %v
}
@@ -263,22 +223,22 @@ define i32 @test_load_prefetch_with_l1(ptr addrspace(1) %p) {
; Store with L2::cache_hint
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_store_cache_hint
-; SM60: st.global.b32
-; SM60-NOT: L2::cache_hint
-
-; SM70-LABEL: test_store_cache_hint
-; SM70: st.global.b32
-; SM70-NOT: L2::cache_hint
-
-; SM80-LABEL: test_store_cache_hint
-; SM80: mov.b64 [[POLICY:%rd[0-9]+]], 67890
-; SM80: st.global.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
-
-; SM80-PTX70-LABEL: test_store_cache_hint
-; SM80-PTX70: st.global.b32
-; SM80-PTX70-NOT: L2::cache_hint
define void @test_store_cache_hint(ptr addrspace(1) %p, i32 %v) {
+; SM60-LABEL: test_store_cache_hint(
+; SM60: st.global.b32 [%rd1], %r1;
+;
+; SM70-LABEL: test_store_cache_hint(
+; SM70: st.global.b32 [%rd1], %r1;
+;
+; SM75-LABEL: test_store_cache_hint(
+; SM75: st.global.b32 [%rd1], %r1;
+;
+; COMMON-LABEL: test_store_cache_hint(
+; COMMON: mov.b64 %rd2, 67890;
+; COMMON: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+;
+; SM80-PTX70-LABEL: test_store_cache_hint(
+; SM80-PTX70: st.global.b32 [%rd1], %r1;
store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !5
ret void
}
@@ -287,16 +247,21 @@ define void @test_store_cache_hint(ptr addrspace(1) %p, i32 %v) {
; Store with L1 eviction hint
;-----------------------------------------------------------------------------
-; SM60-LABEL: test_store_l1_no_allocate
-; SM60: st.global.b32
-; SM60-NOT: L1::no_allocate
-
-; SM70-LABEL: test_store_l1_no_allocate
-; SM70: st.global.L1::no_allocate.b32
-
-; SM80-LABEL: test_store_l1_no_allocate
-; SM80: st.global.L1::no_allocate.b32
define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
+; SM60-LABEL: test_store_l1_no_allocate(
+; SM60: st.global.b32 [%rd1], %r1;
+;
+; SM70-LABEL: test_store_l1_no_allocate(
+; SM70: st.global.L1::no_allocate.b32 [%rd1], %r1;
+;
+; SM75-LABEL: test_store_l1_no_allocate(
+; SM75: st.global.L1::no_allocate.b32 [%rd1], %r1;
+;
+; COMMON-LABEL: test_store_l1_no_allocate(
+; COMMON: st.global.L1::no_allocate.b32 [%rd1], %r1;
+;
+; SM80-PTX70-LABEL: test_store_l1_no_allocate(
+; SM80-PTX70: st.global.L1::no_allocate.b32 [%rd1], %r1;
store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !9
ret void
}
@@ -344,3 +309,7 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
; L1 eviction: no_allocate (for store)
!9 = !{i32 1, !109}
!109 = !{!"nvvm.l1_eviction", !"no_allocate"}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SM80: {{.*}}
+; SM86: {{.*}}
+; SM90: {{.*}}
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index d984dcfc91aaa..9f9c1e54c1174 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -238,8 +238,11 @@ static void cloneMemOperands(MachineInstr &DstMI, MachineInstr &SrcMI,
MachineMemOperand *NewMMO = DstMF.getMachineMemOperand(
NewPtrInfo, OldMMO->getFlags(), OldMMO->getMemoryType(),
- OldMMO->getBaseAlign(), OldMMO->getAAInfo(), OldMMO->getRanges(),
- nullptr, OldMMO->getSyncScopeID(), OldMMO->getSuccessOrdering(),
+ OldMMO->getBaseAlign(),
+ MachineMemOperand::Metadata(/*AAInfo=*/OldMMO->getAAInfo(),
+ /*Ranges=*/OldMMO->getRanges(),
+ /*MemCacheHint=*/nullptr),
+ OldMMO->getSyncScopeID(), OldMMO->getSuccessOrdering(),
OldMMO->getFailureOrdering());
NewMMOs.push_back(NewMMO);
}
>From b251461aaa9f586b5ec3171803cc539abd34f898 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 20:02:19 +0000
Subject: [PATCH 11/33] auto generate
---
.../CodeGen/NVPTX/cache-hint-sm-version.ll | 52 +++++++++----------
1 file changed, 26 insertions(+), 26 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
index 7dbf67cf4ca13..cbd57fbf796d4 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
@@ -2,10 +2,10 @@
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM60
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM70
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM75
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=COMMON,SM80
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS,SM80
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s --check-prefixes=SM80-PTX70
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefixes=COMMON,SM86
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=COMMON,SM90
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS,SM86
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=SM80PLUS,SM90
; Test SM version requirements for cache hints (from PTX ISA documentation):
; - L1::evict_* requires SM 70+
@@ -30,8 +30,8 @@ define i32 @test_load_l1_first(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_l1_first(
; SM75: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_l1_first(
-; COMMON: ld.global.L1::evict_first.b32 %r1, [%rd1];
+; SM80PLUS-LABEL: test_load_l1_first(
+; SM80PLUS: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_l1_first(
; SM80-PTX70: ld.global.L1::evict_first.b32 %r1, [%rd1];
@@ -55,8 +55,8 @@ define i32 @test_load_l2_last(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_l2_last(
; SM75: ld.global.L2::evict_last.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_l2_last(
-; COMMON: ld.global.L2::evict_last.b32 %r1, [%rd1];
+; SM80PLUS-LABEL: test_load_l2_last(
+; SM80PLUS: ld.global.L2::evict_last.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_l2_last(
; SM80-PTX70: ld.global.L2::evict_last.b32 %r1, [%rd1];
@@ -80,8 +80,8 @@ define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_prefetch_64(
; SM75: ld.global.L2::64B.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_prefetch_64(
-; COMMON: ld.global.L2::64B.b32 %r1, [%rd1];
+; SM80PLUS-LABEL: test_load_prefetch_64(
+; SM80PLUS: ld.global.L2::64B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_64(
; SM80-PTX70: ld.global.L2::64B.b32 %r1, [%rd1];
@@ -105,8 +105,8 @@ define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_prefetch_128(
; SM75: ld.global.L2::128B.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_prefetch_128(
-; COMMON: ld.global.L2::128B.b32 %r1, [%rd1];
+; SM80PLUS-LABEL: test_load_prefetch_128(
+; SM80PLUS: ld.global.L2::128B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_128(
; SM80-PTX70: ld.global.L2::128B.b32 %r1, [%rd1];
@@ -130,8 +130,8 @@ define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_prefetch_256(
; SM75: ld.global.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_prefetch_256(
-; COMMON: ld.global.L2::256B.b32 %r1, [%rd1];
+; SM80PLUS-LABEL: test_load_prefetch_256(
+; SM80PLUS: ld.global.L2::256B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_256(
; SM80-PTX70: ld.global.L2::256B.b32 %r1, [%rd1];
@@ -156,9 +156,9 @@ define i32 @test_load_cache_hint(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_cache_hint(
; SM75: ld.global.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_cache_hint(
-; COMMON: mov.b64 %rd2, 12345;
-; COMMON: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+; SM80PLUS-LABEL: test_load_cache_hint(
+; SM80PLUS: mov.b64 %rd2, 12345;
+; SM80PLUS: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
;
; SM80-PTX70-LABEL: test_load_cache_hint(
; SM80-PTX70: ld.global.b32 %r1, [%rd1];
@@ -183,9 +183,9 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_cache_hint_with_l1(
; SM75: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_cache_hint_with_l1(
-; COMMON: mov.b64 %rd2, 44445;
-; COMMON: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+; SM80PLUS-LABEL: test_load_cache_hint_with_l1(
+; SM80PLUS: mov.b64 %rd2, 44445;
+; SM80PLUS: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
;
; SM80-PTX70-LABEL: test_load_cache_hint_with_l1(
; SM80-PTX70: ld.global.L1::evict_first.b32 %r1, [%rd1];
@@ -210,8 +210,8 @@ define i32 @test_load_prefetch_with_l1(ptr addrspace(1) %p) {
; SM75-LABEL: test_load_prefetch_with_l1(
; SM75: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
;
-; COMMON-LABEL: test_load_prefetch_with_l1(
-; COMMON: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
+; SM80PLUS-LABEL: test_load_prefetch_with_l1(
+; SM80PLUS: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_with_l1(
; SM80-PTX70: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
@@ -233,9 +233,9 @@ define void @test_store_cache_hint(ptr addrspace(1) %p, i32 %v) {
; SM75-LABEL: test_store_cache_hint(
; SM75: st.global.b32 [%rd1], %r1;
;
-; COMMON-LABEL: test_store_cache_hint(
-; COMMON: mov.b64 %rd2, 67890;
-; COMMON: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+; SM80PLUS-LABEL: test_store_cache_hint(
+; SM80PLUS: mov.b64 %rd2, 67890;
+; SM80PLUS: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
;
; SM80-PTX70-LABEL: test_store_cache_hint(
; SM80-PTX70: st.global.b32 [%rd1], %r1;
@@ -257,8 +257,8 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
; SM75-LABEL: test_store_l1_no_allocate(
; SM75: st.global.L1::no_allocate.b32 [%rd1], %r1;
;
-; COMMON-LABEL: test_store_l1_no_allocate(
-; COMMON: st.global.L1::no_allocate.b32 [%rd1], %r1;
+; SM80PLUS-LABEL: test_store_l1_no_allocate(
+; SM80PLUS: st.global.L1::no_allocate.b32 [%rd1], %r1;
;
; SM80-PTX70-LABEL: test_store_l1_no_allocate(
; SM80-PTX70: st.global.L1::no_allocate.b32 [%rd1], %r1;
>From 5dca97278b4e5230ce66f987a049dff346b87d7e Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 21:23:44 +0000
Subject: [PATCH 12/33] tests and autogenerated
---
...ic-cache-hint.ll => cache-hint-atomics.ll} | 27 +-
.../CodeGen/NVPTX/cache-hint-cache-policy.ll | 270 +++++
.../CodeGen/NVPTX/cache-hint-intrinsics.ll | 615 ++++++++++++
...id-cache-hint.ll => cache-hint-invalid.ll} | 0
.../CodeGen/NVPTX/cache-hint-load-store.ll | 408 ++++++++
.../CodeGen/NVPTX/cache-hint-sm-version.ll | 10 +-
.../CodeGen/NVPTX/cache-hint-transforms.ll | 195 ++++
.../CodeGen/NVPTX/intrinsics-cache-hint.ll | 444 ---------
.../CodeGen/NVPTX/load-store-cache-hint.ll | 935 ------------------
9 files changed, 1503 insertions(+), 1401 deletions(-)
rename llvm/test/CodeGen/NVPTX/{atomic-cache-hint.ll => cache-hint-atomics.ll} (78%)
create mode 100644 llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll
rename llvm/test/CodeGen/NVPTX/{invalid-cache-hint.ll => cache-hint-invalid.ll} (100%)
create mode 100644 llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll
create mode 100644 llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
delete mode 100644 llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll
delete mode 100644 llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll
diff --git a/llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll b/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
similarity index 78%
rename from llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll
rename to llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
index be331571b3108..83ab4d9f0fa61 100644
--- a/llvm/test/CodeGen/NVPTX/atomic-cache-hint.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-atomics.ll
@@ -1,46 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
; !mem.cache_hint is legal on atomic IR memory instructions, but
; SelectionDAGBuilder currently drops it for atomic loads, stores, RMWs, and
-; cmpxchg.
+; cmpxchg.
; TODO: This should eventually lower to ld.acquire.sys.global.L1::evict_first.b32.
-; CHECK-LABEL: atomic_load_l1_hint(
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK: ld.acquire.sys.global.b32
define i32 @atomic_load_l1_hint(ptr addrspace(1) %p) {
+; CHECK-LABEL: atomic_load_l1_hint(
+; CHECK: ld.acquire.sys.global.b32 %r1, [%rd1];
%v = load atomic i32, ptr addrspace(1) %p acquire, align 4, !mem.cache_hint !0
ret i32 %v
}
; TODO: This should eventually lower to st.release.sys.global.L2::cache_hint.b32.
-; CHECK-LABEL: atomic_store_l2_cache_policy(
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK: st.release.sys.global.b32
define void @atomic_store_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomic_store_l2_cache_policy(
+; CHECK: st.release.sys.global.b32 [%rd1], %r1;
store atomic i32 %v, ptr addrspace(1) %p release, align 4, !mem.cache_hint !2
ret void
}
; TODO: This should eventually lower to atom.relaxed.sys.global.add.L2::cache_hint.u32.
-; CHECK-LABEL: atomicrmw_add_l2_cache_policy(
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK: atom.relaxed.sys.global.add.u32
define i32 @atomicrmw_add_l2_cache_policy(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: atomicrmw_add_l2_cache_policy(
+; CHECK: atom.relaxed.sys.global.add.u32 %r2, [%rd1], %r1;
%old = atomicrmw add ptr addrspace(1) %p, i32 %v monotonic, align 4, !mem.cache_hint !1
ret i32 %old
}
; PTX atom.cas does not have a .level::cache_hint operand.
-; CHECK-LABEL: cmpxchg_l2_cache_policy(
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK: atom.relaxed.sys.global.cas.b32
define i32 @cmpxchg_l2_cache_policy(ptr addrspace(1) %p, i32 %cmp, i32 %new) {
+; CHECK-LABEL: cmpxchg_l2_cache_policy(
+; CHECK: atom.relaxed.sys.global.cas.b32 %r3, [%rd1], %r1, %r2;
%pair = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new monotonic monotonic, align 4, !mem.cache_hint !1
%old = extractvalue { i32, i1 } %pair, 0
ret i32 %old
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll b/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
new file mode 100644
index 0000000000000..bec565949561c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
@@ -0,0 +1,270 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; Test L2::cache_hint metadata lowering with constant cache-policy operands.
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint with constant cache-policy operand (metadata-based)
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_cache_hint_i32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+define i64 @test_load_cache_hint_i64(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_i64(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.b64 %rd3, [%rd1], %rd2;
+ %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !1
+ ret i64 %v
+}
+
+define float @test_load_cache_hint_f32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_f32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load float, ptr addrspace(1) %p, !mem.cache_hint !2
+ ret float %v
+}
+
+define void @test_store_cache_hint_i32(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_cache_hint_i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !3
+ ret void
+}
+
+define void @test_store_cache_hint_i64(ptr addrspace(1) %p, i64 %v) {
+; CHECK-LABEL: test_store_cache_hint_i64(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.b64 [%rd1], %rd3, %rd2;
+ store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !4
+ ret void
+}
+
+define void @test_store_cache_hint_f32(ptr addrspace(1) %p, float %v) {
+; CHECK-LABEL: test_store_cache_hint_f32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+ store float %v, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint with vector types
+;-----------------------------------------------------------------------------
+
+define <2 x i32> @test_load_cache_hint_v2i32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_v2i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.v2.b32 {%r1, %r2}, [%rd1], %rd2;
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !6
+ ret <2 x i32> %v
+}
+
+define <4 x i32> @test_load_cache_hint_v4i32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_v4i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1], %rd2;
+ %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret <4 x i32> %v
+}
+
+define <2 x i64> @test_load_cache_hint_v2i64(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_v2i64(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.v2.b64 {%rd3, %rd4}, [%rd1], %rd2;
+ %v = load <2 x i64>, ptr addrspace(1) %p, !mem.cache_hint !8
+ ret <2 x i64> %v
+}
+
+define <2 x float> @test_load_cache_hint_v2f32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_v2f32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.v2.b32 {%r1, %r2}, [%rd1], %rd2;
+ %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !9
+ ret <2 x float> %v
+}
+
+define <2 x double> @test_load_cache_hint_v2f64(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_v2f64(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.v2.b64 {%rd3, %rd4}, [%rd1], %rd2;
+ %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !10
+ ret <2 x double> %v
+}
+
+define void @test_store_cache_hint_v2i32(ptr addrspace(1) %p, <2 x i32> %v) {
+; CHECK-LABEL: test_store_cache_hint_v2i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.v2.b32 [%rd1], {%r1, %r2}, %rd2;
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !11
+ ret void
+}
+
+define void @test_store_cache_hint_v4i32(ptr addrspace(1) %p, <4 x i32> %v) {
+; CHECK-LABEL: test_store_cache_hint_v4i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.v4.b32 [%rd1], {%r1, %r2, %r3, %r4}, %rd2;
+ store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !12
+ ret void
+}
+
+define void @test_store_cache_hint_v2i64(ptr addrspace(1) %p, <2 x i64> %v) {
+; CHECK-LABEL: test_store_cache_hint_v2i64(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.v2.b64 [%rd1], {%rd3, %rd4}, %rd2;
+ store <2 x i64> %v, ptr addrspace(1) %p, !mem.cache_hint !13
+ ret void
+}
+
+define void @test_store_cache_hint_v2f32(ptr addrspace(1) %p, <2 x float> %v) {
+; CHECK-LABEL: test_store_cache_hint_v2f32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.v2.b32 [%rd1], {%r1, %r2}, %rd2;
+ store <2 x float> %v, ptr addrspace(1) %p, !mem.cache_hint !14
+ ret void
+}
+
+define void @test_store_cache_hint_v2f64(ptr addrspace(1) %p, <2 x double> %v) {
+; CHECK-LABEL: test_store_cache_hint_v2f64(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L2::cache_hint.v2.b64 [%rd1], {%rd3, %rd4}, %rd2;
+ store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !15
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint combined with other hints (L2::cache_hint takes precedence)
+;-----------------------------------------------------------------------------
+
+; L2::cache_hint + L1 eviction: both qualifiers should be emitted
+define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_with_l1(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !16
+ ret i32 %v
+}
+
+; L2::cache_hint + L2 eviction: both qualifiers should be emitted
+define i32 @test_load_cache_hint_with_l2_eviction(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_with_l2_eviction(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::evict_last.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !17
+ ret i32 %v
+}
+
+; L2::cache_hint + L2 prefetch: both qualifiers should be emitted
+define i32 @test_load_cache_hint_with_prefetch(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_with_prefetch(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.L2::128B.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !18
+ ret i32 %v
+}
+
+; L2::cache_hint + all other hints: all qualifiers emitted
+define i32 @test_load_cache_hint_with_all(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_with_all(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::256B.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !19
+ ret i32 %v
+}
+
+; Store: L2::cache_hint + L1 eviction
+define void @test_store_cache_hint_with_l1(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_cache_hint_with_l1(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L1::evict_unchanged.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !20
+ ret void
+}
+
+; Store: L2::cache_hint + L1 + L2 eviction (all qualifiers emitted)
+define void @test_store_cache_hint_with_all(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_cache_hint_with_all(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !21
+ ret void
+}
+
+; Vector load: L2::cache_hint + L1 eviction
+define <2 x i32> @test_load_cache_hint_v2i32_with_l1(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_v2i32_with_l1(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.v2.b32 {%r1, %r2}, [%rd1], %rd2;
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !22
+ ret <2 x i32> %v
+}
+
+; Vector store: L2::cache_hint + L1 + L2 eviction + prefetch (all qualifiers emitted)
+define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32> %v) {
+; CHECK-LABEL: test_store_cache_hint_v2i32_with_all(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::64B.v2.b32 [%rd1], {%r1, %r2}, %rd2;
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !23
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+!0 = !{i32 0, !24}
+!24 = !{!"nvvm.l2_cache_hint", i64 12345}
+!1 = !{i32 0, !25}
+!25 = !{!"nvvm.l2_cache_hint", i64 12345}
+!2 = !{i32 0, !26}
+!26 = !{!"nvvm.l2_cache_hint", i64 12345}
+!3 = !{i32 1, !27}
+!27 = !{!"nvvm.l2_cache_hint", i64 12345}
+!4 = !{i32 1, !28}
+!28 = !{!"nvvm.l2_cache_hint", i64 12345}
+!5 = !{i32 1, !29}
+!29 = !{!"nvvm.l2_cache_hint", i64 12345}
+!6 = !{i32 0, !30}
+!30 = !{!"nvvm.l2_cache_hint", i64 12345}
+!7 = !{i32 0, !31}
+!31 = !{!"nvvm.l2_cache_hint", i64 12345}
+!8 = !{i32 0, !32}
+!32 = !{!"nvvm.l2_cache_hint", i64 12345}
+!9 = !{i32 0, !33}
+!33 = !{!"nvvm.l2_cache_hint", i64 12345}
+!10 = !{i32 0, !34}
+!34 = !{!"nvvm.l2_cache_hint", i64 12345}
+!11 = !{i32 1, !35}
+!35 = !{!"nvvm.l2_cache_hint", i64 12345}
+!12 = !{i32 1, !36}
+!36 = !{!"nvvm.l2_cache_hint", i64 12345}
+!13 = !{i32 1, !37}
+!37 = !{!"nvvm.l2_cache_hint", i64 12345}
+!14 = !{i32 1, !38}
+!38 = !{!"nvvm.l2_cache_hint", i64 12345}
+!15 = !{i32 1, !39}
+!39 = !{!"nvvm.l2_cache_hint", i64 12345}
+!16 = !{i32 0, !40}
+!40 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
+!17 = !{i32 0, !41}
+!41 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_eviction", !"last"}
+!18 = !{i32 0, !42}
+!42 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_prefetch_size", !"128B"}
+!19 = !{i32 0, !43}
+!43 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
+!20 = !{i32 1, !44}
+!44 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"unchanged"}
+!21 = !{i32 1, !45}
+!45 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
+!22 = !{i32 0, !46}
+!46 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
+!23 = !{i32 1, !47}
+!47 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"64B"}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll b/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll
new file mode 100644
index 0000000000000..6919b6b805ed6
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll
@@ -0,0 +1,615 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; Test !mem.cache_hint metadata on LLVM memory intrinsics.
+;
+; For memcpy:
+; operand_no = 0 applies to destination (store side)
+; operand_no = 1 applies to source (load side)
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1), ptr addrspace(1), i64, i1)
+declare <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1), <2 x i1>, <2 x i16>)
+declare <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1), <4 x i1>, <4 x i32>)
+
+;-----------------------------------------------------------------------------
+; Test memcpy with cache hints on both source and destination
+; Source (operand 1): L1::evict_first, L2::evict_first, L2::128B
+; Dest (operand 0): L1::evict_last, L2::evict_last
+;-----------------------------------------------------------------------------
+
+define void @test_memcpy_both_hints(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_both_hints(
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !80
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy with cache hint only on source (load side)
+; Source (operand 1): L1::evict_first
+; Dest (operand 0): no hint
+;-----------------------------------------------------------------------------
+
+define void @test_memcpy_src_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_src_hint_only(
+; CHECK: ld.global.L1::evict_first.b8 %rs1, [%rd2+3];
+; CHECK: st.global.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_first.b8 %rs2, [%rd2+2];
+; CHECK: st.global.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_first.b8 %rs3, [%rd2+1];
+; CHECK: st.global.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_first.b8 %rs4, [%rd2];
+; CHECK: st.global.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !81
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy with cache hint only on destination (store side)
+; Source (operand 1): no hint
+; Dest (operand 0): L2::evict_last
+;-----------------------------------------------------------------------------
+
+define void @test_memcpy_dest_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_dest_hint_only(
+; CHECK: ld.global.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L2::evict_last.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L2::evict_last.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L2::evict_last.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.b8 %rs4, [%rd2];
+; CHECK: st.global.L2::evict_last.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !82
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy with L2::cache_hint on both operands
+;-----------------------------------------------------------------------------
+
+define void @test_memcpy_l2_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_l2_cache_hint(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.L2::cache_hint.b8 [%rd1+3], %rs1, %rd2;
+; CHECK: ld.global.L2::cache_hint.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.L2::cache_hint.b8 [%rd1+2], %rs2, %rd2;
+; CHECK: ld.global.L2::cache_hint.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.L2::cache_hint.b8 [%rd1+1], %rs3, %rd2;
+; CHECK: ld.global.L2::cache_hint.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.L2::cache_hint.b8 [%rd1], %rs4, %rd2;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !83
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Test memcpy without cache hints produces plain load/store
+;-----------------------------------------------------------------------------
+
+define void @test_memcpy_no_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_no_hint(
+; CHECK: ld.global.b8 %rs1, [%rd2+3];
+; CHECK: st.global.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.b8 %rs2, [%rd2+2];
+; CHECK: st.global.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.b8 %rs3, [%rd2+1];
+; CHECK: st.global.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.b8 %rs4, [%rd2];
+; CHECK: st.global.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false)
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Combined L1 + L2 eviction policies
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_first + L2::evict_last
+; Dest: no hint
+define void @test_memcpy_src_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_src_l1_l2_combined(
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs1, [%rd2+3];
+; CHECK: st.global.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs2, [%rd2+2];
+; CHECK: st.global.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs3, [%rd2+1];
+; CHECK: st.global.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs4, [%rd2];
+; CHECK: st.global.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !84
+ ret void
+}
+
+; Source: no hint
+; Dest: L1::evict_unchanged + L2::evict_first
+define void @test_memcpy_dest_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_dest_l1_l2_combined(
+; CHECK: ld.global.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !85
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L1 + prefetch combinations
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_last + L2::256B prefetch
+; Dest: L1::no_allocate
+define void @test_memcpy_l1_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_l1_prefetch(
+; CHECK: ld.global.L1::evict_last.L2::256B.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::no_allocate.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_last.L2::256B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::no_allocate.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_last.L2::256B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::no_allocate.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_last.L2::256B.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::no_allocate.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !86
+ ret void
+}
+
+; Source: L2::64B prefetch only
+; Dest: L2::evict_last only
+define void @test_memcpy_prefetch_vs_eviction(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_prefetch_vs_eviction(
+; CHECK: ld.global.L2::64B.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L2::evict_last.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L2::64B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L2::evict_last.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L2::64B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L2::evict_last.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L2::64B.b8 %rs4, [%rd2];
+; CHECK: st.global.L2::evict_last.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !87
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint combined with other hints
+;-----------------------------------------------------------------------------
+
+; Source: L2::cache_hint + L1::evict_first
+; Dest: no hint
+define void @test_memcpy_src_cache_hint_l1(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_src_cache_hint_l1(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !88
+ ret void
+}
+
+; Source: no hint
+; Dest: L2::cache_hint + L1::evict_last + L2::evict_first
+define void @test_memcpy_dest_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_dest_cache_hint_combined(
+; CHECK: ld.global.b8 %rs1, [%rd2+3];
+; CHECK: mov.b64 %rd3, 12345;
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1+3], %rs1, %rd3;
+; CHECK: ld.global.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1+2], %rs2, %rd3;
+; CHECK: ld.global.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1+1], %rs3, %rd3;
+; CHECK: ld.global.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1], %rs4, %rd3;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !89
+ ret void
+}
+
+; Both operands: L2::cache_hint + L1 eviction + L2 eviction
+; Source: L2::cache_hint + L1::evict_unchanged + L2::evict_last
+; Dest: L2::cache_hint + L1::evict_first + L2::evict_first
+define void @test_memcpy_both_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_both_cache_hint_combined(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1+3], %rs1, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1+2], %rs2, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1+1], %rs3, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1], %rs4, %rd2;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !90
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint + prefetch combinations
+;-----------------------------------------------------------------------------
+
+; Source: L2::cache_hint + L2::128B prefetch
+; Dest: L2::cache_hint + L1::evict_last
+define void @test_memcpy_cache_hint_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_cache_hint_prefetch(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.L2::128B.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+3], %rs1, %rd2;
+; CHECK: ld.global.L2::cache_hint.L2::128B.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+2], %rs2, %rd2;
+; CHECK: ld.global.L2::cache_hint.L2::128B.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+1], %rs3, %rd2;
+; CHECK: ld.global.L2::cache_hint.L2::128B.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1], %rs4, %rd2;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !91
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Asymmetric hint combinations (complex vs simple)
+;-----------------------------------------------------------------------------
+
+; Source: all hints (L1 + L2 eviction + prefetch)
+; Dest: simple L1 hint only
+define void @test_memcpy_complex_src_simple_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_complex_src_simple_dest(
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_last.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !92
+ ret void
+}
+
+; Source: simple L2 prefetch only
+; Dest: all hints (L1 + L2 eviction + L2::cache_hint)
+define void @test_memcpy_simple_src_complex_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_simple_src_complex_dest(
+; CHECK: ld.global.L2::256B.b8 %rs1, [%rd2+3];
+; CHECK: mov.b64 %rd3, 12345;
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1+3], %rs1, %rd3;
+; CHECK: ld.global.L2::256B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1+2], %rs2, %rd3;
+; CHECK: ld.global.L2::256B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1+1], %rs3, %rd3;
+; CHECK: ld.global.L2::256B.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1], %rs4, %rd3;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !93
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Different L1 eviction policies on src vs dest
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_unchanged
+; Dest: L1::evict_first
+define void @test_memcpy_different_l1_policies(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_different_l1_policies(
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::evict_first.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_first.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_first.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_first.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !94
+ ret void
+}
+
+; Source: L1::no_allocate
+; Dest: L1::evict_unchanged
+define void @test_memcpy_no_allocate_vs_unchanged(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_no_allocate_vs_unchanged(
+; CHECK: ld.global.L1::no_allocate.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::no_allocate.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::no_allocate.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::no_allocate.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1], %rs4;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !95
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; All hints maxed out on both operands
+;-----------------------------------------------------------------------------
+
+; Source: L1::evict_first + L2::evict_first + L2::256B + L2::cache_hint
+; Dest: L1::evict_last + L2::evict_last + L2::128B + L2::cache_hint
+define void @test_memcpy_all_hints_both(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_all_hints_both(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1+3], %rs1, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1+2], %rs2, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1+1], %rs3, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1], %rs4, %rd2;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !96
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; TODO: Preserve cache hints on llvm.masked.load.
+; Masked load pointer operand is operand 0. SelectionDAGBuilder::visitMaskedLoad
+; does not currently thread !mem.cache_hint into the MachineMemOperand, so the
+; generated loads are plain today.
+;-----------------------------------------------------------------------------
+
+; TODO: This should eventually lower to ld.global.L1::evict_first.b32.
+define <2 x i16> @test_masked_load_l1_hint_v2i16(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_masked_load_l1_hint_v2i16(
+; CHECK: ld.global.b32 %r1, [%rd1];
+ %v = call <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1) align 4 %p, <2 x i1> <i1 true, i1 false>, <2 x i16> poison), !mem.cache_hint !101
+ ret <2 x i16> %v
+}
+
+; TODO: This should eventually lower to ld.global.L2::cache_hint.v4.b32.
+define <4 x i32> @test_masked_load_l2_cache_policy_v4i32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_masked_load_l2_cache_policy_v4i32(
+; CHECK: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+ %v = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 %p, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> poison), !mem.cache_hint !102
+ ret <4 x i32> %v
+}
+
+;-----------------------------------------------------------------------------
+; Large memcpy tests - verify hints propagate to all expanded load/stores
+; LLVM expands memcpy to multiple load/store pairs. Each pair should
+; get the appropriate cache hints from the original memcpy metadata.
+; The expansion may use various sizes (b8, b16, b32, v2, v4, etc.)
+;-----------------------------------------------------------------------------
+
+; 16-byte memcpy: verify hints are applied to expanded loads/stores
+define void @test_memcpy_16bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_16bytes(
+; CHECK: ld.global.L1::evict_first.b8 %rs1, [%rd2+15];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+15], %rs1;
+; CHECK: ld.global.L1::evict_first.b8 %rs2, [%rd2+14];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+14], %rs2;
+; CHECK: ld.global.L1::evict_first.b8 %rs3, [%rd2+13];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+13], %rs3;
+; CHECK: ld.global.L1::evict_first.b8 %rs4, [%rd2+12];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+12], %rs4;
+; CHECK: ld.global.L1::evict_first.b8 %rs5, [%rd2+11];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+11], %rs5;
+; CHECK: ld.global.L1::evict_first.b8 %rs6, [%rd2+10];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+10], %rs6;
+; CHECK: ld.global.L1::evict_first.b8 %rs7, [%rd2+9];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+9], %rs7;
+; CHECK: ld.global.L1::evict_first.b8 %rs8, [%rd2+8];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+8], %rs8;
+; CHECK: ld.global.L1::evict_first.b8 %rs9, [%rd2+7];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+7], %rs9;
+; CHECK: ld.global.L1::evict_first.b8 %rs10, [%rd2+6];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+6], %rs10;
+; CHECK: ld.global.L1::evict_first.b8 %rs11, [%rd2+5];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+5], %rs11;
+; CHECK: ld.global.L1::evict_first.b8 %rs12, [%rd2+4];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+4], %rs12;
+; CHECK: ld.global.L1::evict_first.b8 %rs13, [%rd2+3];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+3], %rs13;
+; CHECK: ld.global.L1::evict_first.b8 %rs14, [%rd2+2];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+2], %rs14;
+; CHECK: ld.global.L1::evict_first.b8 %rs15, [%rd2+1];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+1], %rs15;
+; CHECK: ld.global.L1::evict_first.b8 %rs16, [%rd2];
+; CHECK: st.global.L1::evict_last.b8 [%rd1], %rs16;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 16, i1 false), !mem.cache_hint !97
+ ret void
+}
+
+; 32-byte memcpy: all loads should have L1::evict_unchanged, all stores L2::evict_first
+define void @test_memcpy_32bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_32bytes(
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs1, [%rd2+31];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+31], %rs1;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs2, [%rd2+30];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+30], %rs2;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs3, [%rd2+29];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+29], %rs3;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs4, [%rd2+28];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+28], %rs4;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs5, [%rd2+27];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+27], %rs5;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs6, [%rd2+26];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+26], %rs6;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs7, [%rd2+25];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+25], %rs7;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs8, [%rd2+24];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+24], %rs8;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs9, [%rd2+23];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+23], %rs9;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs10, [%rd2+22];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+22], %rs10;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs11, [%rd2+21];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+21], %rs11;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs12, [%rd2+20];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+20], %rs12;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs13, [%rd2+19];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+19], %rs13;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs14, [%rd2+18];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+18], %rs14;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs15, [%rd2+17];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+17], %rs15;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs16, [%rd2+16];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+16], %rs16;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs17, [%rd2+15];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+15], %rs17;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs18, [%rd2+14];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+14], %rs18;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs19, [%rd2+13];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+13], %rs19;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs20, [%rd2+12];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+12], %rs20;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs21, [%rd2+11];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+11], %rs21;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs22, [%rd2+10];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+10], %rs22;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs23, [%rd2+9];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+9], %rs23;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs24, [%rd2+8];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+8], %rs24;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs25, [%rd2+7];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+7], %rs25;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs26, [%rd2+6];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+6], %rs26;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs27, [%rd2+5];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+5], %rs27;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs28, [%rd2+4];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+4], %rs28;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs29, [%rd2+3];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+3], %rs29;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs30, [%rd2+2];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+2], %rs30;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs31, [%rd2+1];
+; CHECK: st.global.L2::evict_first.b8 [%rd1+1], %rs31;
+; CHECK: ld.global.L1::evict_unchanged.b8 %rs32, [%rd2];
+; CHECK: st.global.L2::evict_first.b8 [%rd1], %rs32;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 32, i1 false), !mem.cache_hint !98
+ ret void
+}
+
+; 128-byte memcpy with combined hints
+; Note: Large memcpy (>64 bytes) is expanded to a loop in the backend.
+; Cache hints are not preserved for loop-based expansion.
+; This test verifies the code compiles correctly.
+define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_128bytes_combined(
+; CHECK: mov.b64 %rd5, 0;
+; CHECK: ld.global.b8 %rs1, [%rd3];
+; CHECK: st.global.b8 [%rd4], %rs1;
+ call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 128, i1 false), !mem.cache_hint !100
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+; memcpy with both dest and src hints
+!80 = !{i32 0, !180, i32 1, !181}
+; operand 0 (dest/store): L1::evict_last, L2::evict_last
+!180 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last"}
+; operand 1 (src/load): L1::evict_first, L2::evict_first, L2::128B prefetch
+!181 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"128B"}
+
+; memcpy with only source hint (load side)
+!81 = !{i32 1, !182}
+!182 = !{!"nvvm.l1_eviction", !"first"}
+
+; memcpy with only dest hint (store side)
+!82 = !{i32 0, !183}
+!183 = !{!"nvvm.l2_eviction", !"last"}
+
+; memcpy with L2::cache_hint on both operands
+!83 = !{i32 0, !184, i32 1, !185}
+!184 = !{!"nvvm.l2_cache_hint", i64 12345}
+!185 = !{!"nvvm.l2_cache_hint", i64 12345}
+
+; Combined L1 + L2 eviction on source only
+!84 = !{i32 1, !186}
+!186 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last"}
+
+; Combined L1 + L2 eviction on dest only
+!85 = !{i32 0, !187}
+!187 = !{!"nvvm.l1_eviction", !"unchanged", !"nvvm.l2_eviction", !"first"}
+
+; L1 + prefetch on source, L1 on dest
+!86 = !{i32 0, !189, i32 1, !188}
+!188 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
+!189 = !{!"nvvm.l1_eviction", !"no_allocate"}
+
+; Prefetch on source, L2 eviction on dest
+!87 = !{i32 0, !191, i32 1, !190}
+!190 = !{!"nvvm.l2_prefetch_size", !"64B"}
+!191 = !{!"nvvm.l2_eviction", !"last"}
+
+; L2::cache_hint + L1 eviction on source only
+!88 = !{i32 1, !192}
+!192 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
+
+; L2::cache_hint + L1 + L2 eviction on dest only
+!89 = !{i32 0, !193}
+!193 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+
+; Both operands: L2::cache_hint + L1 + L2 eviction
+!90 = !{i32 0, !195, i32 1, !194}
+!194 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"unchanged", !"nvvm.l2_eviction", !"last"}
+!195 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first"}
+
+; L2::cache_hint + prefetch on source, L2::cache_hint + L1 on dest
+!91 = !{i32 0, !197, i32 1, !196}
+!196 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_prefetch_size", !"128B"}
+!197 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last"}
+
+; Complex source (all non-cache_hint), simple dest
+!92 = !{i32 0, !199, i32 1, !198}
+!198 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"64B"}
+!199 = !{!"nvvm.l1_eviction", !"last"}
+
+; Simple source, complex dest (with cache_hint)
+!93 = !{i32 0, !201, i32 1, !200}
+!200 = !{!"nvvm.l2_prefetch_size", !"256B"}
+!201 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
+
+; Different L1 policies: unchanged vs first
+!94 = !{i32 0, !203, i32 1, !202}
+!202 = !{!"nvvm.l1_eviction", !"unchanged"}
+!203 = !{!"nvvm.l1_eviction", !"first"}
+
+; Different L1 policies: no_allocate vs unchanged
+!95 = !{i32 0, !205, i32 1, !204}
+!204 = !{!"nvvm.l1_eviction", !"no_allocate"}
+!205 = !{!"nvvm.l1_eviction", !"unchanged"}
+
+; All hints maxed out on both operands
+!96 = !{i32 0, !207, i32 1, !206}
+!206 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
+!207 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
+
+; Large memcpy metadata
+!97 = !{i32 0, !209, i32 1, !208}
+!208 = !{!"nvvm.l1_eviction", !"first"}
+!209 = !{!"nvvm.l1_eviction", !"last"}
+
+!98 = !{i32 0, !211, i32 1, !210}
+!210 = !{!"nvvm.l1_eviction", !"unchanged"}
+!211 = !{!"nvvm.l2_eviction", !"first"}
+
+!99 = !{i32 0, !213, i32 1, !212}
+!212 = !{!"nvvm.l2_cache_hint", i64 12345}
+!213 = !{!"nvvm.l2_cache_hint", i64 12345}
+
+!100 = !{i32 0, !215, i32 1, !214}
+!214 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
+!215 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+
+; Masked load metadata
+!101 = !{i32 0, !216}
+!216 = !{!"nvvm.l1_eviction", !"first"}
+
+!102 = !{i32 0, !217}
+!217 = !{!"nvvm.l2_cache_hint", i64 12345}
diff --git a/llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll b/llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll
similarity index 100%
rename from llvm/test/CodeGen/NVPTX/invalid-cache-hint.ll
rename to llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll b/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll
new file mode 100644
index 0000000000000..2e715ad316ca0
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll
@@ -0,0 +1,408 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; Test !mem.cache_hint metadata lowering to PTX load/store cache qualifiers.
+
+;-----------------------------------------------------------------------------
+; Basic L1 eviction policies for loads
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_l1_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_first(
+; CHECK: ld.global.L1::evict_first.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+define i32 @test_load_l1_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_last(
+; CHECK: ld.global.L1::evict_last.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
+ ret i32 %v
+}
+
+define i32 @test_load_l1_unchanged(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_unchanged(
+; CHECK: ld.global.L1::evict_unchanged.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
+ ret i32 %v
+}
+
+define i32 @test_load_l1_no_allocate(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_no_allocate(
+; CHECK: ld.global.L1::no_allocate.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Basic L2 eviction policies for loads
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_l2_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l2_first(
+; CHECK: ld.global.L2::evict_first.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
+ ret i32 %v
+}
+
+define i32 @test_load_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l2_last(
+; CHECK: ld.global.L2::evict_last.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L2 prefetch sizes for loads
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_prefetch_64(
+; CHECK: ld.global.L2::64B.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
+ ret i32 %v
+}
+
+define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_prefetch_128(
+; CHECK: ld.global.L2::128B.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret i32 %v
+}
+
+define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_prefetch_256(
+; CHECK: ld.global.L2::256B.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; All L1 + L2 combinations for loads
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_l1_first_l2_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_first_l2_first(
+; CHECK: ld.global.L1::evict_first.L2::evict_first.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !9
+ ret i32 %v
+}
+
+define i32 @test_load_l1_first_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_first_l2_last(
+; CHECK: ld.global.L1::evict_first.L2::evict_last.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !10
+ ret i32 %v
+}
+
+define i32 @test_load_l1_last_l2_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_last_l2_first(
+; CHECK: ld.global.L1::evict_last.L2::evict_first.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !11
+ ret i32 %v
+}
+
+define i32 @test_load_l1_last_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_last_l2_last(
+; CHECK: ld.global.L1::evict_last.L2::evict_last.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !12
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; L1 + L2 + Prefetch combination for loads
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_l1_first_l2_last_prefetch_128(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_first_l2_last_prefetch_128(
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::128B.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !13
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Basic L1 eviction policies for stores
+;-----------------------------------------------------------------------------
+
+define void @test_store_l1_first(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_first(
+; CHECK: st.global.L1::evict_first.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !14
+ ret void
+}
+
+define void @test_store_l1_last(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_last(
+; CHECK: st.global.L1::evict_last.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !15
+ ret void
+}
+
+define void @test_store_l1_unchanged(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_unchanged(
+; CHECK: st.global.L1::evict_unchanged.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !16
+ ret void
+}
+
+define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_no_allocate(
+; CHECK: st.global.L1::no_allocate.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !17
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Basic L2 eviction policies for stores
+;-----------------------------------------------------------------------------
+
+define void @test_store_l2_first(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l2_first(
+; CHECK: st.global.L2::evict_first.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !18
+ ret void
+}
+
+define void @test_store_l2_last(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l2_last(
+; CHECK: st.global.L2::evict_last.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !19
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; All L1 + L2 combinations for stores
+;-----------------------------------------------------------------------------
+
+define void @test_store_l1_first_l2_first(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_first_l2_first(
+; CHECK: st.global.L1::evict_first.L2::evict_first.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !20
+ ret void
+}
+
+define void @test_store_l1_first_l2_last(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_first_l2_last(
+; CHECK: st.global.L1::evict_first.L2::evict_last.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !21
+ ret void
+}
+
+define void @test_store_l1_last_l2_first(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_last_l2_first(
+; CHECK: st.global.L1::evict_last.L2::evict_first.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !22
+ ret void
+}
+
+define void @test_store_l1_last_l2_last(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_l1_last_l2_last(
+; CHECK: st.global.L1::evict_last.L2::evict_last.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !23
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Different data types - loads
+;-----------------------------------------------------------------------------
+
+define i16 @test_load_i16_l1_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_i16_l1_first(
+; CHECK: ld.global.L1::evict_first.b16 %r1, [%rd1];
+ %v = load i16, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret i16 %v
+}
+
+define i64 @test_load_i64_l1_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_i64_l1_last(
+; CHECK: ld.global.L1::evict_last.b64 %rd2, [%rd1];
+ %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !1
+ ret i64 %v
+}
+
+define float @test_load_f32_l2_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_f32_l2_first(
+; CHECK: ld.global.L2::evict_first.b32 %r1, [%rd1];
+ %v = load float, ptr addrspace(1) %p, !mem.cache_hint !4
+ ret float %v
+}
+
+define double @test_load_f64_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_f64_l2_last(
+; CHECK: ld.global.L2::evict_last.b64 %rd2, [%rd1];
+ %v = load double, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret double %v
+}
+
+;-----------------------------------------------------------------------------
+; Different data types - stores
+;-----------------------------------------------------------------------------
+
+define void @test_store_i16_l1_first(ptr addrspace(1) %p, i16 %v) {
+; CHECK-LABEL: test_store_i16_l1_first(
+; CHECK: st.global.L1::evict_first.b16 [%rd1], %rs1;
+ store i16 %v, ptr addrspace(1) %p, !mem.cache_hint !14
+ ret void
+}
+
+define void @test_store_i64_l2_last(ptr addrspace(1) %p, i64 %v) {
+; CHECK-LABEL: test_store_i64_l2_last(
+; CHECK: st.global.L2::evict_last.b64 [%rd1], %rd2;
+ store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !19
+ ret void
+}
+
+define void @test_store_f32_l1_no_allocate(ptr addrspace(1) %p, float %v) {
+; CHECK-LABEL: test_store_f32_l1_no_allocate(
+; CHECK: st.global.L1::no_allocate.b32 [%rd1], %r1;
+ store float %v, ptr addrspace(1) %p, !mem.cache_hint !17
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Vector loads with cache hints
+;-----------------------------------------------------------------------------
+
+define <2 x i32> @test_load_v2i32_l1_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v2i32_l1_first(
+; CHECK: ld.global.L1::evict_first.v2.b32 {%r1, %r2}, [%rd1];
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !0
+ ret <2 x i32> %v
+}
+
+define <4 x i32> @test_load_v4i32_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v4i32_l2_last(
+; CHECK: ld.global.L2::evict_last.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+ %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !5
+ ret <4 x i32> %v
+}
+
+define <2 x float> @test_load_v2f32_l1_unchanged(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v2f32_l1_unchanged(
+; CHECK: ld.global.L1::evict_unchanged.v2.b32 {%r1, %r2}, [%rd1];
+ %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !2
+ ret <2 x float> %v
+}
+
+define <2 x double> @test_load_v2f64_prefetch_128(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v2f64_prefetch_128(
+; CHECK: ld.global.L2::128B.v2.b64 {%rd2, %rd3}, [%rd1];
+ %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret <2 x double> %v
+}
+
+;-----------------------------------------------------------------------------
+; Vector stores with cache hints
+;-----------------------------------------------------------------------------
+
+define void @test_store_v2i32_l1_last(ptr addrspace(1) %p, <2 x i32> %v) {
+; CHECK-LABEL: test_store_v2i32_l1_last(
+; CHECK: st.global.L1::evict_last.v2.b32 [%rd1], {%r1, %r2};
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !15
+ ret void
+}
+
+define void @test_store_v4i32_l2_first(ptr addrspace(1) %p, <4 x i32> %v) {
+; CHECK-LABEL: test_store_v4i32_l2_first(
+; CHECK: st.global.L2::evict_first.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
+ store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !18
+ ret void
+}
+
+define void @test_store_v2f64_l1_no_allocate(ptr addrspace(1) %p, <2 x double> %v) {
+; CHECK-LABEL: test_store_v2f64_l1_no_allocate(
+; CHECK: st.global.L1::no_allocate.v2.b64 [%rd1], {%rd2, %rd3};
+ store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !17
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Invariant loads with cache hints may still use LDG (ld.global.nc)
+;-----------------------------------------------------------------------------
+
+define i32 @test_invariant_load_with_hint(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_invariant_load_with_hint(
+; CHECK: ld.global.nc.L1::evict_first.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !0
+ ret i32 %v
+}
+
+define i32 @test_invariant_load_with_cache_policy(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_invariant_load_with_cache_policy(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.nc.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !24
+ ret i32 %v
+}
+
+define <2 x i32> @test_invariant_load_v2i32_with_hint(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_invariant_load_v2i32_with_hint(
+; CHECK: ld.global.nc.L1::evict_last.L2::evict_first.v2.b32 {%r1, %r2}, [%rd1];
+ %v = load <2 x i32>, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !11
+ ret <2 x i32> %v
+}
+
+;-----------------------------------------------------------------------------
+; No hint should produce plain load/store
+;-----------------------------------------------------------------------------
+
+define i32 @test_load_no_hint(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_no_hint(
+; CHECK: ld.global.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p
+ ret i32 %v
+}
+
+define void @test_store_no_hint(ptr addrspace(1) %p, i32 %v) {
+; CHECK-LABEL: test_store_no_hint(
+; CHECK: st.global.b32 [%rd1], %r1;
+ store i32 %v, ptr addrspace(1) %p
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+!0 = !{i32 0, !25}
+!25 = !{!"nvvm.l1_eviction", !"first"}
+!1 = !{i32 0, !26}
+!26 = !{!"nvvm.l1_eviction", !"last"}
+!2 = !{i32 0, !27}
+!27 = !{!"nvvm.l1_eviction", !"unchanged"}
+!3 = !{i32 0, !28}
+!28 = !{!"nvvm.l1_eviction", !"no_allocate"}
+!4 = !{i32 0, !29}
+!29 = !{!"nvvm.l2_eviction", !"first"}
+!5 = !{i32 0, !30}
+!30 = !{!"nvvm.l2_eviction", !"last"}
+!6 = !{i32 0, !31}
+!31 = !{!"nvvm.l2_prefetch_size", !"64B"}
+!7 = !{i32 0, !32}
+!32 = !{!"nvvm.l2_prefetch_size", !"128B"}
+!8 = !{i32 0, !33}
+!33 = !{!"nvvm.l2_prefetch_size", !"256B"}
+!9 = !{i32 0, !34}
+!34 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first"}
+!10 = !{i32 0, !35}
+!35 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last"}
+!11 = !{i32 0, !36}
+!36 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+!12 = !{i32 0, !37}
+!37 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last"}
+!13 = !{i32 0, !38}
+!38 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
+!14 = !{i32 1, !25}
+!15 = !{i32 1, !26}
+!16 = !{i32 1, !27}
+!17 = !{i32 1, !28}
+!18 = !{i32 1, !29}
+!19 = !{i32 1, !30}
+!20 = !{i32 1, !34}
+!21 = !{i32 1, !35}
+!22 = !{i32 1, !36}
+!23 = !{i32 1, !37}
+!24 = !{i32 0, !39}
+!39 = !{!"nvvm.l2_cache_hint", i64 12345}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
index cbd57fbf796d4..c4748c8001742 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(ld\.global|st\.global|mov\.b64)" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM60
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM70
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM75
@@ -184,7 +184,7 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
; SM75: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
; SM80PLUS-LABEL: test_load_cache_hint_with_l1(
-; SM80PLUS: mov.b64 %rd2, 44445;
+; SM80PLUS: mov.b64 %rd2, 12345;
; SM80PLUS: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
;
; SM80-PTX70-LABEL: test_load_cache_hint_with_l1(
@@ -234,7 +234,7 @@ define void @test_store_cache_hint(ptr addrspace(1) %p, i32 %v) {
; SM75: st.global.b32 [%rd1], %r1;
;
; SM80PLUS-LABEL: test_store_cache_hint(
-; SM80PLUS: mov.b64 %rd2, 67890;
+; SM80PLUS: mov.b64 %rd2, 12345;
; SM80PLUS: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
;
; SM80-PTX70-LABEL: test_store_cache_hint(
@@ -288,11 +288,11 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
; L2::cache_hint + L1 eviction
!4 = !{i32 0, !104}
-!104 = !{!"nvvm.l2_cache_hint", i64 44445, !"nvvm.l1_eviction", !"first"}
+!104 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
; L2::cache_hint for store
!5 = !{i32 1, !105}
-!105 = !{!"nvvm.l2_cache_hint", i64 67890}
+!105 = !{!"nvvm.l2_cache_hint", i64 12345}
; L2 prefetch: 64B
!6 = !{i32 0, !106}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
new file mode 100644
index 0000000000000..fb54c0a5cb4d2
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global|ld\.param\.b32)" --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+
+; Test cache hint handling across shared pointers, CSE, forwarding, and legalization.
+
+;-----------------------------------------------------------------------------
+; Multiple loads sharing same pointer
+;-----------------------------------------------------------------------------
+
+; Two volatile loads from the same pointer should each keep their own cache hint
+; combination, even when they use the same cache-policy value.
+define i32 @test_multiple_loads_same_ptr(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_multiple_loads_same_ptr(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.volatile.global.L1::evict_last.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+; CHECK: ld.volatile.global.L1::evict_first.L2::cache_hint.b32 %r2, [%rd1], %rd2;
+ %v1 = load volatile i32, ptr addrspace(1) %p, !mem.cache_hint !0
+ %v2 = load volatile i32, ptr addrspace(1) %p, !mem.cache_hint !1
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; Non-volatile loads can CSE. Matching cache hints are preserved on the merged
+; DAG node, but conflicting hints are dropped.
+define i32 @test_cse_loads_same_cache_hint(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_cse_loads_same_cache_hint(
+; CHECK: ld.global.L1::evict_first.b32 %r1, [%rd1];
+ %v1 = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
+ %v2 = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+define i32 @test_cse_loads_conflicting_cache_hints(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_cse_loads_conflicting_cache_hints(
+; CHECK: ld.global.b32 %r1, [%rd1];
+ %v1 = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
+ %v2 = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
+ %sum = add i32 %v1, %v2
+ ret i32 %sum
+}
+
+; NVPTXForwardParams rewrites eligible byval loads to ld.param. Since cache
+; hints are not allowed on ld.param, we must drop them.
+define i32 @test_forward_param_drops_l1_hint(ptr byval(i32) %a) {
+; CHECK-LABEL: test_forward_param_drops_l1_hint(
+; CHECK: ld.param.b32 %r1, [test_forward_param_drops_l1_hint_param_0];
+ %v = load i32, ptr %a, !mem.cache_hint !4
+ ret i32 %v
+}
+
+define i32 @test_forward_param_drops_l2_cache_hint(ptr byval(i32) %a) {
+; CHECK-LABEL: test_forward_param_drops_l2_cache_hint(
+; CHECK: ld.param.b32 %r1, [test_forward_param_drops_l2_cache_hint_param_0];
+ %v = load i32, ptr %a, !mem.cache_hint !5
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; Valid edge cases
+;-----------------------------------------------------------------------------
+
+; Test with custom hint key order - should still work
+define i32 @test_load_reordered_metadata(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_reordered_metadata(
+; CHECK: ld.global.L1::evict_last.L2::evict_first.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; nvvm.l2_cache_hint with alternate integer value width
+;-----------------------------------------------------------------------------
+
+; nvvm.l2_cache_hint with i32 instead of i64 - should still work
+; as mdconst::dyn_extract<ConstantInt> accepts any integer type
+define i32 @test_load_cache_hint_i32_value(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_cache_hint_i32_value(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
+ ret i32 %v
+}
+
+; Test "normal" eviction - should not emit any qualifier (default behavior)
+define i32 @test_load_l1_normal(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l1_normal(
+; CHECK: ld.global.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
+ ret i32 %v
+}
+
+define i32 @test_load_l2_normal(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_l2_normal(
+; CHECK: ld.global.b32 %r1, [%rd1];
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !9
+ ret i32 %v
+}
+
+;-----------------------------------------------------------------------------
+; TODO: Preserve cache hints across DAGCombiner-created memory rewrites.
+; This documents the current store-of-concat-trunc behavior: copied MMOs for
+; the split stores do not retain !mem.cache_hint metadata yet.
+;-----------------------------------------------------------------------------
+
+define void @test_dagcombine_store_concat_trunc_v8i32(ptr addrspace(1) %p, <4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: test_dagcombine_store_concat_trunc_v8i32(
+; CHECK: st.global.v4.b32 [%rd1+16], {%r8, %r7, %r6, %r5};
+; CHECK: st.global.v4.b32 [%rd1], {%r4, %r3, %r2, %r1};
+ %ta = trunc <4 x i64> %a to <4 x i32>
+ %tb = trunc <4 x i64> %b to <4 x i32>
+ %c = shufflevector <4 x i32> %ta, <4 x i32> %tb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ store <8 x i32> %c, ptr addrspace(1) %p, align 16, !mem.cache_hint !10
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; TODO: Preserve cache hints across one-to-N DAG memory rewrites.
+; These tests document the current split/scalarized behavior: newly-created
+; memory ops do not retain !mem.cache_hint metadata yet.
+;-----------------------------------------------------------------------------
+
+define <16 x i32> @test_legalize_split_load_v16i32(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_legalize_split_load_v16i32(
+; CHECK: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK: ld.global.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; CHECK: ld.global.v4.b32 {%r9, %r10, %r11, %r12}, [%rd1+32];
+; CHECK: ld.global.v4.b32 {%r13, %r14, %r15, %r16}, [%rd1+48];
+ %v = load <16 x i32>, ptr addrspace(1) %p, align 16, !mem.cache_hint !11
+ ret <16 x i32> %v
+}
+
+define void @test_legalize_split_store_v16i32(ptr addrspace(1) %p, <16 x i32> %v) {
+; CHECK-LABEL: test_legalize_split_store_v16i32(
+; CHECK: st.global.v4.b32 [%rd1+48], {%r1, %r2, %r3, %r4};
+; CHECK: st.global.v4.b32 [%rd1+32], {%r5, %r6, %r7, %r8};
+; CHECK: st.global.v4.b32 [%rd1+16], {%r9, %r10, %r11, %r12};
+; CHECK: st.global.v4.b32 [%rd1], {%r13, %r14, %r15, %r16};
+ store <16 x i32> %v, ptr addrspace(1) %p, align 16, !mem.cache_hint !12
+ ret void
+}
+
+define <3 x i64> @test_legalize_scalarize_load_v3i64(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_legalize_scalarize_load_v3i64(
+; CHECK: ld.global.b64 %rd2, [%rd1+16];
+; CHECK: ld.global.b64 %rd3, [%rd1+8];
+; CHECK: ld.global.b64 %rd4, [%rd1];
+ %v = load <3 x i64>, ptr addrspace(1) %p, align 8, !mem.cache_hint !13
+ ret <3 x i64> %v
+}
+
+define void @test_legalize_scalarize_store_v3i64(ptr addrspace(1) %p, <3 x i64> %v) {
+; CHECK-LABEL: test_legalize_scalarize_store_v3i64(
+; CHECK: st.global.b64 [%rd1+16], %rd2;
+; CHECK: st.global.b64 [%rd1+8], %rd4;
+; CHECK: st.global.b64 [%rd1], %rd3;
+ store <3 x i64> %v, ptr addrspace(1) %p, align 8, !mem.cache_hint !14
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; Metadata definitions
+;-----------------------------------------------------------------------------
+
+!0 = !{i32 0, !15}
+!15 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last"}
+!1 = !{i32 0, !16}
+!16 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
+!2 = !{i32 0, !17}
+!17 = !{!"nvvm.l1_eviction", !"first"}
+!3 = !{i32 0, !18}
+!18 = !{!"nvvm.l1_eviction", !"last"}
+!4 = !{i32 0, !19}
+!19 = !{!"nvvm.l1_eviction", !"first"}
+!5 = !{i32 0, !20}
+!20 = !{!"nvvm.l2_cache_hint", i64 12345}
+!6 = !{i32 0, !21}
+!21 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
+!7 = !{i32 0, !22}
+!22 = !{!"nvvm.l2_cache_hint", i32 12345}
+!8 = !{i32 0, !23}
+!23 = !{!"nvvm.l1_eviction", !"normal"}
+!9 = !{i32 0, !24}
+!24 = !{!"nvvm.l2_eviction", !"normal"}
+!10 = !{i32 1, !25}
+!25 = !{!"nvvm.l2_cache_hint", i64 12345}
+!11 = !{i32 0, !26}
+!26 = !{!"nvvm.l2_cache_hint", i64 12345}
+!12 = !{i32 1, !27}
+!27 = !{!"nvvm.l2_cache_hint", i64 12345}
+!13 = !{i32 0, !28}
+!28 = !{!"nvvm.l1_eviction", !"last"}
+!14 = !{i32 1, !29}
+!29 = !{!"nvvm.l2_eviction", !"first"}
diff --git a/llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll b/llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll
deleted file mode 100644
index 9f05c0d1ce9d8..0000000000000
--- a/llvm/test/CodeGen/NVPTX/intrinsics-cache-hint.ll
+++ /dev/null
@@ -1,444 +0,0 @@
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
-
-; Test !mem.cache_hint metadata on LLVM memory intrinsics.
-;
-; For memcpy:
-; operand_no = 0 applies to destination (store side)
-; operand_no = 1 applies to source (load side)
-
-declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1), ptr addrspace(1), i64, i1)
-declare <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1), <2 x i1>, <2 x i16>)
-declare <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1), <4 x i1>, <4 x i32>)
-
-;-----------------------------------------------------------------------------
-; Test memcpy with cache hints on both source and destination
-; Source (operand 1): L1::evict_first, L2::evict_first, L2::128B
-; Dest (operand 0): L1::evict_last, L2::evict_last
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_memcpy_both_hints
-; CHECK-DAG: ld.global.L1::evict_first.L2::evict_first.L2::128B.b
-; CHECK-DAG: st.global.L1::evict_last.L2::evict_last.b
-define void @test_memcpy_both_hints(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !80
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Test memcpy with cache hint only on source (load side)
-; Source (operand 1): L1::evict_first
-; Dest (operand 0): no hint
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_memcpy_src_hint_only
-; CHECK: ld.global.L1::evict_first.b
-; CHECK: st.global.b
-; CHECK-NOT: st.global.L1
-define void @test_memcpy_src_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !81
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Test memcpy with cache hint only on destination (store side)
-; Source (operand 1): no hint
-; Dest (operand 0): L2::evict_last
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_memcpy_dest_hint_only
-; CHECK: ld.global.b
-; CHECK-NOT: ld.global.L2
-; CHECK: st.global.L2::evict_last.b
-define void @test_memcpy_dest_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !82
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Test memcpy with L2::cache_hint on both operands
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_memcpy_l2_cache_hint
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 34343
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 12121
-; CHECK-DAG: ld.global.L2::cache_hint.b
-; CHECK-DAG: st.global.L2::cache_hint.b
-define void @test_memcpy_l2_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !83
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Test memcpy without cache hints produces plain load/store
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_memcpy_no_hint
-; CHECK: ld.global.b
-; CHECK: st.global.b
-; CHECK-NOT: L1::evict
-; CHECK-NOT: L2::evict
-; CHECK-NOT: L2::cache_hint
-define void @test_memcpy_no_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false)
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Combined L1 + L2 eviction policies
-;-----------------------------------------------------------------------------
-
-; Source: L1::evict_first + L2::evict_last
-; Dest: no hint
-; CHECK-LABEL: test_memcpy_src_l1_l2_combined
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b
-; CHECK: st.global.b
-; CHECK-NOT: st.global.L1
-; CHECK-NOT: st.global.L2
-define void @test_memcpy_src_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !84
- ret void
-}
-
-; Source: no hint
-; Dest: L1::evict_unchanged + L2::evict_first
-; CHECK-LABEL: test_memcpy_dest_l1_l2_combined
-; CHECK: ld.global.b
-; CHECK-NOT: ld.global.L1
-; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b
-define void @test_memcpy_dest_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !85
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; L1 + prefetch combinations
-;-----------------------------------------------------------------------------
-
-; Source: L1::evict_last + L2::256B prefetch
-; Dest: L1::no_allocate
-; CHECK-LABEL: test_memcpy_l1_prefetch
-; CHECK-DAG: ld.global.L1::evict_last.L2::256B.b
-; CHECK-DAG: st.global.L1::no_allocate.b
-define void @test_memcpy_l1_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !86
- ret void
-}
-
-; Source: L2::64B prefetch only
-; Dest: L2::evict_last only
-; CHECK-LABEL: test_memcpy_prefetch_vs_eviction
-; CHECK: ld.global.L2::64B.b
-; CHECK: st.global.L2::evict_last.b
-define void @test_memcpy_prefetch_vs_eviction(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !87
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; L2::cache_hint combined with other hints
-;-----------------------------------------------------------------------------
-
-; Source: L2::cache_hint + L1::evict_first
-; Dest: no hint
-; CHECK-LABEL: test_memcpy_src_cache_hint_l1
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 55555
-; CHECK: ld.global.L1::evict_first.L2::cache_hint.b
-; CHECK: st.global.b
-; CHECK-NOT: st.global.L2::cache_hint
-define void @test_memcpy_src_cache_hint_l1(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !88
- ret void
-}
-
-; Source: no hint
-; Dest: L2::cache_hint + L1::evict_last + L2::evict_first
-; CHECK-LABEL: test_memcpy_dest_cache_hint_combined
-; CHECK: ld.global.b
-; CHECK-NOT: ld.global.L2::cache_hint
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 66666
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b
-define void @test_memcpy_dest_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !89
- ret void
-}
-
-; Both operands: L2::cache_hint + L1 eviction + L2 eviction
-; Source: L2::cache_hint(77777) + L1::evict_unchanged + L2::evict_last
-; Dest: L2::cache_hint(88888) + L1::evict_first + L2::evict_first
-; CHECK-LABEL: test_memcpy_both_cache_hint_combined
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 77777
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 88888
-; CHECK-DAG: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b
-; CHECK-DAG: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b
-define void @test_memcpy_both_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !90
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; L2::cache_hint + prefetch combinations
-;-----------------------------------------------------------------------------
-
-; Source: L2::cache_hint + L2::128B prefetch
-; Dest: L2::cache_hint + L1::evict_last
-; CHECK-LABEL: test_memcpy_cache_hint_prefetch
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 11111
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 22222
-; CHECK-DAG: ld.global.L2::cache_hint.L2::128B.b
-; CHECK-DAG: st.global.L1::evict_last.L2::cache_hint.b
-define void @test_memcpy_cache_hint_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !91
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Asymmetric hint combinations (complex vs simple)
-;-----------------------------------------------------------------------------
-
-; Source: all hints (L1 + L2 eviction + prefetch)
-; Dest: simple L1 hint only
-; CHECK-LABEL: test_memcpy_complex_src_simple_dest
-; CHECK-DAG: ld.global.L1::evict_first.L2::evict_last.L2::64B.b
-; CHECK-DAG: st.global.L1::evict_last.b
-define void @test_memcpy_complex_src_simple_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !92
- ret void
-}
-
-; Source: simple L2 prefetch only
-; Dest: all hints (L1 + L2 eviction + L2::cache_hint)
-; CHECK-LABEL: test_memcpy_simple_src_complex_dest
-; CHECK: ld.global.L2::256B.b
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 99999
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b
-define void @test_memcpy_simple_src_complex_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !93
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Different L1 eviction policies on src vs dest
-;-----------------------------------------------------------------------------
-
-; Source: L1::evict_unchanged
-; Dest: L1::evict_first
-; CHECK-LABEL: test_memcpy_different_l1_policies
-; CHECK-DAG: ld.global.L1::evict_unchanged.b
-; CHECK-DAG: st.global.L1::evict_first.b
-define void @test_memcpy_different_l1_policies(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !94
- ret void
-}
-
-; Source: L1::no_allocate
-; Dest: L1::evict_unchanged
-; CHECK-LABEL: test_memcpy_no_allocate_vs_unchanged
-; CHECK-DAG: ld.global.L1::no_allocate.b
-; CHECK-DAG: st.global.L1::evict_unchanged.b
-define void @test_memcpy_no_allocate_vs_unchanged(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !95
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; All hints maxed out on both operands
-;-----------------------------------------------------------------------------
-
-; Source: L1::evict_first + L2::evict_first + L2::256B + L2::cache_hint
-; Dest: L1::evict_last + L2::evict_last + L2::128B + L2::cache_hint
-; CHECK-LABEL: test_memcpy_all_hints_both
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 12345
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 67890
-; CHECK-DAG: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b
-; CHECK-DAG: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b
-define void @test_memcpy_all_hints_both(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !96
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; TODO: Preserve cache hints on llvm.masked.load.
-; Masked load pointer operand is operand 0. SelectionDAGBuilder::visitMaskedLoad
-; does not currently thread !mem.cache_hint into the MachineMemOperand, so the
-; generated loads are plain today.
-;-----------------------------------------------------------------------------
-
-; TODO: This should eventually lower to ld.global.L1::evict_first.b32.
-; CHECK-LABEL: test_masked_load_l1_hint_v2i16
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK: ld.global.b32
-define <2 x i16> @test_masked_load_l1_hint_v2i16(ptr addrspace(1) %p) {
- %v = call <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1) align 4 %p, <2 x i1> <i1 true, i1 false>, <2 x i16> poison), !mem.cache_hint !101
- ret <2 x i16> %v
-}
-
-; TODO: This should eventually lower to ld.global.L2::cache_hint.v4.b32.
-; CHECK-LABEL: test_masked_load_l2_cache_policy_v4i32
-; CHECK-NOT: mov.b64
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK: ld.global.v4.b32
-define <4 x i32> @test_masked_load_l2_cache_policy_v4i32(ptr addrspace(1) %p) {
- %v = call <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1) align 16 %p, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> poison), !mem.cache_hint !102
- ret <4 x i32> %v
-}
-
-;-----------------------------------------------------------------------------
-; Large memcpy tests - verify hints propagate to all expanded load/stores
-; LLVM expands memcpy to multiple load/store pairs. Each pair should
-; get the appropriate cache hints from the original memcpy metadata.
-; The expansion may use various sizes (b8, b16, b32, v2, v4, etc.)
-;-----------------------------------------------------------------------------
-
-; 16-byte memcpy: verify hints are applied to expanded loads/stores
-; CHECK-LABEL: test_memcpy_16bytes
-; CHECK: ld.global.L1::evict_first.b
-; CHECK: st.global.L1::evict_last.b
-define void @test_memcpy_16bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 16, i1 false), !mem.cache_hint !97
- ret void
-}
-
-; 32-byte memcpy: all loads should have L1::evict_unchanged, all stores L2::evict_first
-; CHECK-LABEL: test_memcpy_32bytes
-; CHECK: ld.global.L1::evict_unchanged.b
-; CHECK: st.global.L2::evict_first.b
-define void @test_memcpy_32bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 32, i1 false), !mem.cache_hint !98
- ret void
-}
-
-; 64-byte memcpy with L2::cache_hint
-; All loads and stores should get the L2::cache_hint with their respective policies
-; CHECK-LABEL: test_memcpy_64bytes_cache_hint
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 11111
-; CHECK-DAG: mov.b64 {{%rd[0-9]+}}, 22222
-; CHECK: ld.global.L2::cache_hint.b
-; CHECK: st.global.L2::cache_hint.b
-define void @test_memcpy_64bytes_cache_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 64, i1 false), !mem.cache_hint !99
- ret void
-}
-
-; 128-byte memcpy with combined hints
-; Note: Large memcpy (>64 bytes) is expanded to a loop in the backend.
-; Cache hints are not preserved for loop-based expansion.
-; This test verifies the code compiles correctly.
-; CHECK-LABEL: test_memcpy_128bytes_combined
-; CHECK: ld.global.b
-; CHECK: st.global.b
-define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
- call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 128, i1 false), !mem.cache_hint !100
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Metadata definitions
-;-----------------------------------------------------------------------------
-
-; memcpy with both dest and src hints
-!80 = !{i32 0, !180, i32 1, !181}
-; operand 0 (dest/store): L1::evict_last, L2::evict_last
-!180 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last"}
-; operand 1 (src/load): L1::evict_first, L2::evict_first, L2::128B prefetch
-!181 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"128B"}
-
-; memcpy with only source hint (load side)
-!81 = !{i32 1, !182}
-!182 = !{!"nvvm.l1_eviction", !"first"}
-
-; memcpy with only dest hint (store side)
-!82 = !{i32 0, !183}
-!183 = !{!"nvvm.l2_eviction", !"last"}
-
-; memcpy with L2::cache_hint on both operands
-!83 = !{i32 0, !184, i32 1, !185}
-!184 = !{!"nvvm.l2_cache_hint", i64 12121}
-!185 = !{!"nvvm.l2_cache_hint", i64 34343}
-
-; Combined L1 + L2 eviction on source only
-!84 = !{i32 1, !186}
-!186 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last"}
-
-; Combined L1 + L2 eviction on dest only
-!85 = !{i32 0, !187}
-!187 = !{!"nvvm.l1_eviction", !"unchanged", !"nvvm.l2_eviction", !"first"}
-
-; L1 + prefetch on source, L1 on dest
-!86 = !{i32 0, !189, i32 1, !188}
-!188 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
-!189 = !{!"nvvm.l1_eviction", !"no_allocate"}
-
-; Prefetch on source, L2 eviction on dest
-!87 = !{i32 0, !191, i32 1, !190}
-!190 = !{!"nvvm.l2_prefetch_size", !"64B"}
-!191 = !{!"nvvm.l2_eviction", !"last"}
-
-; L2::cache_hint + L1 eviction on source only
-!88 = !{i32 1, !192}
-!192 = !{!"nvvm.l2_cache_hint", i64 55555, !"nvvm.l1_eviction", !"first"}
-
-; L2::cache_hint + L1 + L2 eviction on dest only
-!89 = !{i32 0, !193}
-!193 = !{!"nvvm.l2_cache_hint", i64 66666, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
-
-; Both operands: L2::cache_hint + L1 + L2 eviction
-!90 = !{i32 0, !195, i32 1, !194}
-!194 = !{!"nvvm.l2_cache_hint", i64 77777, !"nvvm.l1_eviction", !"unchanged", !"nvvm.l2_eviction", !"last"}
-!195 = !{!"nvvm.l2_cache_hint", i64 88888, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first"}
-
-; L2::cache_hint + prefetch on source, L2::cache_hint + L1 on dest
-!91 = !{i32 0, !197, i32 1, !196}
-!196 = !{!"nvvm.l2_cache_hint", i64 11111, !"nvvm.l2_prefetch_size", !"128B"}
-!197 = !{!"nvvm.l2_cache_hint", i64 22222, !"nvvm.l1_eviction", !"last"}
-
-; Complex source (all non-cache_hint), simple dest
-!92 = !{i32 0, !199, i32 1, !198}
-!198 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"64B"}
-!199 = !{!"nvvm.l1_eviction", !"last"}
-
-; Simple source, complex dest (with cache_hint)
-!93 = !{i32 0, !201, i32 1, !200}
-!200 = !{!"nvvm.l2_prefetch_size", !"256B"}
-!201 = !{!"nvvm.l2_cache_hint", i64 99999, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
-
-; Different L1 policies: unchanged vs first
-!94 = !{i32 0, !203, i32 1, !202}
-!202 = !{!"nvvm.l1_eviction", !"unchanged"}
-!203 = !{!"nvvm.l1_eviction", !"first"}
-
-; Different L1 policies: no_allocate vs unchanged
-!95 = !{i32 0, !205, i32 1, !204}
-!204 = !{!"nvvm.l1_eviction", !"no_allocate"}
-!205 = !{!"nvvm.l1_eviction", !"unchanged"}
-
-; All hints maxed out on both operands
-!96 = !{i32 0, !207, i32 1, !206}
-!206 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
-!207 = !{!"nvvm.l2_cache_hint", i64 67890, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
-
-; Large memcpy metadata
-!97 = !{i32 0, !209, i32 1, !208}
-!208 = !{!"nvvm.l1_eviction", !"first"}
-!209 = !{!"nvvm.l1_eviction", !"last"}
-
-!98 = !{i32 0, !211, i32 1, !210}
-!210 = !{!"nvvm.l1_eviction", !"unchanged"}
-!211 = !{!"nvvm.l2_eviction", !"first"}
-
-!99 = !{i32 0, !213, i32 1, !212}
-!212 = !{!"nvvm.l2_cache_hint", i64 11111}
-!213 = !{!"nvvm.l2_cache_hint", i64 22222}
-
-!100 = !{i32 0, !215, i32 1, !214}
-!214 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"256B"}
-!215 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
-
-; Masked load metadata
-!101 = !{i32 0, !216}
-!216 = !{!"nvvm.l1_eviction", !"first"}
-
-!102 = !{i32 0, !217}
-!217 = !{!"nvvm.l2_cache_hint", i64 24680}
diff --git a/llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll b/llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll
deleted file mode 100644
index 2a6ac808d2d2e..0000000000000
--- a/llvm/test/CodeGen/NVPTX/load-store-cache-hint.ll
+++ /dev/null
@@ -1,935 +0,0 @@
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
-
-; Test !mem.cache_hint metadata lowering to PTX cache qualifiers
-; PTX supports the following cache qualifiers:
-; L1 eviction: L1::evict_first, L1::evict_last, L1::evict_unchanged, L1::no_allocate
-; L2 eviction: L2::evict_first, L2::evict_last
-; L2 prefetch: L2::64B, L2::128B, L2::256B
-
-;-----------------------------------------------------------------------------
-; Basic L1 eviction policies for loads
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_l1_first
-; CHECK: ld.global.L1::evict_first.b32
-define i32 @test_load_l1_first(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l1_last
-; CHECK: ld.global.L1::evict_last.b32
-define i32 @test_load_l1_last(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l1_unchanged
-; CHECK: ld.global.L1::evict_unchanged.b32
-define i32 @test_load_l1_unchanged(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l1_no_allocate
-; CHECK: ld.global.L1::no_allocate.b32
-define i32 @test_load_l1_no_allocate(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; Basic L2 eviction policies for loads
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_l2_first
-; CHECK: ld.global.L2::evict_first.b32
-define i32 @test_load_l2_first(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l2_last
-; CHECK: ld.global.L2::evict_last.b32
-define i32 @test_load_l2_last(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !5
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; L2 prefetch sizes for loads
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_prefetch_64
-; CHECK: ld.global.L2::64B.b32
-define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_prefetch_128
-; CHECK: ld.global.L2::128B.b32
-define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_prefetch_256
-; CHECK: ld.global.L2::256B.b32
-define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; All L1 + L2 combinations for loads
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_l1_first_l2_first
-; CHECK: ld.global.L1::evict_first.L2::evict_first.b32
-define i32 @test_load_l1_first_l2_first(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !20
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l1_first_l2_last
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b32
-define i32 @test_load_l1_first_l2_last(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !21
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l1_last_l2_first
-; CHECK: ld.global.L1::evict_last.L2::evict_first.b32
-define i32 @test_load_l1_last_l2_first(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !22
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l1_last_l2_last
-; CHECK: ld.global.L1::evict_last.L2::evict_last.b32
-define i32 @test_load_l1_last_l2_last(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !23
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; L1 + L2 + Prefetch combination for loads
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_l1_first_l2_last_prefetch_128
-; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::128B.b32
-define i32 @test_load_l1_first_l2_last_prefetch_128(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !24
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; Basic L1 eviction policies for stores
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_store_l1_first
-; CHECK: st.global.L1::evict_first.b32
-define void @test_store_l1_first(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1000
- ret void
-}
-
-; CHECK-LABEL: test_store_l1_last
-; CHECK: st.global.L1::evict_last.b32
-define void @test_store_l1_last(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1001
- ret void
-}
-
-; CHECK-LABEL: test_store_l1_unchanged
-; CHECK: st.global.L1::evict_unchanged.b32
-define void @test_store_l1_unchanged(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1002
- ret void
-}
-
-; CHECK-LABEL: test_store_l1_no_allocate
-; CHECK: st.global.L1::no_allocate.b32
-define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1003
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Basic L2 eviction policies for stores
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_store_l2_first
-; CHECK: st.global.L2::evict_first.b32
-define void @test_store_l2_first(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1004
- ret void
-}
-
-; CHECK-LABEL: test_store_l2_last
-; CHECK: st.global.L2::evict_last.b32
-define void @test_store_l2_last(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1005
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; All L1 + L2 combinations for stores
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_store_l1_first_l2_first
-; CHECK: st.global.L1::evict_first.L2::evict_first.b32
-define void @test_store_l1_first_l2_first(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1020
- ret void
-}
-
-; CHECK-LABEL: test_store_l1_first_l2_last
-; CHECK: st.global.L1::evict_first.L2::evict_last.b32
-define void @test_store_l1_first_l2_last(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1021
- ret void
-}
-
-; CHECK-LABEL: test_store_l1_last_l2_first
-; CHECK: st.global.L1::evict_last.L2::evict_first.b32
-define void @test_store_l1_last_l2_first(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1022
- ret void
-}
-
-; CHECK-LABEL: test_store_l1_last_l2_last
-; CHECK: st.global.L1::evict_last.L2::evict_last.b32
-define void @test_store_l1_last_l2_last(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1023
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Different data types - loads
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_i16_l1_first
-; CHECK: ld.global.L1::evict_first.b16
-define i16 @test_load_i16_l1_first(ptr addrspace(1) %p) {
- %v = load i16, ptr addrspace(1) %p, !mem.cache_hint !0
- ret i16 %v
-}
-
-; CHECK-LABEL: test_load_i64_l1_last
-; CHECK: ld.global.L1::evict_last.b64
-define i64 @test_load_i64_l1_last(ptr addrspace(1) %p) {
- %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !1
- ret i64 %v
-}
-
-; CHECK-LABEL: test_load_f32_l2_first
-; CHECK: ld.global.L2::evict_first.b32
-define float @test_load_f32_l2_first(ptr addrspace(1) %p) {
- %v = load float, ptr addrspace(1) %p, !mem.cache_hint !4
- ret float %v
-}
-
-; CHECK-LABEL: test_load_f64_l2_last
-; CHECK: ld.global.L2::evict_last.b64
-define double @test_load_f64_l2_last(ptr addrspace(1) %p) {
- %v = load double, ptr addrspace(1) %p, !mem.cache_hint !5
- ret double %v
-}
-
-;-----------------------------------------------------------------------------
-; Different data types - stores
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_store_i16_l1_first
-; CHECK: st.global.L1::evict_first.b16
-define void @test_store_i16_l1_first(ptr addrspace(1) %p, i16 %v) {
- store i16 %v, ptr addrspace(1) %p, !mem.cache_hint !1000
- ret void
-}
-
-; CHECK-LABEL: test_store_i64_l2_last
-; CHECK: st.global.L2::evict_last.b64
-define void @test_store_i64_l2_last(ptr addrspace(1) %p, i64 %v) {
- store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !1005
- ret void
-}
-
-; CHECK-LABEL: test_store_f32_l1_no_allocate
-; CHECK: st.global.L1::no_allocate.b32
-define void @test_store_f32_l1_no_allocate(ptr addrspace(1) %p, float %v) {
- store float %v, ptr addrspace(1) %p, !mem.cache_hint !1003
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Vector loads with cache hints
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_v2i32_l1_first
-; CHECK: ld.global.L1::evict_first.v2.b32
-define <2 x i32> @test_load_v2i32_l1_first(ptr addrspace(1) %p) {
- %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !0
- ret <2 x i32> %v
-}
-
-; CHECK-LABEL: test_load_v4i32_l2_last
-; CHECK: ld.global.L2::evict_last.v4.b32
-define <4 x i32> @test_load_v4i32_l2_last(ptr addrspace(1) %p) {
- %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !5
- ret <4 x i32> %v
-}
-
-; CHECK-LABEL: test_load_v2f32_l1_unchanged
-; CHECK: ld.global.L1::evict_unchanged.v2.b32
-define <2 x float> @test_load_v2f32_l1_unchanged(ptr addrspace(1) %p) {
- %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !2
- ret <2 x float> %v
-}
-
-; CHECK-LABEL: test_load_v2f64_prefetch_128
-; CHECK: ld.global.L2::128B.v2.b64
-define <2 x double> @test_load_v2f64_prefetch_128(ptr addrspace(1) %p) {
- %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !7
- ret <2 x double> %v
-}
-
-;-----------------------------------------------------------------------------
-; Vector stores with cache hints
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_store_v2i32_l1_last
-; CHECK: st.global.L1::evict_last.v2.b32
-define void @test_store_v2i32_l1_last(ptr addrspace(1) %p, <2 x i32> %v) {
- store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1001
- ret void
-}
-
-; CHECK-LABEL: test_store_v4i32_l2_first
-; CHECK: st.global.L2::evict_first.v4.b32
-define void @test_store_v4i32_l2_first(ptr addrspace(1) %p, <4 x i32> %v) {
- store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1004
- ret void
-}
-
-; CHECK-LABEL: test_store_v2f64_l1_no_allocate
-; CHECK: st.global.L1::no_allocate.v2.b64
-define void @test_store_v2f64_l1_no_allocate(ptr addrspace(1) %p, <2 x double> %v) {
- store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !1003
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Invariant loads with cache hints may still use LDG (ld.global.nc)
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_invariant_load_with_hint
-; CHECK: ld.global.nc.L1::evict_first.b32
-define i32 @test_invariant_load_with_hint(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !0
- ret i32 %v
-}
-
-; CHECK-LABEL: test_invariant_load_with_cache_policy
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 12345
-; CHECK: ld.global.nc.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_invariant_load_with_cache_policy(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !30
- ret i32 %v
-}
-
-; CHECK-LABEL: test_invariant_load_v2i32_with_hint
-; CHECK: ld.global.nc.L1::evict_last.L2::evict_first.v2.b32
-define <2 x i32> @test_invariant_load_v2i32_with_hint(ptr addrspace(1) %p) {
- %v = load <2 x i32>, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !22
- ret <2 x i32> %v
-}
-
-;-----------------------------------------------------------------------------
-; No hint should produce plain load/store
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_no_hint
-; CHECK: ld.global.b32
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-define i32 @test_load_no_hint(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p
- ret i32 %v
-}
-
-; CHECK-LABEL: test_store_no_hint
-; CHECK: st.global.b32
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-define void @test_store_no_hint(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; L2::cache_hint with constant cache-policy operand (metadata-based)
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_cache_hint_i32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 12345
-; CHECK: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_load_cache_hint_i32(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !30
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_cache_hint_i64
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 98765
-; CHECK: ld.global.L2::cache_hint.b64 {{%rd[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i64 @test_load_cache_hint_i64(ptr addrspace(1) %p) {
- %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !31
- ret i64 %v
-}
-
-; CHECK-LABEL: test_load_cache_hint_f32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 55555
-; CHECK: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define float @test_load_cache_hint_f32(ptr addrspace(1) %p) {
- %v = load float, ptr addrspace(1) %p, !mem.cache_hint !32
- ret float %v
-}
-
-; CHECK-LABEL: test_store_cache_hint_i32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 67890
-; CHECK: st.global.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
-define void @test_store_cache_hint_i32(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1033
- ret void
-}
-
-; CHECK-LABEL: test_store_cache_hint_i64
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 11111
-; CHECK: st.global.L2::cache_hint.b64 [{{%rd[0-9]+}}], {{%rd[0-9]+}}, [[POLICY]]
-define void @test_store_cache_hint_i64(ptr addrspace(1) %p, i64 %v) {
- store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !1034
- ret void
-}
-
-; CHECK-LABEL: test_store_cache_hint_f32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 22222
-; CHECK: st.global.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
-define void @test_store_cache_hint_f32(ptr addrspace(1) %p, float %v) {
- store float %v, ptr addrspace(1) %p, !mem.cache_hint !1035
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; L2::cache_hint with vector types
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_load_cache_hint_v2i32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 33333
-; CHECK: ld.global.L2::cache_hint.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
-define <2 x i32> @test_load_cache_hint_v2i32(ptr addrspace(1) %p) {
- %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !40
- ret <2 x i32> %v
-}
-
-; CHECK-LABEL: test_load_cache_hint_v4i32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 44444
-; CHECK: ld.global.L2::cache_hint.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
-define <4 x i32> @test_load_cache_hint_v4i32(ptr addrspace(1) %p) {
- %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !41
- ret <4 x i32> %v
-}
-
-; CHECK-LABEL: test_load_cache_hint_v2i64
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 55556
-; CHECK: ld.global.L2::cache_hint.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
-define <2 x i64> @test_load_cache_hint_v2i64(ptr addrspace(1) %p) {
- %v = load <2 x i64>, ptr addrspace(1) %p, !mem.cache_hint !42
- ret <2 x i64> %v
-}
-
-; CHECK-LABEL: test_load_cache_hint_v2f32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 66666
-; CHECK: ld.global.L2::cache_hint.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
-define <2 x float> @test_load_cache_hint_v2f32(ptr addrspace(1) %p) {
- %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !43
- ret <2 x float> %v
-}
-
-; CHECK-LABEL: test_load_cache_hint_v2f64
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 77777
-; CHECK: ld.global.L2::cache_hint.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
-define <2 x double> @test_load_cache_hint_v2f64(ptr addrspace(1) %p) {
- %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !44
- ret <2 x double> %v
-}
-
-; CHECK-LABEL: test_store_cache_hint_v2i32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 88888
-; CHECK: st.global.L2::cache_hint.v2.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
-define void @test_store_cache_hint_v2i32(ptr addrspace(1) %p, <2 x i32> %v) {
- store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1045
- ret void
-}
-
-; CHECK-LABEL: test_store_cache_hint_v4i32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 99999
-; CHECK: st.global.L2::cache_hint.v4.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
-define void @test_store_cache_hint_v4i32(ptr addrspace(1) %p, <4 x i32> %v) {
- store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1046
- ret void
-}
-
-; CHECK-LABEL: test_store_cache_hint_v2i64
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 11112
-; CHECK: st.global.L2::cache_hint.v2.b64 [{{%rd[0-9]+}}], {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [[POLICY]]
-define void @test_store_cache_hint_v2i64(ptr addrspace(1) %p, <2 x i64> %v) {
- store <2 x i64> %v, ptr addrspace(1) %p, !mem.cache_hint !1047
- ret void
-}
-
-; CHECK-LABEL: test_store_cache_hint_v2f32
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 22223
-; CHECK: st.global.L2::cache_hint.v2.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
-define void @test_store_cache_hint_v2f32(ptr addrspace(1) %p, <2 x float> %v) {
- store <2 x float> %v, ptr addrspace(1) %p, !mem.cache_hint !1048
- ret void
-}
-
-; CHECK-LABEL: test_store_cache_hint_v2f64
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 33334
-; CHECK: st.global.L2::cache_hint.v2.b64 [{{%rd[0-9]+}}], {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [[POLICY]]
-define void @test_store_cache_hint_v2f64(ptr addrspace(1) %p, <2 x double> %v) {
- store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !1049
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; L2::cache_hint combined with other hints (L2::cache_hint takes precedence)
-;-----------------------------------------------------------------------------
-
-; L2::cache_hint + L1 eviction: both qualifiers should be emitted
-; CHECK-LABEL: test_load_cache_hint_with_l1
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 44445
-; CHECK: ld.global.L1::evict_first.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !50
- ret i32 %v
-}
-
-; L2::cache_hint + L2 eviction: both qualifiers should be emitted
-; CHECK-LABEL: test_load_cache_hint_with_l2_eviction
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 55557
-; CHECK: ld.global.L2::evict_last.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_load_cache_hint_with_l2_eviction(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !51
- ret i32 %v
-}
-
-; L2::cache_hint + L2 prefetch: both qualifiers should be emitted
-; CHECK-LABEL: test_load_cache_hint_with_prefetch
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 66667
-; CHECK: ld.global.L2::cache_hint.L2::128B.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_load_cache_hint_with_prefetch(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !52
- ret i32 %v
-}
-
-; L2::cache_hint + all other hints: all qualifiers emitted
-; CHECK-LABEL: test_load_cache_hint_with_all
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 77778
-; CHECK: ld.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::256B.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_load_cache_hint_with_all(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !53
- ret i32 %v
-}
-
-; Store: L2::cache_hint + L1 eviction
-; CHECK-LABEL: test_store_cache_hint_with_l1
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 88889
-; CHECK: st.global.L1::evict_unchanged.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
-define void @test_store_cache_hint_with_l1(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1054
- ret void
-}
-
-; Store: L2::cache_hint + L1 + L2 eviction (all qualifiers emitted)
-; CHECK-LABEL: test_store_cache_hint_with_all
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 99990
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}, [[POLICY]]
-define void @test_store_cache_hint_with_all(ptr addrspace(1) %p, i32 %v) {
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1055
- ret void
-}
-
-; Vector load: L2::cache_hint + L1 eviction
-; CHECK-LABEL: test_load_cache_hint_v2i32_with_l1
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 11113
-; CHECK: ld.global.L1::evict_first.L2::cache_hint.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [{{%rd[0-9]+}}], [[POLICY]]
-define <2 x i32> @test_load_cache_hint_v2i32_with_l1(ptr addrspace(1) %p) {
- %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !56
- ret <2 x i32> %v
-}
-
-; Vector store: L2::cache_hint + L1 + L2 eviction + prefetch (all qualifiers emitted)
-; CHECK-LABEL: test_store_cache_hint_v2i32_with_all
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 22224
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::64B.v2.b32 [{{%rd[0-9]+}}], {{{%r[0-9]+}}, {{%r[0-9]+}}}, [[POLICY]]
-define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32> %v) {
- store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1057
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Multiple loads sharing same pointer - each gets its own policy
-;-----------------------------------------------------------------------------
-
-; Two volatile loads from the same pointer - each load gets its own cache policy.
-; The per-MMO storage ensures no policy collisions when multiple memops share
-; the same pointer operand but have different cache policies.
-; CHECK-LABEL: test_multiple_loads_same_ptr
-; CHECK-DAG: mov.b64 [[POLICY1:%rd[0-9]+]], 11111
-; CHECK-DAG: mov.b64 [[POLICY2:%rd[0-9]+]], 22222
-; CHECK-DAG: ld.volatile.global.L1::evict_last.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY1]]
-; CHECK-DAG: ld.volatile.global.L1::evict_first.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY2]]
-define i32 @test_multiple_loads_same_ptr(ptr addrspace(1) %p) {
- %v1 = load volatile i32, ptr addrspace(1) %p, !mem.cache_hint !60
- %v2 = load volatile i32, ptr addrspace(1) %p, !mem.cache_hint !61
- %sum = add i32 %v1, %v2
- ret i32 %sum
-}
-
-; Non-volatile loads can CSE. Matching cache hints are preserved on the merged
-; DAG node, but conflicting hints are dropped.
-; CHECK-LABEL: test_cse_loads_same_cache_hint
-; CHECK: ld.global.L1::evict_first.b32
-; CHECK-NOT: ld.global
-define i32 @test_cse_loads_same_cache_hint(ptr addrspace(1) %p) {
- %v1 = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
- %v2 = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
- %sum = add i32 %v1, %v2
- ret i32 %sum
-}
-
-; CHECK-LABEL: test_cse_loads_conflicting_cache_hints
-; CHECK: ld.global.b32
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK-NOT: ld.global
-define i32 @test_cse_loads_conflicting_cache_hints(ptr addrspace(1) %p) {
- %v1 = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
- %v2 = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
- %sum = add i32 %v1, %v2
- ret i32 %sum
-}
-
-; NVPTXForwardParams rewrites eligible byval loads to ld.param. Since cache
-; hints are not allowed on ld.param, we must drop them.
-; CHECK-LABEL: test_forward_param_drops_l1_hint
-; CHECK-NOT: L1::evict_first
-; CHECK: ld.param.b32
-; CHECK-NOT: L1::evict_first
-; CHECK-NOT: ld.local
-define i32 @test_forward_param_drops_l1_hint(ptr byval(i32) %a) {
- %v = load i32, ptr %a, !mem.cache_hint !90
- ret i32 %v
-}
-
-; CHECK-LABEL: test_forward_param_drops_l2_cache_hint
-; CHECK-NOT: mov.b64
-; CHECK-NOT: L2::cache_hint
-; CHECK: ld.param.b32
-; CHECK-NOT: mov.b64
-; CHECK-NOT: L2::cache_hint
-; CHECK-NOT: ld.local
-define i32 @test_forward_param_drops_l2_cache_hint(ptr byval(i32) %a) {
- %v = load i32, ptr %a, !mem.cache_hint !91
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; Valid edge cases
-;-----------------------------------------------------------------------------
-
-; Test with custom hint key order - should still work
-; CHECK-LABEL: test_load_reordered_metadata
-; CHECK: ld.global.L1::evict_last.L2::evict_first.b32
-define i32 @test_load_reordered_metadata(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !11
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; nvvm.l2_cache_hint with alternate integer value width
-;-----------------------------------------------------------------------------
-
-; nvvm.l2_cache_hint with i32 instead of i64 - should still work
-; as mdconst::dyn_extract<ConstantInt> accepts any integer type
-; CHECK-LABEL: test_load_cache_hint_i32_value
-; CHECK: mov.b64 [[POLICY:%rd[0-9]+]], 99999
-; CHECK: ld.global.L2::cache_hint.b32 {{%r[0-9]+}}, [{{%rd[0-9]+}}], [[POLICY]]
-define i32 @test_load_cache_hint_i32_value(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !12
- ret i32 %v
-}
-
-; Test "normal" eviction - should not emit any qualifier (default behavior)
-; CHECK-LABEL: test_load_l1_normal
-; CHECK: ld.global.b32
-; CHECK-NOT: L1::evict_normal
-define i32 @test_load_l1_normal(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !13
- ret i32 %v
-}
-
-; CHECK-LABEL: test_load_l2_normal
-; CHECK: ld.global.b32
-; CHECK-NOT: L2::evict_normal
-define i32 @test_load_l2_normal(ptr addrspace(1) %p) {
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !14
- ret i32 %v
-}
-
-;-----------------------------------------------------------------------------
-; TODO: Preserve cache hints across DAGCombiner-created memory rewrites.
-; This documents the current store-of-concat-trunc behavior: copied MMOs for
-; the split stores do not retain !mem.cache_hint metadata yet.
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_dagcombine_store_concat_trunc_v8i32
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK-COUNT-2: st.global.v4.b32
-define void @test_dagcombine_store_concat_trunc_v8i32(ptr addrspace(1) %p, <4 x i64> %a, <4 x i64> %b) {
- %ta = trunc <4 x i64> %a to <4 x i32>
- %tb = trunc <4 x i64> %b to <4 x i32>
- %c = shufflevector <4 x i32> %ta, <4 x i32> %tb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- store <8 x i32> %c, ptr addrspace(1) %p, align 16, !mem.cache_hint !2004
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; TODO: Preserve cache hints across one-to-N DAG memory rewrites.
-; These tests document the current split/scalarized behavior: newly-created
-; memory ops do not retain !mem.cache_hint metadata yet.
-;-----------------------------------------------------------------------------
-
-; CHECK-LABEL: test_legalize_split_load_v16i32
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK-COUNT-4: ld.global.v4.b32
-define <16 x i32> @test_legalize_split_load_v16i32(ptr addrspace(1) %p) {
- %v = load <16 x i32>, ptr addrspace(1) %p, align 16, !mem.cache_hint !2000
- ret <16 x i32> %v
-}
-
-; CHECK-LABEL: test_legalize_split_store_v16i32
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK-COUNT-4: st.global.v4.b32
-define void @test_legalize_split_store_v16i32(ptr addrspace(1) %p, <16 x i32> %v) {
- store <16 x i32> %v, ptr addrspace(1) %p, align 16, !mem.cache_hint !2001
- ret void
-}
-
-; CHECK-LABEL: test_legalize_scalarize_load_v3i64
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK-COUNT-3: ld.global.{{[bu]}}64
-define <3 x i64> @test_legalize_scalarize_load_v3i64(ptr addrspace(1) %p) {
- %v = load <3 x i64>, ptr addrspace(1) %p, align 8, !mem.cache_hint !2002
- ret <3 x i64> %v
-}
-
-; CHECK-LABEL: test_legalize_scalarize_store_v3i64
-; CHECK-NOT: L1::
-; CHECK-NOT: L2::
-; CHECK-COUNT-3: st.global.{{[bu]}}64
-define void @test_legalize_scalarize_store_v3i64(ptr addrspace(1) %p, <3 x i64> %v) {
- store <3 x i64> %v, ptr addrspace(1) %p, align 8, !mem.cache_hint !2003
- ret void
-}
-
-;-----------------------------------------------------------------------------
-; Metadata definitions
-;-----------------------------------------------------------------------------
-
-; L1 eviction policies
-!0 = !{i32 0, !100}
-!100 = !{!"nvvm.l1_eviction", !"first"}
-
-!1 = !{i32 0, !101}
-!101 = !{!"nvvm.l1_eviction", !"last"}
-
-!2 = !{i32 0, !102}
-!102 = !{!"nvvm.l1_eviction", !"unchanged"}
-
-!3 = !{i32 0, !103}
-!103 = !{!"nvvm.l1_eviction", !"no_allocate"}
-
-; L2 eviction policies
-!4 = !{i32 0, !104}
-!104 = !{!"nvvm.l2_eviction", !"first"}
-
-!5 = !{i32 0, !105}
-!105 = !{!"nvvm.l2_eviction", !"last"}
-
-; L2 prefetch sizes
-!6 = !{i32 0, !106}
-!106 = !{!"nvvm.l2_prefetch_size", !"64B"}
-
-!7 = !{i32 0, !107}
-!107 = !{!"nvvm.l2_prefetch_size", !"128B"}
-
-!8 = !{i32 0, !108}
-!108 = !{!"nvvm.l2_prefetch_size", !"256B"}
-
-; Custom key order in hint node
-!11 = !{i32 0, !111}
-!111 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
-
-; i32 instead of i64 - still valid, ConstantInt accepts any integer type
-!12 = !{i32 0, !112}
-!112 = !{!"nvvm.l2_cache_hint", i32 99999}
-
-; "normal" eviction (default, should not emit qualifier)
-!13 = !{i32 0, !113}
-!113 = !{!"nvvm.l1_eviction", !"normal"}
-
-!14 = !{i32 0, !114}
-!114 = !{!"nvvm.l2_eviction", !"normal"}
-
-; All L1 + L2 combinations
-!20 = !{i32 0, !120}
-!120 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"first"}
-
-!21 = !{i32 0, !121}
-!121 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last"}
-
-!22 = !{i32 0, !122}
-!122 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first"}
-
-!23 = !{i32 0, !123}
-!123 = !{!"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"last"}
-
-; L1 + L2 + Prefetch combination
-!24 = !{i32 0, !124}
-!124 = !{!"nvvm.l1_eviction", !"first", !"nvvm.l2_eviction", !"last", !"nvvm.l2_prefetch_size", !"128B"}
-
-; L2::cache_hint with constant cache-policy
-!30 = !{i32 0, !130}
-!130 = !{!"nvvm.l2_cache_hint", i64 12345}
-
-!31 = !{i32 0, !131}
-!131 = !{!"nvvm.l2_cache_hint", i64 98765}
-
-!32 = !{i32 0, !132}
-!132 = !{!"nvvm.l2_cache_hint", i64 55555}
-
-!133 = !{!"nvvm.l2_cache_hint", i64 67890}
-
-!134 = !{!"nvvm.l2_cache_hint", i64 11111}
-
-!135 = !{!"nvvm.l2_cache_hint", i64 22222}
-
-; L2::cache_hint for vector types
-!40 = !{i32 0, !140}
-!140 = !{!"nvvm.l2_cache_hint", i64 33333}
-
-!41 = !{i32 0, !141}
-!141 = !{!"nvvm.l2_cache_hint", i64 44444}
-
-!42 = !{i32 0, !142}
-!142 = !{!"nvvm.l2_cache_hint", i64 55556}
-
-!43 = !{i32 0, !143}
-!143 = !{!"nvvm.l2_cache_hint", i64 66666}
-
-!44 = !{i32 0, !144}
-!144 = !{!"nvvm.l2_cache_hint", i64 77777}
-
-!145 = !{!"nvvm.l2_cache_hint", i64 88888}
-
-!146 = !{!"nvvm.l2_cache_hint", i64 99999}
-
-!147 = !{!"nvvm.l2_cache_hint", i64 11112}
-
-!148 = !{!"nvvm.l2_cache_hint", i64 22223}
-
-!149 = !{!"nvvm.l2_cache_hint", i64 33334}
-
-; L2::cache_hint combined with other hints (L2::cache_hint takes precedence)
-!50 = !{i32 0, !150}
-!150 = !{!"nvvm.l2_cache_hint", i64 44445, !"nvvm.l1_eviction", !"first"}
-
-!51 = !{i32 0, !151}
-!151 = !{!"nvvm.l2_cache_hint", i64 55557, !"nvvm.l2_eviction", !"last"}
-
-!52 = !{i32 0, !152}
-!152 = !{!"nvvm.l2_cache_hint", i64 66667, !"nvvm.l2_prefetch_size", !"128B"}
-
-!53 = !{i32 0, !153}
-!153 = !{!"nvvm.l2_cache_hint", i64 77778, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
-
-!154 = !{!"nvvm.l2_cache_hint", i64 88889, !"nvvm.l1_eviction", !"unchanged"}
-
-!155 = !{!"nvvm.l2_cache_hint", i64 99990, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
-
-!56 = !{i32 0, !156}
-!156 = !{!"nvvm.l2_cache_hint", i64 11113, !"nvvm.l1_eviction", !"first"}
-
-!157 = !{!"nvvm.l2_cache_hint", i64 22224, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"64B"}
-
-; Multiple loads same pointer test (different policies)
-!60 = !{i32 0, !160}
-!160 = !{!"nvvm.l2_cache_hint", i64 11111, !"nvvm.l1_eviction", !"last"}
-
-!61 = !{i32 0, !161}
-!161 = !{!"nvvm.l2_cache_hint", i64 22222, !"nvvm.l1_eviction", !"first"}
-
-; ForwardParams byval tests
-!90 = !{i32 0, !190}
-!190 = !{!"nvvm.l1_eviction", !"first"}
-
-!91 = !{i32 0, !191}
-!191 = !{!"nvvm.l2_cache_hint", i64 12345}
-
-; Store-side metadata uses operand 1 (the store pointer operand).
-!1000 = !{i32 1, !100}
-!1001 = !{i32 1, !101}
-!1002 = !{i32 1, !102}
-!1003 = !{i32 1, !103}
-!1004 = !{i32 1, !104}
-!1005 = !{i32 1, !105}
-!1020 = !{i32 1, !120}
-!1021 = !{i32 1, !121}
-!1022 = !{i32 1, !122}
-!1023 = !{i32 1, !123}
-!1033 = !{i32 1, !133}
-!1034 = !{i32 1, !134}
-!1035 = !{i32 1, !135}
-!1045 = !{i32 1, !145}
-!1046 = !{i32 1, !146}
-!1047 = !{i32 1, !147}
-!1048 = !{i32 1, !148}
-!1049 = !{i32 1, !149}
-!1054 = !{i32 1, !154}
-!1055 = !{i32 1, !155}
-!1057 = !{i32 1, !157}
-
-; Legalization tests
-!2000 = !{i32 0, !2100}
-!2100 = !{!"nvvm.l2_cache_hint", i64 424242}
-
-!2001 = !{i32 1, !2101}
-!2101 = !{!"nvvm.l2_cache_hint", i64 515151}
-
-!2002 = !{i32 0, !2102}
-!2102 = !{!"nvvm.l1_eviction", !"last"}
-
-!2003 = !{i32 1, !2103}
-!2103 = !{!"nvvm.l2_eviction", !"first"}
-
-!2004 = !{i32 1, !2104}
-!2104 = !{!"nvvm.l2_cache_hint", i64 616161}
>From bce84944196a1b62c6b2e6283c5ba9a5047250e9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 21:35:13 +0000
Subject: [PATCH 13/33] remove dummy
---
llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
index c4748c8001742..2eff4fe830585 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
@@ -2,10 +2,10 @@
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM60
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM70
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM75
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS,SM80
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s --check-prefixes=SM80-PTX70
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS,SM86
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=SM80PLUS,SM90
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=SM80PLUS
; Test SM version requirements for cache hints (from PTX ISA documentation):
; - L1::evict_* requires SM 70+
@@ -309,7 +309,3 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
; L1 eviction: no_allocate (for store)
!9 = !{i32 1, !109}
!109 = !{!"nvvm.l1_eviction", !"no_allocate"}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; SM80: {{.*}}
-; SM86: {{.*}}
-; SM90: {{.*}}
>From 986d1f14e3a6f41a5237b81b5db78e069356fe8b Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 22:20:08 +0000
Subject: [PATCH 14/33] test fixes
---
llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp | 2 +-
llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp | 4 +++-
llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp | 8 +++++---
.../unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp | 3 +--
4 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp b/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp
index 8eefc77a90f61..0dcda954e6c5e 100644
--- a/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/GISelAliasTest.cpp
@@ -53,7 +53,7 @@ TEST_F(AArch64GISelMITest, SimpleAlias) {
// Same for atomics.
auto *LoadAtomicMMO = MF->getMachineMemOperand(
PtrInfo, MachineMemOperand::Flags::MOLoad, S64, Align(8), AAMDNodes(),
- nullptr, nullptr, SyncScope::System, AtomicOrdering::Acquire);
+ SyncScope::System, AtomicOrdering::Acquire);
auto AtomicLd1 = B.buildLoad(S64, Addr, *LoadAtomicMMO);
auto AtomicLd2 = B.buildLoad(S64, Base2, *LoadAtomicMMO);
EXPECT_TRUE(
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index 8563d7f1f15c9..7eb381c5713d2 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -1180,7 +1180,9 @@ static void AddRangeMetadata(LLVMContext &Context, MachineInstr *Load) {
MachineMemOperand *NewMMO =
Load->getParent()->getParent()->getMachineMemOperand(
OldMMO->getPointerInfo(), OldMMO->getFlags(), OldMMO->getMemoryType(),
- OldMMO->getAlign(), OldMMO->getAAInfo(), NewMDNode);
+ OldMMO->getAlign(),
+ MachineMemOperand::Metadata(/*AAInfo=*/OldMMO->getAAInfo(),
+ /*Ranges=*/NewMDNode));
MachineIRBuilder MIB(*Load);
MIB.buildLoadInstr(Load->getOpcode(), Load->getOperand(0),
Load->getOperand(1), *NewMMO);
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
index 96c5b39384a8a..321b1380b5efb 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
@@ -162,9 +162,11 @@ TEST_F(AArch64GISelMITest, TestVectorMetadata) {
ConstantAsMetadata::get(ConstantInt::get(Int8Ty, 2))};
auto *NewMDNode = MDNode::get(Context, LowAndHigh);
const MachineMemOperand *OldMMO = *Load->memoperands_begin();
- MachineMemOperand NewMMO(OldMMO->getPointerInfo(), OldMMO->getFlags(),
- OldMMO->getMemoryType(), OldMMO->getAlign(),
- OldMMO->getAAInfo(), NewMDNode);
+ MachineMemOperand NewMMO(
+ OldMMO->getPointerInfo(), OldMMO->getFlags(), OldMMO->getMemoryType(),
+ OldMMO->getAlign(), MachineMemOperand::Metadata(
+ /*AAInfo=*/OldMMO->getAAInfo(),
+ /*Ranges=*/NewMDNode));
MachineIRBuilder MIB(*Load);
MIB.buildLoad(Load->getOperand(0), Load->getOperand(1), NewMMO);
Load->eraseFromParent();
diff --git a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
index 1fc20f6f238b4..24c08169f5cc4 100644
--- a/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -305,8 +305,7 @@ TEST_F(AArch64GISelMITest, BuildAtomicRMW) {
MachineMemOperand *MMO = MF->getMachineMemOperand(
MachinePointerInfo(),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore, 8, Align(8),
- AAMDNodes(), nullptr, nullptr, SyncScope::System,
- AtomicOrdering::Unordered);
+ AAMDNodes(), SyncScope::System, AtomicOrdering::Unordered);
auto Ptr = B.buildUndef(P0);
B.buildAtomicRMWFAdd(S64, Ptr, Copies[0], *MMO);
>From 5a506f7aa21c55d959e24f785ca74eb79f839dd9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Mon, 22 Jun 2026 22:26:35 +0000
Subject: [PATCH 15/33] format
---
llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
index 321b1380b5efb..205ec0c446915 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
@@ -162,9 +162,9 @@ TEST_F(AArch64GISelMITest, TestVectorMetadata) {
ConstantAsMetadata::get(ConstantInt::get(Int8Ty, 2))};
auto *NewMDNode = MDNode::get(Context, LowAndHigh);
const MachineMemOperand *OldMMO = *Load->memoperands_begin();
- MachineMemOperand NewMMO(
- OldMMO->getPointerInfo(), OldMMO->getFlags(), OldMMO->getMemoryType(),
- OldMMO->getAlign(), MachineMemOperand::Metadata(
+ MachineMemOperand NewMMO(OldMMO->getPointerInfo(), OldMMO->getFlags(),
+ OldMMO->getMemoryType(), OldMMO->getAlign(),
+ MachineMemOperand::Metadata(
/*AAInfo=*/OldMMO->getAAInfo(),
/*Ranges=*/NewMDNode));
MachineIRBuilder MIB(*Load);
>From e576195940f616432c528eb8c2d023186bbe2eb4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 23 Jun 2026 00:36:08 +0000
Subject: [PATCH 16/33] fix aarch64 test
---
llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
index b144060afe925..37a5854652426 100644
--- a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
@@ -1387,7 +1387,8 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_known01) {
MDBuilder MDHelper(*DAG->getContext());
MDNode *Range = MDHelper.createRange(APInt(8, 0), APInt(8, 2));
MachineMemOperand *MMO = DAG->getMachineFunction().getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOLoad, 8, Align(8), AA, Range);
+ PtrInfo, MachineMemOperand::MOLoad, 8, Align(8),
+ MachineMemOperand::Metadata(/*AAInfo=*/AA, /*Ranges=*/Range));
auto ALoad = DAG->getExtLoad(ISD::EXTLOAD, Loc, Int32VT, DAG->getEntryNode(),
Ptr, Int8VT, MMO);
@@ -1420,7 +1421,8 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_knownnegative) {
MDBuilder MDHelper(*DAG->getContext());
MDNode *Range = MDHelper.createRange(APInt(8, 0xf0), APInt(8, 0xff));
MachineMemOperand *MMO = DAG->getMachineFunction().getMachineMemOperand(
- PtrInfo, MachineMemOperand::MOLoad, 8, Align(8), AA, Range);
+ PtrInfo, MachineMemOperand::MOLoad, 8, Align(8),
+ MachineMemOperand::Metadata(/*AAInfo=*/AA, /*Ranges=*/Range));
auto ALoad = DAG->getExtLoad(ISD::EXTLOAD, Loc, Int32VT, DAG->getEntryNode(),
Ptr, Int8VT, MMO);
>From d20ad10a843cd9737a04e23045d8dfbdae428f88 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 30 Jun 2026 21:42:56 +0000
Subject: [PATCH 17/33] fix version requirements
---
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 72 +++++--
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 13 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 24 ++-
.../CodeGen/NVPTX/cache-hint-cache-policy.ll | 43 +++-
.../CodeGen/NVPTX/cache-hint-intrinsics.ll | 201 +++++++++---------
.../CodeGen/NVPTX/cache-hint-load-store.ll | 166 +++++++--------
.../CodeGen/NVPTX/cache-hint-sm-version.ll | 107 +++++++---
.../CodeGen/NVPTX/cache-hint-transforms.ll | 2 +-
8 files changed, 371 insertions(+), 257 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index d74e273e22cf1..82a38c94902f3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1163,23 +1163,44 @@ static void parseMemCacheHintStringValue(LLVMContext &Ctx, StringRef Key,
Key + "'");
}
+static bool isGlobalOrGeneric(NVPTX::AddressSpace AddrSpace) {
+ return AddrSpace == NVPTX::AddressSpace::Global ||
+ AddrSpace == NVPTX::AddressSpace::Generic;
+}
+
static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
- NVPTX::L2Prefetch Prefetch) {
+ NVPTX::L2Prefetch Prefetch,
+ NVPTXMemCacheHintAccess Access) {
switch (Prefetch) {
case NVPTX::L2Prefetch::None:
return true;
case NVPTX::L2Prefetch::Bytes64:
- return Subtarget.hasL2Prefetch64B();
+ return Access.IsLoad && isGlobalOrGeneric(Access.AddrSpace) &&
+ Subtarget.hasL2Prefetch64B();
case NVPTX::L2Prefetch::Bytes128:
- return Subtarget.hasL2Prefetch128B();
+ return Access.IsLoad && isGlobalOrGeneric(Access.AddrSpace) &&
+ Subtarget.hasL2Prefetch128B();
case NVPTX::L2Prefetch::Bytes256:
- return Subtarget.hasL2Prefetch256B();
+ return Access.IsLoad && isGlobalOrGeneric(Access.AddrSpace) &&
+ Subtarget.hasL2Prefetch256B();
}
llvm_unreachable("Unexpected L2 prefetch hint");
}
+static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
+ NVPTXMemCacheHintAccess Access,
+ NVPTX::L2Eviction Eviction) {
+ if (Eviction == NVPTX::L2Eviction::Normal)
+ return true;
+
+ return Subtarget.hasL2EvictionHint() &&
+ isGlobalOrGeneric(Access.AddrSpace) &&
+ ((Access.NumElts == 8 && Access.EltWidth == 32) ||
+ (Access.NumElts == 4 && Access.EltWidth == 64));
+}
+
std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
- const MemSDNode *N, unsigned CodeAddrSpace, const SDLoc &DL) {
+ const MemSDNode *N, NVPTXMemCacheHintAccess Access, const SDLoc &DL) {
LLVMContext &Ctx = *CurDAG->getContext();
const MDNode *Node = N->getMemCacheHint();
SDValue PolicyReg = CurDAG->getRegister(NVPTX::NoRegister, MVT::i64);
@@ -1207,8 +1228,11 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
}
if (KeyStr == "nvvm.l2_eviction") {
- if (Subtarget->hasL2EvictionHint())
- parseMemCacheHintStringValue(Ctx, KeyStr, Value, L2, parseL2Eviction);
+ NVPTX::L2Eviction ParsedL2 = NVPTX::L2Eviction::Normal;
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, ParsedL2,
+ parseL2Eviction);
+ if (isL2EvictionSupported(*Subtarget, Access, ParsedL2))
+ L2 = ParsedL2;
continue;
}
@@ -1216,13 +1240,13 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
NVPTX::L2Prefetch ParsedPrefetch = NVPTX::L2Prefetch::None;
parseMemCacheHintStringValue(Ctx, KeyStr, Value, ParsedPrefetch,
parseL2Prefetch);
- if (isL2PrefetchSupported(*Subtarget, ParsedPrefetch))
+ if (isL2PrefetchSupported(*Subtarget, ParsedPrefetch, Access))
Prefetch = ParsedPrefetch;
continue;
}
if (KeyStr == "nvvm.l2_cache_hint") {
- if (CodeAddrSpace == NVPTX::AddressSpace::Global &&
+ if (isGlobalOrGeneric(Access.AddrSpace) &&
Subtarget->hasL2CacheHint()) {
if (auto *ValCI = mdconst::dyn_extract<ConstantInt>(Value))
CachePolicy = ValCI->getZExtValue();
@@ -1292,7 +1316,10 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
const auto [EvictionAndPrefetchHint, PolicyReg] =
- getMemCacheHintOperands(LD, CodeAddrSpace, DL);
+ getMemCacheHintOperands(LD,
+ {CodeAddrSpace, /*IsLoad=*/true,
+ /*NumElts=*/1, /*EltWidth=*/FromTypeWidth},
+ DL);
// Create the machine instruction DAG
SDValue Ops[] = {getI32Imm(Ordering, DL),
@@ -1368,8 +1395,11 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
assert(!(EltVT.isVector() && ExtensionType != ISD::NON_EXTLOAD));
- const auto [EvictionAndPrefetchHint, PolicyReg] =
- getMemCacheHintOperands(LD, CodeAddrSpace, DL);
+ const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
+ LD,
+ {CodeAddrSpace, /*IsLoad=*/true,
+ /*NumElts=*/LD->getNumValues() - 1, /*EltWidth=*/FromTypeWidth},
+ DL);
const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
SDValue Ops[] = {getI32Imm(Ordering, DL),
getI32Imm(Scope, DL),
@@ -1434,8 +1464,11 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
ExtensionType != ISD::NON_EXTLOAD));
const auto [Base, Offset] = selectADDR(LD->getOperand(1), CurDAG);
- const auto [EvictionAndPrefetchHint, PolicyReg] =
- getMemCacheHintOperands(LD, NVPTX::AddressSpace::Global, DL);
+ const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
+ LD,
+ {NVPTX::AddressSpace::Global,
+ /*IsLoad=*/true, LD->getNumValues() - 1, FromTypeWidth},
+ DL);
SDValue Ops[] = {getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL),
getI32Imm(UsedBytesMask, DL),
@@ -1564,7 +1597,10 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
// Extract eviction/prefetch hint and cache policy register.
const auto [EvictionAndPrefetchHint, PolicyReg] =
- getMemCacheHintOperands(ST, CodeAddrSpace, DL);
+ getMemCacheHintOperands(ST,
+ {CodeAddrSpace, /*IsLoad=*/false,
+ /*NumElts=*/1, /*EltWidth=*/ToTypeWidth},
+ DL);
SDValue Ops[] = {selectPossiblyImm(Value),
getI32Imm(Ordering, DL),
@@ -1621,8 +1657,10 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
TotalWidth <= 256 && "Invalid width for store");
// Extract eviction/prefetch hint and cache policy register.
- const auto [EvictionAndPrefetchHint, PolicyReg] =
- getMemCacheHintOperands(ST, CodeAddrSpace, DL);
+ const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
+ ST, {CodeAddrSpace, /*IsLoad=*/false, /*NumElts=*/NumElts,
+ /*EltWidth=*/ToTypeWidth},
+ DL);
const auto [Base, Offset] = selectADDR(Addr, CurDAG);
Ops.append({getI32Imm(Ordering, DL), getI32Imm(Scope, DL),
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index fef115fcea6b0..fb932c10b64c3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -39,6 +39,13 @@ struct NVPTXScopes {
LLVMContext *Context = nullptr;
};
+struct NVPTXMemCacheHintAccess {
+ NVPTX::AddressSpace AddrSpace;
+ bool IsLoad;
+ unsigned NumElts;
+ unsigned EltWidth;
+};
+
class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
const NVPTXTargetMachine &TM;
@@ -110,9 +117,9 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
// dropped. If L2::cache_hint is active, returns the hint with
// L2CacheHintBit set and a register containing the 64-bit cache policy
// value. Otherwise returns NOREG for the policy operand.
- std::pair<unsigned, SDValue> getMemCacheHintOperands(const MemSDNode *N,
- unsigned CodeAddrSpace,
- const SDLoc &DL);
+ std::pair<unsigned, SDValue>
+ getMemCacheHintOperands(const MemSDNode *N, NVPTXMemCacheHintAccess Access,
+ const SDLoc &DL);
// Returns the Memory Order and Scope that the PTX memory instruction should
// use, and inserts appropriate fence instruction before the memory
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index add11e4eea961..6304b3ada26e8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -125,22 +125,24 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
bool hasDotInstructions() const {
return SmVersion >= 61 && PTXVersion >= 50;
}
- // Cache hint SM version requirements
+ // Cache hint SM/PTX version requirements
//
// | Cache Hint | SM Requirement | PTX Requirement |
// |-----------------|----------------|-----------------|
- // | L1::evict_* | SM 70+ | - |
- // | L2::evict_* | SM 70+ | - |
- // | L2::64B | SM 75+ | - |
- // | L2::128B | SM 75+ | - |
- // | L2::256B | SM 80+ | - |
+ // | L1::evict_* | SM 70+ | PTX 7.4+ |
+ // | L2::evict_* | SM 100+ | PTX 8.8+ |
+ // | L2::64B | SM 75+ | PTX 7.4+ |
+ // | L2::128B | SM 75+ | PTX 7.4+ |
+ // | L2::256B | SM 80+ | PTX 7.4+ |
// | L2::cache_hint | SM 80+ | PTX 7.4+ |
//
- bool hasL1EvictionHint() const { return SmVersion >= 70; }
- bool hasL2EvictionHint() const { return SmVersion >= 70; }
- bool hasL2Prefetch64B() const { return SmVersion >= 75; }
- bool hasL2Prefetch128B() const { return SmVersion >= 75; }
- bool hasL2Prefetch256B() const { return SmVersion >= 80; }
+ bool hasL1EvictionHint() const { return SmVersion >= 70 && PTXVersion >= 74; }
+ bool hasL2EvictionHint() const {
+ return SmVersion >= 100 && PTXVersion >= 88;
+ }
+ bool hasL2Prefetch64B() const { return SmVersion >= 75 && PTXVersion >= 74; }
+ bool hasL2Prefetch128B() const { return SmVersion >= 75 && PTXVersion >= 74; }
+ bool hasL2Prefetch256B() const { return SmVersion >= 80 && PTXVersion >= 74; }
bool hasL2CacheHint() const { return SmVersion >= 80 && PTXVersion >= 74; }
// Checks following instructions support:
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll b/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
index bec565949561c..7dff40ebfba64 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|ld(?:\.[A-Za-z0-9_:]+)*\.L2::cache_hint|st(?:\.[A-Za-z0-9_:]+)*\.L2::cache_hint|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
@@ -141,7 +141,27 @@ define void @test_store_cache_hint_v2f64(ptr addrspace(1) %p, <2 x double> %v) {
}
;-----------------------------------------------------------------------------
-; L2::cache_hint combined with other hints (L2::cache_hint takes precedence)
+; L2::cache_hint with generic addressing
+;-----------------------------------------------------------------------------
+
+define i32 @test_generic_load_cache_hint_i32(ptr %p) {
+; CHECK-LABEL: test_generic_load_cache_hint_i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: ld.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+ %v = load i32, ptr %p, !mem.cache_hint !48
+ ret i32 %v
+}
+
+define void @test_generic_store_cache_hint_i32(ptr %p, i32 %v) {
+; CHECK-LABEL: test_generic_store_cache_hint_i32(
+; CHECK: mov.b64 %rd2, 12345;
+; CHECK: st.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+ store i32 %v, ptr %p, !mem.cache_hint !49
+ ret void
+}
+
+;-----------------------------------------------------------------------------
+; L2::cache_hint combined with other hints (valid qualifiers are preserved)
;-----------------------------------------------------------------------------
; L2::cache_hint + L1 eviction: both qualifiers should be emitted
@@ -153,11 +173,11 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
ret i32 %v
}
-; L2::cache_hint + L2 eviction: both qualifiers should be emitted
+; L2::cache_hint + L2 eviction: cache policy is emitted, scalar L2 eviction is dropped
define i32 @test_load_cache_hint_with_l2_eviction(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_with_l2_eviction(
; CHECK: mov.b64 %rd2, 12345;
-; CHECK: ld.global.L2::evict_last.L2::cache_hint.b32 %r1, [%rd1], %rd2;
+; CHECK: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !17
ret i32 %v
}
@@ -171,11 +191,11 @@ define i32 @test_load_cache_hint_with_prefetch(ptr addrspace(1) %p) {
ret i32 %v
}
-; L2::cache_hint + all other hints: all qualifiers emitted
+; L2::cache_hint + all other hints: valid load qualifiers are emitted
define i32 @test_load_cache_hint_with_all(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_with_all(
; CHECK: mov.b64 %rd2, 12345;
-; CHECK: ld.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::256B.b32 %r1, [%rd1], %rd2;
+; CHECK: ld.global.L1::evict_last.L2::cache_hint.L2::256B.b32 %r1, [%rd1], %rd2;
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !19
ret i32 %v
}
@@ -189,11 +209,11 @@ define void @test_store_cache_hint_with_l1(ptr addrspace(1) %p, i32 %v) {
ret void
}
-; Store: L2::cache_hint + L1 + L2 eviction (all qualifiers emitted)
+; Store: L2::cache_hint + L1 + L2 eviction (scalar L2 eviction is dropped)
define void @test_store_cache_hint_with_all(ptr addrspace(1) %p, i32 %v) {
; CHECK-LABEL: test_store_cache_hint_with_all(
; CHECK: mov.b64 %rd2, 12345;
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b32 [%rd1], %r1, %rd2;
+; CHECK: st.global.L1::no_allocate.L2::cache_hint.b32 [%rd1], %r1, %rd2;
store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !21
ret void
}
@@ -207,11 +227,11 @@ define <2 x i32> @test_load_cache_hint_v2i32_with_l1(ptr addrspace(1) %p) {
ret <2 x i32> %v
}
-; Vector store: L2::cache_hint + L1 + L2 eviction + prefetch (all qualifiers emitted)
+; Vector store: cache policy and L1 are emitted; L2 eviction/prefetch are dropped
define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32> %v) {
; CHECK-LABEL: test_store_cache_hint_v2i32_with_all(
; CHECK: mov.b64 %rd2, 12345;
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.L2::64B.v2.b32 [%rd1], {%r1, %r2}, %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.v2.b32 [%rd1], {%r1, %r2}, %rd2;
store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !23
ret void
}
@@ -268,3 +288,6 @@ define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32>
!46 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
!23 = !{i32 1, !47}
!47 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"64B"}
+!48 = !{i32 0, !50}
+!49 = !{i32 1, !50}
+!50 = !{!"nvvm.l2_cache_hint", i64 12345}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll b/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll
index 6919b6b805ed6..59bac44244460 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-intrinsics.ll
@@ -7,6 +7,11 @@
; For memcpy:
; operand_no = 0 applies to destination (store side)
; operand_no = 1 applies to source (load side)
+;
+; TODO: Teach memcpy lowering to preserve L2::evict_* on SM100/PTX 8.8 when
+; an aligned, 32-byte-multiple copy can use a PTX-legal 256-bit vector memory
+; operation. Current memcpy expansion uses scalar or short accesses, so the
+; L2 eviction metadata in the cases below is intentionally dropped.
declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1), ptr addrspace(1), i64, i1)
declare <2 x i16> @llvm.masked.load.v2i16.p1(ptr addrspace(1), <2 x i1>, <2 x i16>)
@@ -20,14 +25,14 @@ declare <4 x i32> @llvm.masked.load.v4i32.p1(ptr addrspace(1), <4 x i1>, <4 x i3
define void @test_memcpy_both_hints(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_both_hints(
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs1, [%rd2+3];
-; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1+3], %rs1;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs2, [%rd2+2];
-; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1+2], %rs2;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs3, [%rd2+1];
-; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1+1], %rs3;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::128B.b8 %rs4, [%rd2];
-; CHECK: st.global.L1::evict_last.L2::evict_last.b8 [%rd1], %rs4;
+; CHECK: ld.global.L1::evict_first.L2::128B.b8 %rs1, [%rd2+3];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L1::evict_first.L2::128B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L1::evict_first.L2::128B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.L1::evict_last.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L1::evict_first.L2::128B.b8 %rs4, [%rd2];
+; CHECK: st.global.L1::evict_last.b8 [%rd1], %rs4;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !80
ret void
}
@@ -53,21 +58,21 @@ define void @test_memcpy_src_hint_only(ptr addrspace(1) %dest, ptr addrspace(1)
}
;-----------------------------------------------------------------------------
-; Test memcpy with cache hint only on destination (store side)
-; Source (operand 1): no hint
-; Dest (operand 0): L2::evict_last
+; Test memcpy with L2 prefetch only on source (load side)
+; Source (operand 1): L2::128B
+; Dest (operand 0): no hint
;-----------------------------------------------------------------------------
-define void @test_memcpy_dest_hint_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
-; CHECK-LABEL: test_memcpy_dest_hint_only(
-; CHECK: ld.global.b8 %rs1, [%rd2+3];
-; CHECK: st.global.L2::evict_last.b8 [%rd1+3], %rs1;
-; CHECK: ld.global.b8 %rs2, [%rd2+2];
-; CHECK: st.global.L2::evict_last.b8 [%rd1+2], %rs2;
-; CHECK: ld.global.b8 %rs3, [%rd2+1];
-; CHECK: st.global.L2::evict_last.b8 [%rd1+1], %rs3;
-; CHECK: ld.global.b8 %rs4, [%rd2];
-; CHECK: st.global.L2::evict_last.b8 [%rd1], %rs4;
+define void @test_memcpy_src_prefetch_only(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
+; CHECK-LABEL: test_memcpy_src_prefetch_only(
+; CHECK: ld.global.L2::128B.b8 %rs1, [%rd2+3];
+; CHECK: st.global.b8 [%rd1+3], %rs1;
+; CHECK: ld.global.L2::128B.b8 %rs2, [%rd2+2];
+; CHECK: st.global.b8 [%rd1+2], %rs2;
+; CHECK: ld.global.L2::128B.b8 %rs3, [%rd2+1];
+; CHECK: st.global.b8 [%rd1+1], %rs3;
+; CHECK: ld.global.L2::128B.b8 %rs4, [%rd2];
+; CHECK: st.global.b8 [%rd1], %rs4;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !82
ret void
}
@@ -117,13 +122,13 @@ define void @test_memcpy_no_hint(ptr addrspace(1) %dest, ptr addrspace(1) %src)
; Dest: no hint
define void @test_memcpy_src_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_src_l1_l2_combined(
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs1, [%rd2+3];
+; CHECK: ld.global.L1::evict_first.b8 %rs1, [%rd2+3];
; CHECK: st.global.b8 [%rd1+3], %rs1;
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs2, [%rd2+2];
+; CHECK: ld.global.L1::evict_first.b8 %rs2, [%rd2+2];
; CHECK: st.global.b8 [%rd1+2], %rs2;
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs3, [%rd2+1];
+; CHECK: ld.global.L1::evict_first.b8 %rs3, [%rd2+1];
; CHECK: st.global.b8 [%rd1+1], %rs3;
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b8 %rs4, [%rd2];
+; CHECK: ld.global.L1::evict_first.b8 %rs4, [%rd2];
; CHECK: st.global.b8 [%rd1], %rs4;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !84
ret void
@@ -134,13 +139,13 @@ define void @test_memcpy_src_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspac
define void @test_memcpy_dest_l1_l2_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_dest_l1_l2_combined(
; CHECK: ld.global.b8 %rs1, [%rd2+3];
-; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1+3], %rs1;
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1+3], %rs1;
; CHECK: ld.global.b8 %rs2, [%rd2+2];
-; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1+2], %rs2;
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1+2], %rs2;
; CHECK: ld.global.b8 %rs3, [%rd2+1];
-; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1+1], %rs3;
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1+1], %rs3;
; CHECK: ld.global.b8 %rs4, [%rd2];
-; CHECK: st.global.L1::evict_unchanged.L2::evict_first.b8 [%rd1], %rs4;
+; CHECK: st.global.L1::evict_unchanged.b8 [%rd1], %rs4;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !85
ret void
}
@@ -170,13 +175,13 @@ define void @test_memcpy_l1_prefetch(ptr addrspace(1) %dest, ptr addrspace(1) %s
define void @test_memcpy_prefetch_vs_eviction(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_prefetch_vs_eviction(
; CHECK: ld.global.L2::64B.b8 %rs1, [%rd2+3];
-; CHECK: st.global.L2::evict_last.b8 [%rd1+3], %rs1;
+; CHECK: st.global.b8 [%rd1+3], %rs1;
; CHECK: ld.global.L2::64B.b8 %rs2, [%rd2+2];
-; CHECK: st.global.L2::evict_last.b8 [%rd1+2], %rs2;
+; CHECK: st.global.b8 [%rd1+2], %rs2;
; CHECK: ld.global.L2::64B.b8 %rs3, [%rd2+1];
-; CHECK: st.global.L2::evict_last.b8 [%rd1+1], %rs3;
+; CHECK: st.global.b8 [%rd1+1], %rs3;
; CHECK: ld.global.L2::64B.b8 %rs4, [%rd2];
-; CHECK: st.global.L2::evict_last.b8 [%rd1], %rs4;
+; CHECK: st.global.b8 [%rd1], %rs4;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !87
ret void
}
@@ -208,13 +213,13 @@ define void @test_memcpy_dest_cache_hint_combined(ptr addrspace(1) %dest, ptr ad
; CHECK-LABEL: test_memcpy_dest_cache_hint_combined(
; CHECK: ld.global.b8 %rs1, [%rd2+3];
; CHECK: mov.b64 %rd3, 12345;
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1+3], %rs1, %rd3;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+3], %rs1, %rd3;
; CHECK: ld.global.b8 %rs2, [%rd2+2];
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1+2], %rs2, %rd3;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+2], %rs2, %rd3;
; CHECK: ld.global.b8 %rs3, [%rd2+1];
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1+1], %rs3, %rd3;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+1], %rs3, %rd3;
; CHECK: ld.global.b8 %rs4, [%rd2];
-; CHECK: st.global.L1::evict_last.L2::evict_first.L2::cache_hint.b8 [%rd1], %rs4, %rd3;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1], %rs4, %rd3;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !89
ret void
}
@@ -225,14 +230,14 @@ define void @test_memcpy_dest_cache_hint_combined(ptr addrspace(1) %dest, ptr ad
define void @test_memcpy_both_cache_hint_combined(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_both_cache_hint_combined(
; CHECK: mov.b64 %rd2, 12345;
-; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs1, [%rd3+3], %rd2;
-; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1+3], %rs1, %rd2;
-; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs2, [%rd3+2], %rd2;
-; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1+2], %rs2, %rd2;
-; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs3, [%rd3+1], %rd2;
-; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1+1], %rs3, %rd2;
-; CHECK: ld.global.L1::evict_unchanged.L2::evict_last.L2::cache_hint.b8 %rs4, [%rd3], %rd2;
-; CHECK: st.global.L1::evict_first.L2::evict_first.L2::cache_hint.b8 [%rd1], %rs4, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::cache_hint.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.L1::evict_first.L2::cache_hint.b8 [%rd1+3], %rs1, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::cache_hint.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.L1::evict_first.L2::cache_hint.b8 [%rd1+2], %rs2, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::cache_hint.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.L1::evict_first.L2::cache_hint.b8 [%rd1+1], %rs3, %rd2;
+; CHECK: ld.global.L1::evict_unchanged.L2::cache_hint.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.L1::evict_first.L2::cache_hint.b8 [%rd1], %rs4, %rd2;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !90
ret void
}
@@ -266,13 +271,13 @@ define void @test_memcpy_cache_hint_prefetch(ptr addrspace(1) %dest, ptr addrspa
; Dest: simple L1 hint only
define void @test_memcpy_complex_src_simple_dest(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_complex_src_simple_dest(
-; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs1, [%rd2+3];
+; CHECK: ld.global.L1::evict_first.L2::64B.b8 %rs1, [%rd2+3];
; CHECK: st.global.L1::evict_last.b8 [%rd1+3], %rs1;
-; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs2, [%rd2+2];
+; CHECK: ld.global.L1::evict_first.L2::64B.b8 %rs2, [%rd2+2];
; CHECK: st.global.L1::evict_last.b8 [%rd1+2], %rs2;
-; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs3, [%rd2+1];
+; CHECK: ld.global.L1::evict_first.L2::64B.b8 %rs3, [%rd2+1];
; CHECK: st.global.L1::evict_last.b8 [%rd1+1], %rs3;
-; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::64B.b8 %rs4, [%rd2];
+; CHECK: ld.global.L1::evict_first.L2::64B.b8 %rs4, [%rd2];
; CHECK: st.global.L1::evict_last.b8 [%rd1], %rs4;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !92
ret void
@@ -284,13 +289,13 @@ define void @test_memcpy_simple_src_complex_dest(ptr addrspace(1) %dest, ptr add
; CHECK-LABEL: test_memcpy_simple_src_complex_dest(
; CHECK: ld.global.L2::256B.b8 %rs1, [%rd2+3];
; CHECK: mov.b64 %rd3, 12345;
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1+3], %rs1, %rd3;
+; CHECK: st.global.L1::no_allocate.L2::cache_hint.b8 [%rd1+3], %rs1, %rd3;
; CHECK: ld.global.L2::256B.b8 %rs2, [%rd2+2];
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1+2], %rs2, %rd3;
+; CHECK: st.global.L1::no_allocate.L2::cache_hint.b8 [%rd1+2], %rs2, %rd3;
; CHECK: ld.global.L2::256B.b8 %rs3, [%rd2+1];
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1+1], %rs3, %rd3;
+; CHECK: st.global.L1::no_allocate.L2::cache_hint.b8 [%rd1+1], %rs3, %rd3;
; CHECK: ld.global.L2::256B.b8 %rs4, [%rd2];
-; CHECK: st.global.L1::no_allocate.L2::evict_last.L2::cache_hint.b8 [%rd1], %rs4, %rd3;
+; CHECK: st.global.L1::no_allocate.L2::cache_hint.b8 [%rd1], %rs4, %rd3;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !93
ret void
}
@@ -340,14 +345,14 @@ define void @test_memcpy_no_allocate_vs_unchanged(ptr addrspace(1) %dest, ptr ad
define void @test_memcpy_all_hints_both(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_all_hints_both(
; CHECK: mov.b64 %rd2, 12345;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs1, [%rd3+3], %rd2;
-; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1+3], %rs1, %rd2;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs2, [%rd3+2], %rd2;
-; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1+2], %rs2, %rd2;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs3, [%rd3+1], %rd2;
-; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1+1], %rs3, %rd2;
-; CHECK: ld.global.L1::evict_first.L2::evict_first.L2::cache_hint.L2::256B.b8 %rs4, [%rd3], %rd2;
-; CHECK: st.global.L1::evict_last.L2::evict_last.L2::cache_hint.L2::128B.b8 [%rd1], %rs4, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.L2::256B.b8 %rs1, [%rd3+3], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+3], %rs1, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.L2::256B.b8 %rs2, [%rd3+2], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+2], %rs2, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.L2::256B.b8 %rs3, [%rd3+1], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1+1], %rs3, %rd2;
+; CHECK: ld.global.L1::evict_first.L2::cache_hint.L2::256B.b8 %rs4, [%rd3], %rd2;
+; CHECK: st.global.L1::evict_last.L2::cache_hint.b8 [%rd1], %rs4, %rd2;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 4, i1 false), !mem.cache_hint !96
ret void
}
@@ -421,73 +426,75 @@ define void @test_memcpy_16bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src)
ret void
}
-; 32-byte memcpy: all loads should have L1::evict_unchanged, all stores L2::evict_first
+; 32-byte memcpy: all loads should have L1::evict_unchanged.
+; TODO: Preserve the destination L2::evict_first metadata when memcpy lowering
+; can select a PTX-legal 256-bit vector store for aligned 32-byte copies.
define void @test_memcpy_32bytes(ptr addrspace(1) %dest, ptr addrspace(1) %src) {
; CHECK-LABEL: test_memcpy_32bytes(
; CHECK: ld.global.L1::evict_unchanged.b8 %rs1, [%rd2+31];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+31], %rs1;
+; CHECK: st.global.b8 [%rd1+31], %rs1;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs2, [%rd2+30];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+30], %rs2;
+; CHECK: st.global.b8 [%rd1+30], %rs2;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs3, [%rd2+29];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+29], %rs3;
+; CHECK: st.global.b8 [%rd1+29], %rs3;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs4, [%rd2+28];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+28], %rs4;
+; CHECK: st.global.b8 [%rd1+28], %rs4;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs5, [%rd2+27];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+27], %rs5;
+; CHECK: st.global.b8 [%rd1+27], %rs5;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs6, [%rd2+26];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+26], %rs6;
+; CHECK: st.global.b8 [%rd1+26], %rs6;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs7, [%rd2+25];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+25], %rs7;
+; CHECK: st.global.b8 [%rd1+25], %rs7;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs8, [%rd2+24];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+24], %rs8;
+; CHECK: st.global.b8 [%rd1+24], %rs8;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs9, [%rd2+23];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+23], %rs9;
+; CHECK: st.global.b8 [%rd1+23], %rs9;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs10, [%rd2+22];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+22], %rs10;
+; CHECK: st.global.b8 [%rd1+22], %rs10;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs11, [%rd2+21];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+21], %rs11;
+; CHECK: st.global.b8 [%rd1+21], %rs11;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs12, [%rd2+20];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+20], %rs12;
+; CHECK: st.global.b8 [%rd1+20], %rs12;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs13, [%rd2+19];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+19], %rs13;
+; CHECK: st.global.b8 [%rd1+19], %rs13;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs14, [%rd2+18];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+18], %rs14;
+; CHECK: st.global.b8 [%rd1+18], %rs14;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs15, [%rd2+17];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+17], %rs15;
+; CHECK: st.global.b8 [%rd1+17], %rs15;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs16, [%rd2+16];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+16], %rs16;
+; CHECK: st.global.b8 [%rd1+16], %rs16;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs17, [%rd2+15];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+15], %rs17;
+; CHECK: st.global.b8 [%rd1+15], %rs17;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs18, [%rd2+14];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+14], %rs18;
+; CHECK: st.global.b8 [%rd1+14], %rs18;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs19, [%rd2+13];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+13], %rs19;
+; CHECK: st.global.b8 [%rd1+13], %rs19;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs20, [%rd2+12];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+12], %rs20;
+; CHECK: st.global.b8 [%rd1+12], %rs20;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs21, [%rd2+11];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+11], %rs21;
+; CHECK: st.global.b8 [%rd1+11], %rs21;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs22, [%rd2+10];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+10], %rs22;
+; CHECK: st.global.b8 [%rd1+10], %rs22;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs23, [%rd2+9];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+9], %rs23;
+; CHECK: st.global.b8 [%rd1+9], %rs23;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs24, [%rd2+8];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+8], %rs24;
+; CHECK: st.global.b8 [%rd1+8], %rs24;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs25, [%rd2+7];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+7], %rs25;
+; CHECK: st.global.b8 [%rd1+7], %rs25;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs26, [%rd2+6];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+6], %rs26;
+; CHECK: st.global.b8 [%rd1+6], %rs26;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs27, [%rd2+5];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+5], %rs27;
+; CHECK: st.global.b8 [%rd1+5], %rs27;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs28, [%rd2+4];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+4], %rs28;
+; CHECK: st.global.b8 [%rd1+4], %rs28;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs29, [%rd2+3];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+3], %rs29;
+; CHECK: st.global.b8 [%rd1+3], %rs29;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs30, [%rd2+2];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+2], %rs30;
+; CHECK: st.global.b8 [%rd1+2], %rs30;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs31, [%rd2+1];
-; CHECK: st.global.L2::evict_first.b8 [%rd1+1], %rs31;
+; CHECK: st.global.b8 [%rd1+1], %rs31;
; CHECK: ld.global.L1::evict_unchanged.b8 %rs32, [%rd2];
-; CHECK: st.global.L2::evict_first.b8 [%rd1], %rs32;
+; CHECK: st.global.b8 [%rd1], %rs32;
call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 32, i1 false), !mem.cache_hint !98
ret void
}
@@ -520,9 +527,9 @@ define void @test_memcpy_128bytes_combined(ptr addrspace(1) %dest, ptr addrspace
!81 = !{i32 1, !182}
!182 = !{!"nvvm.l1_eviction", !"first"}
-; memcpy with only dest hint (store side)
-!82 = !{i32 0, !183}
-!183 = !{!"nvvm.l2_eviction", !"last"}
+; memcpy with source L2 prefetch (load side)
+!82 = !{i32 1, !183}
+!183 = !{!"nvvm.l2_prefetch_size", !"128B"}
; memcpy with L2::cache_hint on both operands
!83 = !{i32 0, !184, i32 1, !185}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll b/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll
index 2e715ad316ca0..80601623b127c 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-load-store.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | %ptxas-verify %}
; Test !mem.cache_hint metadata lowering to PTX load/store cache qualifiers.
@@ -37,21 +37,21 @@ define i32 @test_load_l1_no_allocate(ptr addrspace(1) %p) {
}
;-----------------------------------------------------------------------------
-; Basic L2 eviction policies for loads
+; L2 eviction policies for loads (<8 x i32> and <4 x i64> are PTX-legal 256-bit forms)
;-----------------------------------------------------------------------------
-define i32 @test_load_l2_first(ptr addrspace(1) %p) {
+define <8 x i32> @test_load_l2_first(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l2_first(
-; CHECK: ld.global.L2::evict_first.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
- ret i32 %v
+; CHECK: ld.global.L2::evict_first.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <8 x i32>, ptr addrspace(1) %p, align 32, !mem.cache_hint !4
+ ret <8 x i32> %v
}
-define i32 @test_load_l2_last(ptr addrspace(1) %p) {
+define <4 x i64> @test_load_l2_last(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l2_last(
-; CHECK: ld.global.L2::evict_last.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !5
- ret i32 %v
+; CHECK: ld.global.L2::evict_last.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <4 x i64>, ptr addrspace(1) %p, align 32, !mem.cache_hint !5
+ ret <4 x i64> %v
}
;-----------------------------------------------------------------------------
@@ -80,46 +80,46 @@ define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
}
;-----------------------------------------------------------------------------
-; All L1 + L2 combinations for loads
+; L1 + L2 eviction combinations for loads
;-----------------------------------------------------------------------------
-define i32 @test_load_l1_first_l2_first(ptr addrspace(1) %p) {
+define <8 x i32> @test_load_l1_first_l2_first(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l1_first_l2_first(
-; CHECK: ld.global.L1::evict_first.L2::evict_first.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !9
- ret i32 %v
+; CHECK: ld.global.L1::evict_first.L2::evict_first.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <8 x i32>, ptr addrspace(1) %p, align 32, !mem.cache_hint !9
+ ret <8 x i32> %v
}
-define i32 @test_load_l1_first_l2_last(ptr addrspace(1) %p) {
+define <4 x i64> @test_load_l1_first_l2_last(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l1_first_l2_last(
-; CHECK: ld.global.L1::evict_first.L2::evict_last.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !10
- ret i32 %v
+; CHECK: ld.global.L1::evict_first.L2::evict_last.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <4 x i64>, ptr addrspace(1) %p, align 32, !mem.cache_hint !10
+ ret <4 x i64> %v
}
-define i32 @test_load_l1_last_l2_first(ptr addrspace(1) %p) {
+define <8 x i32> @test_load_l1_last_l2_first(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l1_last_l2_first(
-; CHECK: ld.global.L1::evict_last.L2::evict_first.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !11
- ret i32 %v
+; CHECK: ld.global.L1::evict_last.L2::evict_first.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <8 x i32>, ptr addrspace(1) %p, align 32, !mem.cache_hint !11
+ ret <8 x i32> %v
}
-define i32 @test_load_l1_last_l2_last(ptr addrspace(1) %p) {
+define <4 x i64> @test_load_l1_last_l2_last(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l1_last_l2_last(
-; CHECK: ld.global.L1::evict_last.L2::evict_last.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !12
- ret i32 %v
+; CHECK: ld.global.L1::evict_last.L2::evict_last.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <4 x i64>, ptr addrspace(1) %p, align 32, !mem.cache_hint !12
+ ret <4 x i64> %v
}
;-----------------------------------------------------------------------------
; L1 + L2 + Prefetch combination for loads
;-----------------------------------------------------------------------------
-define i32 @test_load_l1_first_l2_last_prefetch_128(ptr addrspace(1) %p) {
+define <4 x i64> @test_load_l1_first_l2_last_prefetch_128(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_l1_first_l2_last_prefetch_128(
-; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::128B.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !13
- ret i32 %v
+; CHECK: ld.global.L1::evict_first.L2::evict_last.L2::128B.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <4 x i64>, ptr addrspace(1) %p, align 32, !mem.cache_hint !13
+ ret <4 x i64> %v
}
;-----------------------------------------------------------------------------
@@ -155,52 +155,52 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
}
;-----------------------------------------------------------------------------
-; Basic L2 eviction policies for stores
+; L2 eviction policies for stores
;-----------------------------------------------------------------------------
-define void @test_store_l2_first(ptr addrspace(1) %p, i32 %v) {
+define void @test_store_l2_first(ptr addrspace(1) %p, <8 x i32> %v) {
; CHECK-LABEL: test_store_l2_first(
-; CHECK: st.global.L2::evict_first.b32 [%rd1], %r1;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !18
+; CHECK: st.global.L2::evict_first.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <8 x i32> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !18
ret void
}
-define void @test_store_l2_last(ptr addrspace(1) %p, i32 %v) {
+define void @test_store_l2_last(ptr addrspace(1) %p, <4 x i64> %v) {
; CHECK-LABEL: test_store_l2_last(
-; CHECK: st.global.L2::evict_last.b32 [%rd1], %r1;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !19
+; CHECK: st.global.L2::evict_last.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <4 x i64> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !19
ret void
}
;-----------------------------------------------------------------------------
-; All L1 + L2 combinations for stores
+; L1 + L2 eviction combinations for stores
;-----------------------------------------------------------------------------
-define void @test_store_l1_first_l2_first(ptr addrspace(1) %p, i32 %v) {
+define void @test_store_l1_first_l2_first(ptr addrspace(1) %p, <8 x i32> %v) {
; CHECK-LABEL: test_store_l1_first_l2_first(
-; CHECK: st.global.L1::evict_first.L2::evict_first.b32 [%rd1], %r1;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !20
+; CHECK: st.global.L1::evict_first.L2::evict_first.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <8 x i32> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !20
ret void
}
-define void @test_store_l1_first_l2_last(ptr addrspace(1) %p, i32 %v) {
+define void @test_store_l1_first_l2_last(ptr addrspace(1) %p, <4 x i64> %v) {
; CHECK-LABEL: test_store_l1_first_l2_last(
-; CHECK: st.global.L1::evict_first.L2::evict_last.b32 [%rd1], %r1;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !21
+; CHECK: st.global.L1::evict_first.L2::evict_last.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <4 x i64> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !21
ret void
}
-define void @test_store_l1_last_l2_first(ptr addrspace(1) %p, i32 %v) {
+define void @test_store_l1_last_l2_first(ptr addrspace(1) %p, <8 x i32> %v) {
; CHECK-LABEL: test_store_l1_last_l2_first(
-; CHECK: st.global.L1::evict_last.L2::evict_first.b32 [%rd1], %r1;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !22
+; CHECK: st.global.L1::evict_last.L2::evict_first.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <8 x i32> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !22
ret void
}
-define void @test_store_l1_last_l2_last(ptr addrspace(1) %p, i32 %v) {
+define void @test_store_l1_last_l2_last(ptr addrspace(1) %p, <4 x i64> %v) {
; CHECK-LABEL: test_store_l1_last_l2_last(
-; CHECK: st.global.L1::evict_last.L2::evict_last.b32 [%rd1], %r1;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !23
+; CHECK: st.global.L1::evict_last.L2::evict_last.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <4 x i64> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !23
ret void
}
@@ -222,18 +222,18 @@ define i64 @test_load_i64_l1_last(ptr addrspace(1) %p) {
ret i64 %v
}
-define float @test_load_f32_l2_first(ptr addrspace(1) %p) {
-; CHECK-LABEL: test_load_f32_l2_first(
-; CHECK: ld.global.L2::evict_first.b32 %r1, [%rd1];
- %v = load float, ptr addrspace(1) %p, !mem.cache_hint !4
- ret float %v
+define <8 x float> @test_load_v8f32_l2_first(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v8f32_l2_first(
+; CHECK: ld.global.L2::evict_first.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <8 x float>, ptr addrspace(1) %p, align 32, !mem.cache_hint !4
+ ret <8 x float> %v
}
-define double @test_load_f64_l2_last(ptr addrspace(1) %p) {
-; CHECK-LABEL: test_load_f64_l2_last(
-; CHECK: ld.global.L2::evict_last.b64 %rd2, [%rd1];
- %v = load double, ptr addrspace(1) %p, !mem.cache_hint !5
- ret double %v
+define <4 x double> @test_load_v4f64_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v4f64_l2_last(
+; CHECK: ld.global.L2::evict_last.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <4 x double>, ptr addrspace(1) %p, align 32, !mem.cache_hint !5
+ ret <4 x double> %v
}
;-----------------------------------------------------------------------------
@@ -247,10 +247,10 @@ define void @test_store_i16_l1_first(ptr addrspace(1) %p, i16 %v) {
ret void
}
-define void @test_store_i64_l2_last(ptr addrspace(1) %p, i64 %v) {
-; CHECK-LABEL: test_store_i64_l2_last(
-; CHECK: st.global.L2::evict_last.b64 [%rd1], %rd2;
- store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !19
+define void @test_store_v4i64_l2_last(ptr addrspace(1) %p, <4 x i64> %v) {
+; CHECK-LABEL: test_store_v4i64_l2_last(
+; CHECK: st.global.L2::evict_last.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <4 x i64> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !19
ret void
}
@@ -267,21 +267,21 @@ define void @test_store_f32_l1_no_allocate(ptr addrspace(1) %p, float %v) {
define <2 x i32> @test_load_v2i32_l1_first(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_v2i32_l1_first(
-; CHECK: ld.global.L1::evict_first.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK: ld.global.L1::evict_first.b64 %rd2, [%rd1];
%v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !0
ret <2 x i32> %v
}
-define <4 x i32> @test_load_v4i32_l2_last(ptr addrspace(1) %p) {
-; CHECK-LABEL: test_load_v4i32_l2_last(
-; CHECK: ld.global.L2::evict_last.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
- %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !5
- ret <4 x i32> %v
+define <4 x i64> @test_load_v4i64_l2_last(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_load_v4i64_l2_last(
+; CHECK: ld.global.L2::evict_last.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <4 x i64>, ptr addrspace(1) %p, align 32, !mem.cache_hint !5
+ ret <4 x i64> %v
}
define <2 x float> @test_load_v2f32_l1_unchanged(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_v2f32_l1_unchanged(
-; CHECK: ld.global.L1::evict_unchanged.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK: ld.global.L1::evict_unchanged.b64 %rd2, [%rd1];
%v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !2
ret <2 x float> %v
}
@@ -299,15 +299,15 @@ define <2 x double> @test_load_v2f64_prefetch_128(ptr addrspace(1) %p) {
define void @test_store_v2i32_l1_last(ptr addrspace(1) %p, <2 x i32> %v) {
; CHECK-LABEL: test_store_v2i32_l1_last(
-; CHECK: st.global.L1::evict_last.v2.b32 [%rd1], {%r1, %r2};
+; CHECK: st.global.L1::evict_last.b64 [%rd1], %rd2;
store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !15
ret void
}
-define void @test_store_v4i32_l2_first(ptr addrspace(1) %p, <4 x i32> %v) {
-; CHECK-LABEL: test_store_v4i32_l2_first(
-; CHECK: st.global.L2::evict_first.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
- store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !18
+define void @test_store_v8i32_l2_first(ptr addrspace(1) %p, <8 x i32> %v) {
+; CHECK-LABEL: test_store_v8i32_l2_first(
+; CHECK: st.global.L2::evict_first.v4.b64 [%rd1], {%rd4, %rd5, %rd2, %rd3};
+ store <8 x i32> %v, ptr addrspace(1) %p, align 32, !mem.cache_hint !18
ret void
}
@@ -337,11 +337,11 @@ define i32 @test_invariant_load_with_cache_policy(ptr addrspace(1) %p) {
ret i32 %v
}
-define <2 x i32> @test_invariant_load_v2i32_with_hint(ptr addrspace(1) %p) {
-; CHECK-LABEL: test_invariant_load_v2i32_with_hint(
-; CHECK: ld.global.nc.L1::evict_last.L2::evict_first.v2.b32 {%r1, %r2}, [%rd1];
- %v = load <2 x i32>, ptr addrspace(1) %p, !invariant.load !{}, !mem.cache_hint !11
- ret <2 x i32> %v
+define <8 x i32> @test_invariant_load_v8i32_with_hint(ptr addrspace(1) %p) {
+; CHECK-LABEL: test_invariant_load_v8i32_with_hint(
+; CHECK: ld.global.nc.L1::evict_last.L2::evict_first.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <8 x i32>, ptr addrspace(1) %p, align 32, !invariant.load !{}, !mem.cache_hint !11
+ ret <8 x i32> %v
}
;-----------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
index 2eff4fe830585..511a0fd4a41ca 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-sm-version.ll
@@ -1,29 +1,35 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(ld\.global|st\.global|mov\.b64)" --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global)" --version 6
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM60
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx73 | FileCheck %s --check-prefixes=SM70-PTX73
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM70
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_75 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM75
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s --check-prefixes=SM80-PTX70
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_86 -mattr=+ptx74 | FileCheck %s --check-prefixes=SM80PLUS
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s --check-prefixes=SM80PLUS
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx86 | FileCheck %s --check-prefixes=SM100-PTX86
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s --check-prefixes=SM100-PTX88
; Test SM version requirements for cache hints (from PTX ISA documentation):
-; - L1::evict_* requires SM 70+
-; - L2::evict_* requires SM 70+
-; - L2::64B and L2::128B require SM 75+
-; - L2::256B requires SM 80+
+; - L1::evict_* requires SM 70+ and PTX 7.4+
+; - L2::evict_* requires SM 100+, PTX 8.8+, and a .v8.b32 or .v4.b64 memory op
+; - L2::64B and L2::128B require SM 75+ and PTX 7.4+
+; - L2::256B requires SM 80+ and PTX 7.4+
; - L2::cache_hint requires SM 80+ and PTX 7.4+
;-----------------------------------------------------------------------------
-; L1 eviction - requires SM 70+
-; SM60 should NOT emit L1::evict_first (fall back to plain load)
-; SM70+ should emit L1::evict_first
+; L1 eviction - requires SM 70+ and PTX 7.4+
+; SM60 and PTX 7.3 should NOT emit L1::evict_first
+; SM70+ with PTX 7.4+ should emit L1::evict_first
;-----------------------------------------------------------------------------
define i32 @test_load_l1_first(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_l1_first(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_l1_first(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_l1_first(
; SM70: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
@@ -34,46 +40,56 @@ define i32 @test_load_l1_first(ptr addrspace(1) %p) {
; SM80PLUS: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_l1_first(
-; SM80-PTX70: ld.global.L1::evict_first.b32 %r1, [%rd1];
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !0
ret i32 %v
}
;-----------------------------------------------------------------------------
-; L2 eviction - requires SM 70+
-; SM60 should NOT emit L2::evict_last (fall back to plain load)
-; SM70+ should emit L2::evict_last
+; L2 eviction - requires SM 100+, PTX 8.8+, and a 256-bit vector access.
;-----------------------------------------------------------------------------
-define i32 @test_load_l2_last(ptr addrspace(1) %p) {
+define <8 x i32> @test_load_l2_last(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_l2_last(
-; SM60: ld.global.b32 %r1, [%rd1];
+; SM60: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+;
+; SM70-PTX73-LABEL: test_load_l2_last(
+; SM70-PTX73: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
;
; SM70-LABEL: test_load_l2_last(
-; SM70: ld.global.L2::evict_last.b32 %r1, [%rd1];
+; SM70: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
;
; SM75-LABEL: test_load_l2_last(
-; SM75: ld.global.L2::evict_last.b32 %r1, [%rd1];
+; SM75: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
;
; SM80PLUS-LABEL: test_load_l2_last(
-; SM80PLUS: ld.global.L2::evict_last.b32 %r1, [%rd1];
+; SM80PLUS: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
;
; SM80-PTX70-LABEL: test_load_l2_last(
-; SM80-PTX70: ld.global.L2::evict_last.b32 %r1, [%rd1];
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !1
- ret i32 %v
+; SM80-PTX70: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+;
+; SM100-PTX86-LABEL: test_load_l2_last(
+; SM100-PTX86: ld.global.v2.b64 {%rd2, %rd3}, [%rd1];
+;
+; SM100-PTX88-LABEL: test_load_l2_last(
+; SM100-PTX88: ld.global.L2::evict_last.v4.b64 {%rd2, %rd3, %rd4, %rd5}, [%rd1];
+ %v = load <8 x i32>, ptr addrspace(1) %p, align 32, !mem.cache_hint !1
+ ret <8 x i32> %v
}
;-----------------------------------------------------------------------------
-; L2::64B prefetch - requires SM 75+
-; SM60/SM70 should NOT emit L2::64B (fall back to plain load)
-; SM75+ should emit L2::64B
+; L2::64B prefetch - requires SM 75+ and PTX 7.4+
+; SM60/SM70 and PTX 7.0 should NOT emit L2::64B
+; SM75+ with PTX 7.4+ should emit L2::64B
;-----------------------------------------------------------------------------
define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_prefetch_64(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_prefetch_64(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_prefetch_64(
; SM70: ld.global.b32 %r1, [%rd1];
;
@@ -84,21 +100,24 @@ define i32 @test_load_prefetch_64(ptr addrspace(1) %p) {
; SM80PLUS: ld.global.L2::64B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_64(
-; SM80-PTX70: ld.global.L2::64B.b32 %r1, [%rd1];
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
ret i32 %v
}
;-----------------------------------------------------------------------------
-; L2::128B prefetch - requires SM 75+
-; SM60/SM70 should NOT emit L2::128B (fall back to plain load)
-; SM75+ should emit L2::128B
+; L2::128B prefetch - requires SM 75+ and PTX 7.4+
+; SM60/SM70 and PTX 7.0 should NOT emit L2::128B
+; SM75+ with PTX 7.4+ should emit L2::128B
;-----------------------------------------------------------------------------
define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_prefetch_128(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_prefetch_128(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_prefetch_128(
; SM70: ld.global.b32 %r1, [%rd1];
;
@@ -109,21 +128,24 @@ define i32 @test_load_prefetch_128(ptr addrspace(1) %p) {
; SM80PLUS: ld.global.L2::128B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_128(
-; SM80-PTX70: ld.global.L2::128B.b32 %r1, [%rd1];
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !2
ret i32 %v
}
;-----------------------------------------------------------------------------
-; L2::256B prefetch - requires SM 80+
-; SM60/SM70/SM75 should NOT emit L2::256B (fall back to plain load)
-; SM80+ should emit L2::256B
+; L2::256B prefetch - requires SM 80+ and PTX 7.4+
+; SM60/SM70/SM75 and PTX 7.0 should NOT emit L2::256B
+; SM80+ with PTX 7.4+ should emit L2::256B
;-----------------------------------------------------------------------------
define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_prefetch_256(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_prefetch_256(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_prefetch_256(
; SM70: ld.global.b32 %r1, [%rd1];
;
@@ -134,7 +156,7 @@ define i32 @test_load_prefetch_256(ptr addrspace(1) %p) {
; SM80PLUS: ld.global.L2::256B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_256(
-; SM80-PTX70: ld.global.L2::256B.b32 %r1, [%rd1];
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
ret i32 %v
}
@@ -150,6 +172,9 @@ define i32 @test_load_cache_hint(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_cache_hint(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_cache_hint(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_cache_hint(
; SM70: ld.global.b32 %r1, [%rd1];
;
@@ -177,6 +202,9 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_cache_hint_with_l1(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_cache_hint_with_l1(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_cache_hint_with_l1(
; SM70: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
@@ -188,7 +216,7 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
; SM80PLUS: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
;
; SM80-PTX70-LABEL: test_load_cache_hint_with_l1(
-; SM80-PTX70: ld.global.L1::evict_first.b32 %r1, [%rd1];
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !4
ret i32 %v
}
@@ -204,6 +232,9 @@ define i32 @test_load_prefetch_with_l1(ptr addrspace(1) %p) {
; SM60-LABEL: test_load_prefetch_with_l1(
; SM60: ld.global.b32 %r1, [%rd1];
;
+; SM70-PTX73-LABEL: test_load_prefetch_with_l1(
+; SM70-PTX73: ld.global.b32 %r1, [%rd1];
+;
; SM70-LABEL: test_load_prefetch_with_l1(
; SM70: ld.global.L1::evict_first.b32 %r1, [%rd1];
;
@@ -214,7 +245,7 @@ define i32 @test_load_prefetch_with_l1(ptr addrspace(1) %p) {
; SM80PLUS: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
;
; SM80-PTX70-LABEL: test_load_prefetch_with_l1(
-; SM80-PTX70: ld.global.L1::evict_first.L2::128B.b32 %r1, [%rd1];
+; SM80-PTX70: ld.global.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !8
ret i32 %v
}
@@ -227,6 +258,9 @@ define void @test_store_cache_hint(ptr addrspace(1) %p, i32 %v) {
; SM60-LABEL: test_store_cache_hint(
; SM60: st.global.b32 [%rd1], %r1;
;
+; SM70-PTX73-LABEL: test_store_cache_hint(
+; SM70-PTX73: st.global.b32 [%rd1], %r1;
+;
; SM70-LABEL: test_store_cache_hint(
; SM70: st.global.b32 [%rd1], %r1;
;
@@ -251,6 +285,9 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
; SM60-LABEL: test_store_l1_no_allocate(
; SM60: st.global.b32 [%rd1], %r1;
;
+; SM70-PTX73-LABEL: test_store_l1_no_allocate(
+; SM70-PTX73: st.global.b32 [%rd1], %r1;
+;
; SM70-LABEL: test_store_l1_no_allocate(
; SM70: st.global.L1::no_allocate.b32 [%rd1], %r1;
;
@@ -261,7 +298,7 @@ define void @test_store_l1_no_allocate(ptr addrspace(1) %p, i32 %v) {
; SM80PLUS: st.global.L1::no_allocate.b32 [%rd1], %r1;
;
; SM80-PTX70-LABEL: test_store_l1_no_allocate(
-; SM80-PTX70: st.global.L1::no_allocate.b32 [%rd1], %r1;
+; SM80-PTX70: st.global.b32 [%rd1], %r1;
store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !9
ret void
}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
index fb54c0a5cb4d2..04b091514d0d4 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
@@ -64,7 +64,7 @@ define i32 @test_forward_param_drops_l2_cache_hint(ptr byval(i32) %a) {
; Test with custom hint key order - should still work
define i32 @test_load_reordered_metadata(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_reordered_metadata(
-; CHECK: ld.global.L1::evict_last.L2::evict_first.b32 %r1, [%rd1];
+; CHECK: ld.global.L1::evict_last.b32 %r1, [%rd1];
%v = load i32, ptr addrspace(1) %p, !mem.cache_hint !6
ret i32 %v
}
>From e6ffdd00b6bd619fd0faf64bcae1c312c0b0d705 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 30 Jun 2026 22:09:38 +0000
Subject: [PATCH 18/33] cleanup
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +--
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 5 +--
llvm/lib/CodeGen/MachineFunction.cpp | 21 ++++-----
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 2 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 44 ++++++-------------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 15 +++----
.../SelectionDAG/SelectionDAGBuilder.cpp | 23 ++++------
.../Target/Hexagon/HexagonISelLowering.cpp | 3 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 15 +++----
llvm/tools/llvm-reduce/ReducerWorkItem.cpp | 3 +-
10 files changed, 48 insertions(+), 88 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 00df15a08f78d..3a6e894c45f5c 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1419,9 +1419,8 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI),
- MachineMemOperand::Metadata(
- /*AAInfo=*/AAInfo,
- /*Ranges=*/LI.getMetadata(LLVMContext::MD_range)),
+ MachineMemOperand::Metadata(AAInfo,
+ LI.getMetadata(LLVMContext::MD_range)),
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
return true;
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 0222301f550f7..c6afbbd5b1942 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3862,9 +3862,8 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
return true;
Dest = MF.getMachineMemOperand(
Ptr, Flags, MemoryType, Align(BaseAlignment),
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Range,
- /*MemCacheHint=*/MemCacheHint),
- SSID, Order, FailureOrder);
+ MachineMemOperand::Metadata(AAInfo, Range, MemCacheHint), SSID, Order,
+ FailureOrder);
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 1ae81665f45f1..c807ed5ddb7d8 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -598,8 +598,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/AAMDNodes(), /*Ranges=*/nullptr,
- /*MemCacheHint=*/MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -608,8 +607,7 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/AAMDNodes(), /*Ranges=*/nullptr,
- /*MemCacheHint=*/MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -629,9 +627,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// are anymore.
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/MMO->getAAInfo(),
- /*Ranges=*/nullptr,
- /*MemCacheHint=*/MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(MMO->getAAInfo(), /*Ranges=*/nullptr,
+ MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -645,9 +642,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo,
- /*Ranges=*/MMO->getRanges(),
- /*MemCacheHint=*/MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(AAInfo, MMO->getRanges(),
+ MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -657,9 +653,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand::Flags Flags) {
return new (Allocator) MachineMemOperand(
MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/MMO->getAAInfo(),
- /*Ranges=*/MMO->getRanges(),
- /*MemCacheHint=*/MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges(),
+ MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 3ea38345f0f37..a2696fcde40af 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -2376,7 +2376,7 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
return FuncInfo.MF->getMachineMemOperand(
MachinePointerInfo(Ptr), Flags, Size, *Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ MachineMemOperand::Metadata(AAInfo, Ranges));
}
CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 765fe644a9a47..537034ed28389 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2515,8 +2515,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/LD->getAAInfo(),
- /*Ranges=*/LD->getRanges()));
+ MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
Lo =
DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
@@ -2541,8 +2540,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/LD->getAAInfo(),
- /*Ranges=*/LD->getRanges()));
+ MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2587,8 +2585,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/LD->getAAInfo(),
- /*Ranges=*/LD->getRanges()));
+ MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
Lo = DAG.getLoadFFVP(LoVT, dl, Ch, Ptr, MaskLo, EVLLo, MMO);
@@ -2663,8 +2660,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/SLD->getAAInfo(),
- /*Ranges=*/SLD->getRanges()));
+ MachineMemOperand::Metadata(SLD->getAAInfo(), SLD->getRanges()));
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2725,8 +2721,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/MLD->getAAInfo(),
- /*Ranges=*/MLD->getRanges()));
+ MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
MMO, MLD->getAddressingMode(), ExtType,
@@ -2750,8 +2745,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/MLD->getAAInfo(),
- /*Ranges=*/MLD->getRanges()));
+ MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
@@ -2814,9 +2808,7 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -4486,8 +4478,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4511,9 +4502,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4586,9 +4575,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
SDValue Hi = DAG.getStridedStoreVP(
N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
@@ -4640,8 +4627,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4667,9 +4653,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4734,9 +4718,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/N->getAAInfo(),
- /*Ranges=*/N->getRanges()));
+ Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f6a1c56473a29..49300cc2ef621 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -10601,8 +10601,7 @@ SDValue SelectionDAG::getLoad(
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges,
- /*MemCacheHint=*/MemCacheHint));
+ MachineMemOperand::Metadata(AAInfo, Ranges, MemCacheHint));
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
}
@@ -10735,8 +10734,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/nullptr,
- /*MemCacheHint=*/MemCacheHint));
+ MachineMemOperand::Metadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
return getStore(Chain, dl, Val, Ptr, MMO);
}
@@ -10815,8 +10813,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/nullptr,
- /*MemCacheHint=*/MemCacheHint));
+ MachineMemOperand::Metadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -10852,9 +10849,9 @@ SDValue SelectionDAG::getLoadVP(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment,
+ MachineMemOperand::Metadata(AAInfo, Ranges));
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
MMO, IsExpanding);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 504a02a6077fd..0839fcbe2d4e9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5140,7 +5140,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::upperBound(VT.getStoreSize()), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ MachineMemOperand::Metadata(AAInfo, Ranges));
const auto &TLI = DAG.getTargetLoweringInfo();
@@ -5185,8 +5185,7 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/I.getAAMetadata(),
- /*Ranges=*/Ranges));
+ MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -5347,9 +5346,8 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const MDNode *Ranges = getRangeMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/AAMDNodes(), /*Ranges=*/Ranges),
- SSID, Order);
+ I.getAlign(), MachineMemOperand::Metadata(AAMDNodes(), Ranges), SSID,
+ Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -6588,8 +6586,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
MachinePointerInfo(AS),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
MemoryLocation::UnknownSize, Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/I.getAAMetadata(),
- /*Ranges=*/Ranges));
+ MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -8774,7 +8771,7 @@ void SelectionDAGBuilder::visitVPLoad(
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::beforeOrAfterPointer(), *Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ MachineMemOperand::Metadata(AAInfo, Ranges));
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
if (AddToChain)
@@ -8802,7 +8799,7 @@ void SelectionDAGBuilder::visitVPLoadFF(
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), *Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ MachineMemOperand::Metadata(AAInfo, Ranges));
LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO);
SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
@@ -8829,8 +8826,7 @@ void SelectionDAGBuilder::visitVPGather(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
SDValue Base, Index, Scale;
bool UniformBase =
getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
@@ -8939,8 +8935,7 @@ void SelectionDAGBuilder::visitVPStridedLoad(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/AAInfo, /*Ranges=*/Ranges));
+ *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO,
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 2450ebeb4381b..a93af05d151f6 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3169,8 +3169,7 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
MachineFunction &MF = DAG.getMachineFunction();
WideMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
- MachineMemOperand::Metadata(/*AAInfo=*/MMO->getAAInfo(),
- /*Ranges=*/MMO->getRanges()),
+ MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index cbe6adb2c61b4..1c2b883032b3f 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8927,8 +8927,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
- /*Ranges=*/RLI.Ranges));
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8939,8 +8938,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
- /*Ranges=*/RLI.Ranges));
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8974,8 +8972,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
- /*Ranges=*/RLI.Ranges));
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ?
PPCISD::LFIWZX : PPCISD::LFIWAX,
@@ -9037,8 +9034,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
- /*Ranges=*/RLI.Ranges));
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other), Ops,
@@ -12080,8 +12076,7 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(/*AAInfo=*/RLI.AAInfo,
- /*Ranges=*/RLI.Ranges));
+ MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
SDValue Bits = DAG.getMemIntrinsicNode(
PPCISD::LD_SPLAT, dl, DAG.getVTList(MVT::v4i32, MVT::Other), Ops,
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index 9f9c1e54c1174..de4c5d232f03a 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -239,8 +239,7 @@ static void cloneMemOperands(MachineInstr &DstMI, MachineInstr &SrcMI,
MachineMemOperand *NewMMO = DstMF.getMachineMemOperand(
NewPtrInfo, OldMMO->getFlags(), OldMMO->getMemoryType(),
OldMMO->getBaseAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/OldMMO->getAAInfo(),
- /*Ranges=*/OldMMO->getRanges(),
+ MachineMemOperand::Metadata(OldMMO->getAAInfo(), OldMMO->getRanges(),
/*MemCacheHint=*/nullptr),
OldMMO->getSyncScopeID(), OldMMO->getSuccessOrdering(),
OldMMO->getFailureOrdering());
>From de8aa8bc187dbe428fe63ec4ea79018e1bb890ba Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 30 Jun 2026 22:10:45 +0000
Subject: [PATCH 19/33] format
---
llvm/lib/CodeGen/MachineFunction.cpp | 6 ++++--
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 11 +++++------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index c807ed5ddb7d8..ca8f20fdd816d 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -598,7 +598,8 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr,
+ MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -607,7 +608,8 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
+ MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr,
+ MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 82a38c94902f3..5108a010d9e14 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1193,8 +1193,7 @@ static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
if (Eviction == NVPTX::L2Eviction::Normal)
return true;
- return Subtarget.hasL2EvictionHint() &&
- isGlobalOrGeneric(Access.AddrSpace) &&
+ return Subtarget.hasL2EvictionHint() && isGlobalOrGeneric(Access.AddrSpace) &&
((Access.NumElts == 8 && Access.EltWidth == 32) ||
(Access.NumElts == 4 && Access.EltWidth == 64));
}
@@ -1246,8 +1245,7 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
}
if (KeyStr == "nvvm.l2_cache_hint") {
- if (isGlobalOrGeneric(Access.AddrSpace) &&
- Subtarget->hasL2CacheHint()) {
+ if (isGlobalOrGeneric(Access.AddrSpace) && Subtarget->hasL2CacheHint()) {
if (auto *ValCI = mdconst::dyn_extract<ConstantInt>(Value))
CachePolicy = ValCI->getZExtValue();
else
@@ -1658,8 +1656,9 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
// Extract eviction/prefetch hint and cache policy register.
const auto [EvictionAndPrefetchHint, PolicyReg] = getMemCacheHintOperands(
- ST, {CodeAddrSpace, /*IsLoad=*/false, /*NumElts=*/NumElts,
- /*EltWidth=*/ToTypeWidth},
+ ST,
+ {CodeAddrSpace, /*IsLoad=*/false, /*NumElts=*/NumElts,
+ /*EltWidth=*/ToTypeWidth},
DL);
const auto [Base, Offset] = selectADDR(Addr, CurDAG);
>From 2d924c8b72b88eabccaafddcfe724c2b7a14aa3e Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 30 Jun 2026 23:28:16 +0000
Subject: [PATCH 20/33] build break fix
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index 2c3f27c40ab37..d9900e98fe694 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -20,7 +20,6 @@
using namespace llvm;
#define GET_INSTRINFO_CTOR_DTOR
-#define GET_INSTRINFO_NAMED_OPS
#include "NVPTXGenInstrInfo.inc"
// Pin the vtable to this file.
>From b416451c5a1fd07a13d7064ec8b5ad19f9775f44 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 1 Jul 2026 16:26:03 +0000
Subject: [PATCH 21/33] fix test
---
llvm/test/CodeGen/NVPTX/address-folder.mir | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/address-folder.mir b/llvm/test/CodeGen/NVPTX/address-folder.mir
index ac50925579506..5b4ad84a9a8d1 100644
--- a/llvm/test/CodeGen/NVPTX/address-folder.mir
+++ b/llvm/test/CodeGen/NVPTX/address-folder.mir
@@ -15,17 +15,17 @@
# operand of loads and stores, and the dead `mov` is removed.
# CHECK-LABEL: name: fold_global
# CHECK-NOT: MOV_B64_sym
-# CHECK: %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 0
-# CHECK: %2:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 8
-# CHECK: ST_i64 %1, 0, 0, 1, 64, @g, 16
+# CHECK: %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 0, 0, $noreg
+# CHECK: %2:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 8, 0, $noreg
+# CHECK: ST_i64 %1, 0, 0, 1, 64, @g, 16, 0, $noreg
name: fold_global
tracksRegLiveness: true
body: |
bb.0:
%0:b64 = MOV_B64_sym @g
- %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 0 :: (load (s64), addrspace 1)
- %2:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 8 :: (load (s64), addrspace 1)
- ST_i64 %1, 0, 0, 1, 64, %0, 16 :: (store (s64), addrspace 1)
+ %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 0, 0, $noreg :: (load (s64), addrspace 1)
+ %2:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 8, 0, $noreg :: (load (s64), addrspace 1)
+ ST_i64 %1, 0, 0, 1, 64, %0, 16, 0, $noreg :: (store (s64), addrspace 1)
Return
...
---
@@ -33,13 +33,13 @@ body: |
# stay CSE-able rather than be duplicated into every access.
# CHECK-LABEL: name: skip_shared
# CHECK: %0:b64 = MOV_B64_sym @g_shared
-# CHECK: %1:b64 = LD_i64 0, 0, 3, 3, 64, -1, %0, 0
+# CHECK: %1:b64 = LD_i64 0, 0, 3, 3, 64, -1, %0, 0, 0, $noreg
name: skip_shared
tracksRegLiveness: true
body: |
bb.0:
%0:b64 = MOV_B64_sym @g_shared
- %1:b64 = LD_i64 0, 0, 3, 3, 64, -1, %0, 0 :: (load (s64), addrspace 3)
+ %1:b64 = LD_i64 0, 0, 3, 3, 64, -1, %0, 0, 0, $noreg :: (load (s64), addrspace 3)
Return
...
---
@@ -47,14 +47,14 @@ body: |
# the `mov` is kept for the remaining use.
# CHECK-LABEL: name: fold_partial
# CHECK: %0:b64 = MOV_B64_sym @g
-# CHECK: %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 0
+# CHECK: %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, @g, 0, 0, $noreg
# CHECK: %2:b64 = MULT64rr %0, %0
name: fold_partial
tracksRegLiveness: true
body: |
bb.0:
%0:b64 = MOV_B64_sym @g
- %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 0 :: (load (s64), addrspace 1)
+ %1:b64 = LD_i64 0, 0, 1, 3, 64, -1, %0, 0, 0, $noreg :: (load (s64), addrspace 1)
%2:b64 = MULT64rr %0, %0
Return
...
>From 4cf509f889dc401b2ed90b27ef99ed137f16260d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 08:51:54 +0000
Subject: [PATCH 22/33] fix feedback
---
llvm/include/llvm/CodeGen/CodeGenCommonISel.h | 2 +
llvm/include/llvm/CodeGen/MachineFunction.h | 12 +-
llvm/include/llvm/CodeGen/MachineMemOperand.h | 36 +++---
llvm/include/llvm/CodeGen/SelectionDAG.h | 33 ++---
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 28 +---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 2 +-
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 2 +-
llvm/lib/CodeGen/MachineFunction.cpp | 18 +--
llvm/lib/CodeGen/MachineOperand.cpp | 10 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 14 +-
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 2 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 28 ++--
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 80 ++++++------
.../SelectionDAG/SelectionDAGBuilder.cpp | 28 ++--
.../CodeGen/MIR/X86/mem-cache-hint-error.mir | 26 ++++
.../X86/mem-cache-hint-undefined-metadata.mir | 29 +++++
llvm/test/CodeGen/MIR/X86/mem-cache-hint.mir | 42 ++++++
.../CodeGen/NVPTX/cache-hint-cache-policy.ll | 120 +++++++-----------
18 files changed, 280 insertions(+), 232 deletions(-)
create mode 100644 llvm/test/CodeGen/MIR/X86/mem-cache-hint-error.mir
create mode 100644 llvm/test/CodeGen/MIR/X86/mem-cache-hint-undefined-metadata.mir
create mode 100644 llvm/test/CodeGen/MIR/X86/mem-cache-hint.mir
diff --git a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
index d0080bea5b1aa..cc4b92306d797 100644
--- a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
+++ b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
@@ -228,6 +228,8 @@ findSplitPointForStackProtector(MachineBasicBlock *BB,
/// simpler test.
LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp);
+/// Return the cache hint metadata node for memory operand \p OperandNo on \p I,
+/// or nullptr when the instruction has no hint for that operand.
LLVM_ABI const MDNode *getMemCacheHintMetadata(const Instruction &I,
unsigned OperandNo = 0);
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index fbb9235c04090..9fc271bbc56bc 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1115,37 +1115,37 @@ class LLVM_ABI MachineFunction {
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
Align BaseAlignment,
- MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
Align BaseAlignment,
- MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
Align BaseAlignment,
- MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
- BaseAlignment, MMOMetadata, SSID, Ordering,
+ BaseAlignment, Metadata, SSID, Ordering,
FailureOrdering);
}
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize Size,
Align BaseAlignment,
- MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
return getMachineMemOperand(PtrInfo, F, LocationSize::precise(Size),
- BaseAlignment, MMOMetadata, SSID, Ordering,
+ BaseAlignment, Metadata, SSID, Ordering,
FailureOrdering);
}
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index e85159346a7be..584f3925d0097 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -119,6 +119,18 @@ struct MachinePointerInfo {
};
+/// LLVM IR metadata carried by a MachineMemOperand.
+struct MMOMetadata {
+ AAMDNodes AAInfo;
+ const MDNode *Ranges = nullptr;
+ const MDNode *MemCacheHint = nullptr;
+
+ MMOMetadata() = default;
+ MMOMetadata(const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr,
+ const MDNode *MemCacheHint = nullptr)
+ : AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {}
+};
+
//===----------------------------------------------------------------------===//
/// A description of a memory reference used in the backend.
/// Instead of holding a StoreInst or LoadInst, this class holds the address
@@ -129,18 +141,6 @@ struct MachinePointerInfo {
///
class MachineMemOperand {
public:
- /// LLVM IR metadata carried by a MachineMemOperand.
- struct Metadata {
- AAMDNodes AAInfo;
- const MDNode *Ranges;
- const MDNode *MemCacheHint;
-
- Metadata() : Ranges(nullptr), MemCacheHint(nullptr) {}
- Metadata(const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr)
- : AAInfo(AAInfo), Ranges(Ranges), MemCacheHint(MemCacheHint) {}
- };
-
/// Flags values. These may be or'd together.
enum Flags : uint16_t {
// No flags set.
@@ -198,21 +198,21 @@ class MachineMemOperand {
public:
/// Construct a MachineMemOperand object with the specified PtrInfo, flags,
- /// size, and base alignment. For atomic operations the synchronization scope
- /// and atomic ordering requirements must also be specified. For cmpxchg
- /// atomic operations the atomic ordering requirements when store does not
- /// occur must also be specified.
+ /// size, base alignment, and metadata. For atomic operations the
+ /// synchronization scope and atomic ordering requirements must also be
+ /// specified. For cmpxchg atomic operations the atomic ordering requirements
+ /// when store does not occur must also be specified.
LLVM_ABI
MachineMemOperand(
MachinePointerInfo PtrInfo, Flags Flags, LocationSize TS, Align A,
- MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
LLVM_ABI
MachineMemOperand(
MachinePointerInfo PtrInfo, Flags Flags, LLT Type, Align A,
- MachineMemOperand::Metadata MMOMetadata = MachineMemOperand::Metadata(),
+ MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 2b866d0e5def1..0869b83f34a0f 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1526,8 +1526,7 @@ class SelectionDAG {
getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr);
+ MMOMetadata Metadata = MMOMetadata());
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
MachineMemOperand *MMO);
LLVM_ABI SDValue
@@ -1535,8 +1534,7 @@ class SelectionDAG {
SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *MemCacheHint = nullptr);
+ MMOMetadata Metadata = MMOMetadata());
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
SDValue Chain, SDValue Ptr, EVT MemVT,
MachineMemOperand *MMO);
@@ -1548,20 +1546,17 @@ class SelectionDAG {
const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr);
+ MMOMetadata Metadata = MMOMetadata());
inline SDValue
getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
MachinePointerInfo PtrInfo, EVT MemVT,
MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
- const MDNode *MemCacheHint = nullptr) {
+ MMOMetadata Metadata = MMOMetadata()) {
// Ensures that codegen never sees a None Alignment.
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
- Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo,
- Ranges, MemCacheHint);
+ Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, Metadata);
}
LLVM_ABI SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
@@ -1576,17 +1571,15 @@ class SelectionDAG {
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, Align Alignment,
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *MemCacheHint = nullptr);
+ MMOMetadata Metadata = MMOMetadata());
inline SDValue
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *MemCacheHint = nullptr) {
+ MMOMetadata Metadata = MMOMetadata()) {
return getStore(Chain, dl, Val, Ptr, PtrInfo,
Alignment.value_or(getEVTAlign(Val.getValueType())),
- MMOFlags, AAInfo, MemCacheHint);
+ MMOFlags, Metadata);
}
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, MachineMemOperand *MMO);
@@ -1594,18 +1587,16 @@ class SelectionDAG {
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *MemCacheHint = nullptr);
+ MMOMetadata Metadata = MMOMetadata());
inline SDValue
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
MachinePointerInfo PtrInfo, EVT SVT,
MaybeAlign Alignment = MaybeAlign(),
MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
- const AAMDNodes &AAInfo = AAMDNodes(),
- const MDNode *MemCacheHint = nullptr) {
+ MMOMetadata Metadata = MMOMetadata()) {
return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
- Alignment.value_or(getEVTAlign(SVT)), MMOFlags, AAInfo,
- MemCacheHint);
+ Alignment.value_or(getEVTAlign(SVT)), MMOFlags,
+ Metadata);
}
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 79716099c2e05..f54ce397c341d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1560,38 +1560,24 @@ class MemSDNode : public SDNode {
refineAlignment(ArrayRef(NewMMO));
}
- /// Refine range metadata for all MMOs. The NewMMOs array must parallel
- /// memoperands(). For each pair, if ranges differ, the stored range is
- /// cleared.
- void refineRanges(ArrayRef<MachineMemOperand *> NewMMOs) {
+ /// Refine LLVM IR metadata for all MMOs. The NewMMOs array must parallel
+ /// memoperands(). For each pair, if metadata differs, the stored metadata is
+ /// cleared conservatively.
+ void refineMMOMetadata(ArrayRef<MachineMemOperand *> NewMMOs) {
ArrayRef<MachineMemOperand *> MMOs = memoperands();
assert(NewMMOs.size() == MMOs.size() && "MMO count mismatch");
- // FIXME: Union the ranges instead?
for (auto [MMO, NewMMO] : zip(MMOs, NewMMOs)) {
+ // FIXME: Union the ranges instead?
if (MMO->getRanges() && MMO->getRanges() != NewMMO->getRanges())
MMO->clearRanges();
- }
- }
-
- void refineRanges(MachineMemOperand *NewMMO) {
- refineRanges(ArrayRef(NewMMO));
- }
-
- /// Refine cache hint metadata for all MMOs. The NewMMOs array must parallel
- /// memoperands(). For each pair, if cache hints differ, the stored hint is
- /// cleared.
- void refineMemCacheHints(ArrayRef<MachineMemOperand *> NewMMOs) {
- ArrayRef<MachineMemOperand *> MMOs = memoperands();
- assert(NewMMOs.size() == MMOs.size() && "MMO count mismatch");
- for (auto [MMO, NewMMO] : zip(MMOs, NewMMOs)) {
if (MMO->getMemCacheHint() &&
MMO->getMemCacheHint() != NewMMO->getMemCacheHint())
MMO->clearMemCacheHint();
}
}
- void refineMemCacheHints(MachineMemOperand *NewMMO) {
- refineMemCacheHints(ArrayRef(NewMMO));
+ void refineMMOMetadata(MachineMemOperand *NewMMO) {
+ refineMMOMetadata(ArrayRef(NewMMO));
}
const SDValue &getChain() const { return getOperand(0); }
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index c551049b261f0..25840d1b48f6d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1419,7 +1419,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI),
- MachineMemOperand::Metadata(AAInfo,
+ MMOMetadata(AAInfo,
LI.getMetadata(LLVMContext::MD_range)),
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index c6afbbd5b1942..9df40851154e8 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3862,7 +3862,7 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
return true;
Dest = MF.getMachineMemOperand(
Ptr, Flags, MemoryType, Align(BaseAlignment),
- MachineMemOperand::Metadata(AAInfo, Range, MemCacheHint), SSID, Order,
+ MMOMetadata(AAInfo, Range, MemCacheHint), SSID, Order,
FailureOrder);
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index ca8f20fdd816d..7fe21bd3bc4fd 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -566,7 +566,7 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment, MachineMemOperand::Metadata MMOMetadata,
+ Align BaseAlignment, MMOMetadata Metadata,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
assert((!Size.hasValue() ||
@@ -574,17 +574,17 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
"Unexpected an unknown size to be represented using "
"LocationSize::beforeOrAfter()");
return new (Allocator)
- MachineMemOperand(PtrInfo, F, Size, BaseAlignment, MMOMetadata, SSID,
+ MachineMemOperand(PtrInfo, F, Size, BaseAlignment, Metadata, SSID,
Ordering, FailureOrdering);
}
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
- Align BaseAlignment, MachineMemOperand::Metadata MMOMetadata,
+ Align BaseAlignment, MMOMetadata Metadata,
SyncScope::ID SSID, AtomicOrdering Ordering,
AtomicOrdering FailureOrdering) {
return new (Allocator)
- MachineMemOperand(PtrInfo, F, MemTy, BaseAlignment, MMOMetadata, SSID,
+ MachineMemOperand(PtrInfo, F, MemTy, BaseAlignment, Metadata, SSID,
Ordering, FailureOrdering);
}
@@ -598,7 +598,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr,
+ MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr,
MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
@@ -608,7 +608,7 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAMDNodes(), /*Ranges=*/nullptr,
+ MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr,
MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
@@ -629,7 +629,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// are anymore.
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
- MachineMemOperand::Metadata(MMO->getAAInfo(), /*Ranges=*/nullptr,
+ MMOMetadata(MMO->getAAInfo(), /*Ranges=*/nullptr,
MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
@@ -644,7 +644,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(),
- MachineMemOperand::Metadata(AAInfo, MMO->getRanges(),
+ MMOMetadata(AAInfo, MMO->getRanges(),
MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
@@ -655,7 +655,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand::Flags Flags) {
return new (Allocator) MachineMemOperand(
MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
- MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges(),
+ MMOMetadata(MMO->getAAInfo(), MMO->getRanges(),
MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 03060808c7d8b..f47ae85fb1e38 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1171,13 +1171,13 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
LLT Type, Align A,
- MachineMemOperand::Metadata MMOMetadata,
+ MMOMetadata Metadata,
SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: PtrInfo(PtrInfo), MemoryType(Type), FlagVals(F), BaseAlign(A),
- AAInfo(MMOMetadata.AAInfo), Ranges(MMOMetadata.Ranges),
- MemCacheHint(MMOMetadata.MemCacheHint) {
+ AAInfo(Metadata.AAInfo), Ranges(Metadata.Ranges),
+ MemCacheHint(Metadata.MemCacheHint) {
assert((PtrInfo.V.isNull() || isa<const PseudoSourceValue *>(PtrInfo.V) ||
isa<PointerType>(cast<const Value *>(PtrInfo.V)->getType())) &&
"invalid pointer value");
@@ -1193,7 +1193,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
LocationSize TS, Align BaseAlignment,
- MachineMemOperand::Metadata MMOMetadata,
+ MMOMetadata Metadata,
SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
@@ -1203,7 +1203,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
: TS.isScalable()
? LLT::scalable_vector(1, 8 * TS.getValue().getKnownMinValue())
: LLT::scalar(8 * TS.getValue().getKnownMinValue()),
- BaseAlignment, MMOMetadata, SSID, Ordering, FailureOrdering) {}
+ BaseAlignment, Metadata, SSID, Ordering, FailureOrdering) {}
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
// The Value and Offset may differ due to CSE. But the flags and size
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e04e9578a3232..a189bec7b17ea 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13656,7 +13656,8 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
SDValue NewLd = DAG.getLoad(
N->getValueType(0), SDLoc(N), MLD->getChain(), MLD->getBasePtr(),
MLD->getPointerInfo(), MLD->getBaseAlign(),
- MLD->getMemOperand()->getFlags(), MLD->getAAInfo(), MLD->getRanges());
+ MLD->getMemOperand()->getFlags(),
+ MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
return CombineTo(N, NewLd, NewLd.getValue(1));
}
@@ -16778,12 +16779,13 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr,
LN0->getPointerInfo().getWithOffset(PtrOff),
LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
- LN0->getAAInfo(), NewRanges);
+ MMOMetadata(LN0->getAAInfo(), NewRanges));
} else
- Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
- LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
- LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
- LN0->getAAInfo());
+ Load = DAG.getExtLoad(
+ ExtType, DL, VT, LN0->getChain(), NewPtr,
+ LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
+ LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
+ LN0->getAAInfo());
// Replace the old load's chain with the new load's chain.
WorklistRemover DeadNodes(*this);
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index a2696fcde40af..992e3c7e412a1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -2376,7 +2376,7 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
return FuncInfo.MF->getMachineMemOperand(
MachinePointerInfo(Ptr), Flags, Size, *Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MMOMetadata(AAInfo, Ranges));
}
CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index c9c0dd4450734..63597140d62b1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2515,7 +2515,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
+ MMOMetadata(LD->getAAInfo(), LD->getRanges()));
Lo =
DAG.getLoadVP(LD->getAddressingMode(), ExtType, LoVT, dl, Ch, Ptr, Offset,
@@ -2540,7 +2540,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
+ MMOMetadata(LD->getAAInfo(), LD->getRanges()));
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2585,7 +2585,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
LD->getPointerInfo(), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(LD->getAAInfo(), LD->getRanges()));
+ MMOMetadata(LD->getAAInfo(), LD->getRanges()));
Lo = DAG.getLoadFFVP(LoVT, dl, Ch, Ptr, MaskLo, EVLLo, MMO);
@@ -2660,7 +2660,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(SLD->getAAInfo(), SLD->getRanges()));
+ MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2721,7 +2721,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
Alignment,
- MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
+ MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
MMO, MLD->getAddressingMode(), ExtType,
@@ -2745,7 +2745,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MMOFlags, LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(MLD->getAAInfo(), MLD->getRanges()));
+ MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
@@ -2808,7 +2808,8 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MMOMetadata(N->getAAInfo(), N->getRanges()));
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -4478,7 +4479,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ MMOMetadata(N->getAAInfo(), N->getRanges()));
Lo = DAG.getStoreVP(Ch, DL, DataLo, Ptr, Offset, MaskLo, EVLLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4502,7 +4503,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) {
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
Hi = DAG.getStoreVP(Ch, DL, DataHi, Ptr, Offset, MaskHi, EVLHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4575,7 +4576,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(N->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
SDValue Hi = DAG.getStridedStoreVP(
N->getChain(), DL, HiData, Ptr, N->getOffset(), N->getStride(), HiMask,
@@ -4627,7 +4628,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MachineMemOperand::MOStore,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ MMOMetadata(N->getAAInfo(), N->getRanges()));
Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4653,7 +4654,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
N->getAddressingMode(), N->isTruncatingStore(),
@@ -4718,7 +4719,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment, MachineMemOperand::Metadata(N->getAAInfo(), N->getRanges()));
+ Alignment,
+ MMOMetadata(N->getAAInfo(), N->getRanges()));
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 49300cc2ef621..3ff254bf7616b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1339,8 +1339,7 @@ SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
// Range and cache hint metadata are not part of the DAG CSE key because
// we prefer to CSE even when metadata does not match. Merge potentially
// differing metadata conservatively.
- MemNode->refineRanges(NewMMOs);
- MemNode->refineMemCacheHints(NewMMOs);
+ MemNode->refineMMOMetadata(NewMMOs);
}
ReplaceAllUsesWith(N, Existing);
@@ -9349,7 +9348,9 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
Align SrcAlign, bool isVol, bool AlwaysInline,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
- BatchAAResults *BatchAA, const CallInst *CI) {
+ BatchAAResults *BatchAA,
+ const MDNode *DstMemCacheHint,
+ const MDNode *SrcMemCacheHint) {
// Turn a memcpy of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
if (Src.isUndef())
@@ -9417,10 +9418,6 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
MachineMemOperand::Flags MMOFlags =
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
- const MDNode *DstMemCacheHint =
- CI ? getMemCacheHintMetadata(*CI, /*OperandNo=*/0) : nullptr;
- const MDNode *SrcMemCacheHint =
- CI ? getMemCacheHintMetadata(*CI, /*OperandNo=*/1) : nullptr;
SmallVector<SDValue, 16> OutLoadChains;
SmallVector<SDValue, 16> OutStoreChains;
SmallVector<SDValue, 32> OutChains;
@@ -9461,8 +9458,8 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
Store = DAG.getStore(
Chain, dl, Value,
DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
- DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags, NewAAInfo,
- DstMemCacheHint);
+ DstPtrInfo.getWithOffset(DstOff), DstAlign, MMOFlags,
+ MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
OutChains.push_back(Store);
}
}
@@ -9488,15 +9485,15 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
ISD::EXTLOAD, dl, NVT, Chain,
DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)),
SrcPtrInfo.getWithOffset(SrcOff), VT,
- commonAlignment(SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo,
- SrcMemCacheHint);
+ commonAlignment(SrcAlign, SrcOff), SrcMMOFlags,
+ MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, SrcMemCacheHint));
OutLoadChains.push_back(Value.getValue(1));
Store = DAG.getTruncStore(
Chain, dl, Value,
DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)),
- DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags, NewAAInfo,
- DstMemCacheHint);
+ DstPtrInfo.getWithOffset(DstOff), VT, DstAlign, MMOFlags,
+ MMOMetadata(NewAAInfo, /*Ranges=*/nullptr, DstMemCacheHint));
OutStoreChains.push_back(Store);
}
SrcOff += VTSize;
@@ -9982,6 +9979,11 @@ SDValue SelectionDAG::getMemcpy(
const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
+ const MDNode *DstMemCacheHint =
+ CI ? getMemCacheHintMetadata(*CI, /*OperandNo=*/0) : nullptr;
+ const MDNode *SrcMemCacheHint =
+ CI ? getMemCacheHintMetadata(*CI, /*OperandNo=*/1) : nullptr;
+
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
if (ConstantSize) {
// Memcpy with size zero? Just return the original chain.
@@ -9990,7 +9992,8 @@ SDValue SelectionDAG::getMemcpy(
SDValue Result = getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
- SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, CI);
+ SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, DstMemCacheHint,
+ SrcMemCacheHint);
if (Result.getNode())
return Result;
}
@@ -10011,7 +10014,8 @@ SDValue SelectionDAG::getMemcpy(
assert(ConstantSize && "AlwaysInline requires a constant size!");
return getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
- SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, CI);
+ SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, DstMemCacheHint,
+ SrcMemCacheHint);
}
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
@@ -10334,7 +10338,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
void* IP = nullptr;
if (auto *E = cast_or_null<AtomicSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
E->refineAlignment(MMO);
- E->refineRanges(MMO);
+ E->refineMMOMetadata(MMO);
return SDValue(E, 0);
}
@@ -10586,7 +10590,7 @@ SDValue SelectionDAG::getLoad(
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo, const MDNode *Ranges, const MDNode *MemCacheHint) {
+ MMOMetadata Metadata) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
@@ -10600,8 +10604,7 @@ SDValue SelectionDAG::getLoad(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges, MemCacheHint));
+ PtrInfo, MMOFlags, Size, Alignment, Metadata);
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
}
@@ -10648,8 +10651,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
void *IP = nullptr;
if (auto *E = cast_or_null<LoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
E->refineAlignment(MMO);
- E->refineRanges(MMO);
- E->refineMemCacheHints(MMO);
+ E->refineMMOMetadata(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
@@ -10666,13 +10668,10 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
SDValue Ptr, MachinePointerInfo PtrInfo,
MaybeAlign Alignment,
- MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo, const MDNode *Ranges,
- const MDNode *MemCacheHint) {
+ MachineMemOperand::Flags MMOFlags, MMOMetadata Metadata) {
SDValue Undef = getUNDEF(Ptr.getValueType());
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
- PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
- MemCacheHint);
+ PtrInfo, VT, Alignment, MMOFlags, Metadata);
}
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
@@ -10687,11 +10686,10 @@ SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
MachinePointerInfo PtrInfo, EVT MemVT,
MaybeAlign Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo,
- const MDNode *MemCacheHint) {
+ MMOMetadata Metadata) {
SDValue Undef = getUNDEF(Ptr.getValueType());
return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
- MemVT, Alignment, MMOFlags, AAInfo, nullptr, MemCacheHint);
+ MemVT, Alignment, MMOFlags, Metadata);
}
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
@@ -10713,19 +10711,21 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
LD->getChain(), Base, Offset, LD->getPointerInfo(),
- LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
+ LD->getMemoryVT(), LD->getAlign(), MMOFlags,
+ MMOMetadata(LD->getAAInfo(), LD->getRanges(),
+ LD->getMemCacheHint()));
}
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, MachinePointerInfo PtrInfo,
Align Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo,
- const MDNode *MemCacheHint) {
+ MMOMetadata Metadata) {
assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
MMOFlags |= MachineMemOperand::MOStore;
assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
+ assert(!Metadata.Ranges && "range metadata is invalid for stores");
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
@@ -10733,8 +10733,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
TypeSize Size = Val.getValueType().getStoreSize();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
+ PtrInfo, MMOFlags, Size, Alignment, Metadata);
return getStore(Chain, dl, Val, Ptr, MMO);
}
@@ -10781,7 +10780,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
cast<StoreSDNode>(E)->refineAlignment(MMO);
- cast<StoreSDNode>(E)->refineMemCacheHints(MMO);
+ cast<StoreSDNode>(E)->refineMMOMetadata(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
@@ -10799,21 +10798,20 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
SDValue Ptr, MachinePointerInfo PtrInfo,
EVT SVT, Align Alignment,
MachineMemOperand::Flags MMOFlags,
- const AAMDNodes &AAInfo,
- const MDNode *MemCacheHint) {
+ MMOMetadata Metadata) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
MMOFlags |= MachineMemOperand::MOStore;
assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
+ assert(!Metadata.Ranges && "range metadata is invalid for stores");
if (PtrInfo.V.isNull())
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment,
- MachineMemOperand::Metadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
+ PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, Metadata);
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
}
@@ -10851,7 +10849,7 @@ SDValue SelectionDAG::getLoadVP(
MachineFunction &MF = getMachineFunction();
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MMOMetadata(AAInfo, Ranges));
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
MMO, IsExpanding);
}
@@ -10883,7 +10881,7 @@ SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
void *IP = nullptr;
if (auto *E = cast_or_null<VPLoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
E->refineAlignment(MMO);
- E->refineRanges(MMO);
+ E->refineMMOMetadata(MMO);
return SDValue(E, 0);
}
auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 26be75b4b38da..48b728ff1ee3c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4790,7 +4790,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
- MMOFlags, AAInfo, Ranges, MemCacheHint);
+ MMOFlags,
+ MMOMetadata(AAInfo, Ranges, MemCacheHint));
Chains[ChainI] = L.getValue(1);
if (MemVTs[i] != ValueVTs[i])
@@ -4947,7 +4948,8 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
if (MemVTs[i] != ValueVTs[i])
Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
SDValue St = DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
- AAInfo, MemCacheHint);
+ MMOMetadata(AAInfo, /*Ranges=*/nullptr,
+ MemCacheHint));
Chains[ChainI] = St;
}
@@ -5140,7 +5142,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::upperBound(VT.getStoreSize()), Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MMOMetadata(AAInfo, Ranges));
const auto &TLI = DAG.getTargetLoweringInfo();
@@ -5185,7 +5187,7 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), Alignment,
- MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
+ MMOMetadata(I.getAAMetadata(), Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -5227,7 +5229,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MachineMemOperand::Metadata(), SSID, SuccessOrdering,
+ I.getAlign(), MMOMetadata(), SSID, SuccessOrdering,
FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
@@ -5299,7 +5301,7 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MachineMemOperand::Metadata(), SSID, Ordering);
+ I.getAlign(), MMOMetadata(), SSID, Ordering);
SDValue L =
DAG.getAtomic(NT, dl, MemVT, InChain,
@@ -5346,7 +5348,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const MDNode *Ranges = getRangeMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MachineMemOperand::Metadata(AAMDNodes(), Ranges), SSID,
+ I.getAlign(), MMOMetadata(AAMDNodes(), Ranges), SSID,
Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
@@ -5384,7 +5386,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MachineMemOperand::Metadata(), SSID, Ordering);
+ I.getAlign(), MMOMetadata(), SSID, Ordering);
SDValue Val = getValue(I.getValueOperand());
if (Val.getValueType() != MemVT)
@@ -6611,7 +6613,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
MachinePointerInfo(AS),
MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
MemoryLocation::UnknownSize, Alignment,
- MachineMemOperand::Metadata(I.getAAMetadata(), Ranges));
+ MMOMetadata(I.getAAMetadata(), Ranges));
if (!UniformBase) {
Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
@@ -8796,7 +8798,7 @@ void SelectionDAGBuilder::visitVPLoad(
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MMOFlags,
LocationSize::beforeOrAfterPointer(), *Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MMOMetadata(AAInfo, Ranges));
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO, false /*IsExpanding */);
if (AddToChain)
@@ -8824,7 +8826,7 @@ void SelectionDAGBuilder::visitVPLoadFF(
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
LocationSize::beforeOrAfterPointer(), *Alignment,
- MachineMemOperand::Metadata(AAInfo, Ranges));
+ MMOMetadata(AAInfo, Ranges));
LD = DAG.getLoadFFVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
MMO);
SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, EVLVT, LD.getValue(1));
@@ -8851,7 +8853,7 @@ void SelectionDAGBuilder::visitVPGather(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
+ *Alignment, MMOMetadata(AAInfo, Ranges));
SDValue Base, Index, Scale;
bool UniformBase =
getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
@@ -8960,7 +8962,7 @@ void SelectionDAGBuilder::visitVPStridedLoad(
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
- *Alignment, MachineMemOperand::Metadata(AAInfo, Ranges));
+ *Alignment, MMOMetadata(AAInfo, Ranges));
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
OpValues[2], OpValues[3], MMO,
diff --git a/llvm/test/CodeGen/MIR/X86/mem-cache-hint-error.mir b/llvm/test/CodeGen/MIR/X86/mem-cache-hint-error.mir
new file mode 100644
index 0000000000000..1b78db70512e8
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/X86/mem-cache-hint-error.mir
@@ -0,0 +1,26 @@
+# RUN: not llc -mtriple=x86_64 -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+
+--- |
+ define i32 @test(ptr %p) {
+ entry:
+ %v = load i32, ptr %p
+ ret i32 %v
+ }
+
+...
+---
+name: test
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+body: |
+ bb.0.entry:
+ liveins: $rdi
+
+ %0:gr64 = COPY $rdi
+ ; CHECK: [[@LINE+1]]:92: expected metadata id after '!'
+ %1:gr32 = MOV32rm %0, 1, $noreg, 0, $noreg :: (load (s32) from %ir.p, !mem.cache_hint !)
+ $eax = COPY %1
+ RET64 $eax
+...
diff --git a/llvm/test/CodeGen/MIR/X86/mem-cache-hint-undefined-metadata.mir b/llvm/test/CodeGen/MIR/X86/mem-cache-hint-undefined-metadata.mir
new file mode 100644
index 0000000000000..5e6c464b90b7c
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/X86/mem-cache-hint-undefined-metadata.mir
@@ -0,0 +1,29 @@
+# RUN: not llc -mtriple=x86_64 -run-pass none -filetype=null %s 2>&1 | FileCheck %s
+
+--- |
+ define i32 @undefined_metadata(ptr %p) {
+ entry:
+ %v = load i32, ptr %p, !mem.cache_hint !0
+ ret i32 %v
+ }
+
+ !0 = !{i32 0, !1}
+ !1 = !{!"nvvm.l2_cache_hint", i64 12345}
+
+...
+---
+name: undefined_metadata
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+body: |
+ bb.0.entry:
+ liveins: $rdi
+
+ ; CHECK: use of undefined metadata '!3'
+ %0:gr64 = COPY $rdi
+ %1:gr32 = MOV32rm %0, 1, $noreg, 0, $noreg :: (load (s32) from %ir.p, !mem.cache_hint !3)
+ $eax = COPY %1
+ RET64 $eax
+...
diff --git a/llvm/test/CodeGen/MIR/X86/mem-cache-hint.mir b/llvm/test/CodeGen/MIR/X86/mem-cache-hint.mir
new file mode 100644
index 0000000000000..196307e14ef1d
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/X86/mem-cache-hint.mir
@@ -0,0 +1,42 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=x86_64 -run-pass=none -o - %s | FileCheck %s
+
+--- |
+ define i32 @test(ptr %p, i32 %v) {
+ entry:
+ %ld = load i32, ptr %p, !mem.cache_hint !0
+ store i32 %v, ptr %p, !mem.cache_hint !2
+ ret i32 %ld
+ }
+
+ !0 = !{i32 0, !1}
+ !1 = !{!"nvvm.l2_cache_hint", i64 12345}
+ !2 = !{i32 1, !1}
+
+...
+---
+name: test
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+ - { id: 1, class: gr32 }
+body: |
+ bb.0.entry:
+ liveins: $rdi, $esi
+
+ ; CHECK-LABEL: name: test
+ ; CHECK: liveins: $rdi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY]], 1, $noreg, 0, $noreg :: (load (s32) from %ir.p, !mem.cache_hint !0)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: MOV32mr [[COPY]], 1, $noreg, 0, $noreg, [[COPY1]] :: (store (s32) into %ir.p, !mem.cache_hint !2)
+ ; CHECK-NEXT: $eax = COPY [[MOV32rm]]
+ ; CHECK-NEXT: RET 0, $eax
+ %0:gr64 = COPY $rdi
+ %1:gr32 = MOV32rm %0, 1, $noreg, 0, $noreg :: (load (s32) from %ir.p, !mem.cache_hint !0)
+ %2:gr32 = COPY $esi
+ MOV32mr %0, 1, $noreg, 0, $noreg, %2 :: (store (s32) into %ir.p, !mem.cache_hint !2)
+ $eax = COPY %1
+ RET 0, $eax
+...
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll b/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
index 7dff40ebfba64..5f3761d91b3a6 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-cache-policy.ll
@@ -20,7 +20,7 @@ define i64 @test_load_cache_hint_i64(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_i64(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.b64 %rd3, [%rd1], %rd2;
- %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !1
+ %v = load i64, ptr addrspace(1) %p, !mem.cache_hint !0
ret i64 %v
}
@@ -28,7 +28,7 @@ define float @test_load_cache_hint_f32(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_f32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
- %v = load float, ptr addrspace(1) %p, !mem.cache_hint !2
+ %v = load float, ptr addrspace(1) %p, !mem.cache_hint !0
ret float %v
}
@@ -36,7 +36,7 @@ define void @test_store_cache_hint_i32(ptr addrspace(1) %p, i32 %v) {
; CHECK-LABEL: test_store_cache_hint_i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !3
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -44,7 +44,7 @@ define void @test_store_cache_hint_i64(ptr addrspace(1) %p, i64 %v) {
; CHECK-LABEL: test_store_cache_hint_i64(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.b64 [%rd1], %rd3, %rd2;
- store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !4
+ store i64 %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -52,7 +52,7 @@ define void @test_store_cache_hint_f32(ptr addrspace(1) %p, float %v) {
; CHECK-LABEL: test_store_cache_hint_f32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.b32 [%rd1], %r1, %rd2;
- store float %v, ptr addrspace(1) %p, !mem.cache_hint !5
+ store float %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -64,7 +64,7 @@ define <2 x i32> @test_load_cache_hint_v2i32(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_v2i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.v2.b32 {%r1, %r2}, [%rd1], %rd2;
- %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !6
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !0
ret <2 x i32> %v
}
@@ -72,7 +72,7 @@ define <4 x i32> @test_load_cache_hint_v4i32(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_v4i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1], %rd2;
- %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !7
+ %v = load <4 x i32>, ptr addrspace(1) %p, !mem.cache_hint !0
ret <4 x i32> %v
}
@@ -80,7 +80,7 @@ define <2 x i64> @test_load_cache_hint_v2i64(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_v2i64(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.v2.b64 {%rd3, %rd4}, [%rd1], %rd2;
- %v = load <2 x i64>, ptr addrspace(1) %p, !mem.cache_hint !8
+ %v = load <2 x i64>, ptr addrspace(1) %p, !mem.cache_hint !0
ret <2 x i64> %v
}
@@ -88,7 +88,7 @@ define <2 x float> @test_load_cache_hint_v2f32(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_v2f32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.v2.b32 {%r1, %r2}, [%rd1], %rd2;
- %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !9
+ %v = load <2 x float>, ptr addrspace(1) %p, !mem.cache_hint !0
ret <2 x float> %v
}
@@ -96,7 +96,7 @@ define <2 x double> @test_load_cache_hint_v2f64(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_v2f64(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.v2.b64 {%rd3, %rd4}, [%rd1], %rd2;
- %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !10
+ %v = load <2 x double>, ptr addrspace(1) %p, !mem.cache_hint !0
ret <2 x double> %v
}
@@ -104,7 +104,7 @@ define void @test_store_cache_hint_v2i32(ptr addrspace(1) %p, <2 x i32> %v) {
; CHECK-LABEL: test_store_cache_hint_v2i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.v2.b32 [%rd1], {%r1, %r2}, %rd2;
- store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !11
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -112,7 +112,7 @@ define void @test_store_cache_hint_v4i32(ptr addrspace(1) %p, <4 x i32> %v) {
; CHECK-LABEL: test_store_cache_hint_v4i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.v4.b32 [%rd1], {%r1, %r2, %r3, %r4}, %rd2;
- store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !12
+ store <4 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -120,7 +120,7 @@ define void @test_store_cache_hint_v2i64(ptr addrspace(1) %p, <2 x i64> %v) {
; CHECK-LABEL: test_store_cache_hint_v2i64(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.v2.b64 [%rd1], {%rd3, %rd4}, %rd2;
- store <2 x i64> %v, ptr addrspace(1) %p, !mem.cache_hint !13
+ store <2 x i64> %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -128,7 +128,7 @@ define void @test_store_cache_hint_v2f32(ptr addrspace(1) %p, <2 x float> %v) {
; CHECK-LABEL: test_store_cache_hint_v2f32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.v2.b32 [%rd1], {%r1, %r2}, %rd2;
- store <2 x float> %v, ptr addrspace(1) %p, !mem.cache_hint !14
+ store <2 x float> %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -136,7 +136,7 @@ define void @test_store_cache_hint_v2f64(ptr addrspace(1) %p, <2 x double> %v) {
; CHECK-LABEL: test_store_cache_hint_v2f64(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L2::cache_hint.v2.b64 [%rd1], {%rd3, %rd4}, %rd2;
- store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !15
+ store <2 x double> %v, ptr addrspace(1) %p, !mem.cache_hint !1
ret void
}
@@ -148,7 +148,7 @@ define i32 @test_generic_load_cache_hint_i32(ptr %p) {
; CHECK-LABEL: test_generic_load_cache_hint_i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.L2::cache_hint.b32 %r1, [%rd1], %rd2;
- %v = load i32, ptr %p, !mem.cache_hint !48
+ %v = load i32, ptr %p, !mem.cache_hint !0
ret i32 %v
}
@@ -156,7 +156,7 @@ define void @test_generic_store_cache_hint_i32(ptr %p, i32 %v) {
; CHECK-LABEL: test_generic_store_cache_hint_i32(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.L2::cache_hint.b32 [%rd1], %r1, %rd2;
- store i32 %v, ptr %p, !mem.cache_hint !49
+ store i32 %v, ptr %p, !mem.cache_hint !1
ret void
}
@@ -169,7 +169,7 @@ define i32 @test_load_cache_hint_with_l1(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_with_l1(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L1::evict_first.L2::cache_hint.b32 %r1, [%rd1], %rd2;
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !16
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !3
ret i32 %v
}
@@ -178,7 +178,7 @@ define i32 @test_load_cache_hint_with_l2_eviction(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_with_l2_eviction(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.b32 %r1, [%rd1], %rd2;
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !17
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !5
ret i32 %v
}
@@ -187,7 +187,7 @@ define i32 @test_load_cache_hint_with_prefetch(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_with_prefetch(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L2::cache_hint.L2::128B.b32 %r1, [%rd1], %rd2;
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !18
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !7
ret i32 %v
}
@@ -196,7 +196,7 @@ define i32 @test_load_cache_hint_with_all(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_with_all(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L1::evict_last.L2::cache_hint.L2::256B.b32 %r1, [%rd1], %rd2;
- %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !19
+ %v = load i32, ptr addrspace(1) %p, !mem.cache_hint !9
ret i32 %v
}
@@ -205,7 +205,7 @@ define void @test_store_cache_hint_with_l1(ptr addrspace(1) %p, i32 %v) {
; CHECK-LABEL: test_store_cache_hint_with_l1(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L1::evict_unchanged.L2::cache_hint.b32 [%rd1], %r1, %rd2;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !20
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !11
ret void
}
@@ -214,7 +214,7 @@ define void @test_store_cache_hint_with_all(ptr addrspace(1) %p, i32 %v) {
; CHECK-LABEL: test_store_cache_hint_with_all(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L1::no_allocate.L2::cache_hint.b32 [%rd1], %r1, %rd2;
- store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !21
+ store i32 %v, ptr addrspace(1) %p, !mem.cache_hint !13
ret void
}
@@ -223,7 +223,7 @@ define <2 x i32> @test_load_cache_hint_v2i32_with_l1(ptr addrspace(1) %p) {
; CHECK-LABEL: test_load_cache_hint_v2i32_with_l1(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: ld.global.L1::evict_first.L2::cache_hint.v2.b32 {%r1, %r2}, [%rd1], %rd2;
- %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !22
+ %v = load <2 x i32>, ptr addrspace(1) %p, !mem.cache_hint !15
ret <2 x i32> %v
}
@@ -232,7 +232,7 @@ define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32>
; CHECK-LABEL: test_store_cache_hint_v2i32_with_all(
; CHECK: mov.b64 %rd2, 12345;
; CHECK: st.global.L1::evict_last.L2::cache_hint.v2.b32 [%rd1], {%r1, %r2}, %rd2;
- store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !23
+ store <2 x i32> %v, ptr addrspace(1) %p, !mem.cache_hint !17
ret void
}
@@ -240,54 +240,22 @@ define void @test_store_cache_hint_v2i32_with_all(ptr addrspace(1) %p, <2 x i32>
; Metadata definitions
;-----------------------------------------------------------------------------
-!0 = !{i32 0, !24}
-!24 = !{!"nvvm.l2_cache_hint", i64 12345}
-!1 = !{i32 0, !25}
-!25 = !{!"nvvm.l2_cache_hint", i64 12345}
-!2 = !{i32 0, !26}
-!26 = !{!"nvvm.l2_cache_hint", i64 12345}
-!3 = !{i32 1, !27}
-!27 = !{!"nvvm.l2_cache_hint", i64 12345}
-!4 = !{i32 1, !28}
-!28 = !{!"nvvm.l2_cache_hint", i64 12345}
-!5 = !{i32 1, !29}
-!29 = !{!"nvvm.l2_cache_hint", i64 12345}
-!6 = !{i32 0, !30}
-!30 = !{!"nvvm.l2_cache_hint", i64 12345}
-!7 = !{i32 0, !31}
-!31 = !{!"nvvm.l2_cache_hint", i64 12345}
-!8 = !{i32 0, !32}
-!32 = !{!"nvvm.l2_cache_hint", i64 12345}
-!9 = !{i32 0, !33}
-!33 = !{!"nvvm.l2_cache_hint", i64 12345}
-!10 = !{i32 0, !34}
-!34 = !{!"nvvm.l2_cache_hint", i64 12345}
-!11 = !{i32 1, !35}
-!35 = !{!"nvvm.l2_cache_hint", i64 12345}
-!12 = !{i32 1, !36}
-!36 = !{!"nvvm.l2_cache_hint", i64 12345}
-!13 = !{i32 1, !37}
-!37 = !{!"nvvm.l2_cache_hint", i64 12345}
-!14 = !{i32 1, !38}
-!38 = !{!"nvvm.l2_cache_hint", i64 12345}
-!15 = !{i32 1, !39}
-!39 = !{!"nvvm.l2_cache_hint", i64 12345}
-!16 = !{i32 0, !40}
-!40 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
-!17 = !{i32 0, !41}
-!41 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_eviction", !"last"}
-!18 = !{i32 0, !42}
-!42 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_prefetch_size", !"128B"}
-!19 = !{i32 0, !43}
-!43 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
-!20 = !{i32 1, !44}
-!44 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"unchanged"}
-!21 = !{i32 1, !45}
-!45 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
-!22 = !{i32 0, !46}
-!46 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
-!23 = !{i32 1, !47}
-!47 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"64B"}
-!48 = !{i32 0, !50}
-!49 = !{i32 1, !50}
-!50 = !{!"nvvm.l2_cache_hint", i64 12345}
+!0 = !{i32 0, !2}
+!1 = !{i32 1, !2}
+!2 = !{!"nvvm.l2_cache_hint", i64 12345}
+!3 = !{i32 0, !4}
+!4 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
+!5 = !{i32 0, !6}
+!6 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_eviction", !"last"}
+!7 = !{i32 0, !8}
+!8 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l2_prefetch_size", !"128B"}
+!9 = !{i32 0, !10}
+!10 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"256B"}
+!11 = !{i32 1, !12}
+!12 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"unchanged"}
+!13 = !{i32 1, !14}
+!14 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"no_allocate", !"nvvm.l2_eviction", !"last"}
+!15 = !{i32 0, !16}
+!16 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"first"}
+!17 = !{i32 1, !18}
+!18 = !{!"nvvm.l2_cache_hint", i64 12345, !"nvvm.l1_eviction", !"last", !"nvvm.l2_eviction", !"first", !"nvvm.l2_prefetch_size", !"64B"}
>From 9e26f862284b108bd9c0123ba4642a00c754f4d2 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 08:53:08 +0000
Subject: [PATCH 23/33] format
---
llvm/include/llvm/CodeGen/MachineFunction.h | 12 ++---
llvm/include/llvm/CodeGen/MachineMemOperand.h | 23 ++++-----
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 3 +-
llvm/lib/CodeGen/MIRParser/MIParser.cpp | 7 ++-
llvm/lib/CodeGen/MachineFunction.cpp | 25 ++++------
llvm/lib/CodeGen/MachineOperand.cpp | 6 +--
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 19 ++++---
llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 6 +--
.../SelectionDAG/LegalizeVectorTypes.cpp | 15 ++----
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 49 ++++++++++---------
.../SelectionDAG/SelectionDAGBuilder.cpp | 18 +++----
11 files changed, 79 insertions(+), 104 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 9fc271bbc56bc..3be79fb128849 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1114,22 +1114,19 @@ class LLVM_ABI MachineFunction {
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
- Align BaseAlignment,
- MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment,
- MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
- Align BaseAlignment,
- MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
@@ -1139,8 +1136,7 @@ class LLVM_ABI MachineFunction {
}
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize Size,
- Align BaseAlignment,
- MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 584f3925d0097..242778ad7b523 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -118,7 +118,6 @@ struct MachinePointerInfo {
LLVM_ABI static MachinePointerInfo getUnknownStack(MachineFunction &MF);
};
-
/// LLVM IR metadata carried by a MachineMemOperand.
struct MMOMetadata {
AAMDNodes AAInfo;
@@ -203,19 +202,17 @@ class MachineMemOperand {
/// specified. For cmpxchg atomic operations the atomic ordering requirements
/// when store does not occur must also be specified.
LLVM_ABI
- MachineMemOperand(
- MachinePointerInfo PtrInfo, Flags Flags, LocationSize TS, Align A,
- MMOMetadata Metadata = MMOMetadata(),
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags Flags, LocationSize TS,
+ Align A, MMOMetadata Metadata = MMOMetadata(),
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
LLVM_ABI
- MachineMemOperand(
- MachinePointerInfo PtrInfo, Flags Flags, LLT Type, Align A,
- MMOMetadata Metadata = MMOMetadata(),
- SyncScope::ID SSID = SyncScope::System,
- AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
- AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
+ MachineMemOperand(MachinePointerInfo PtrInfo, Flags Flags, LLT Type, Align A,
+ MMOMetadata Metadata = MMOMetadata(),
+ SyncScope::ID SSID = SyncScope::System,
+ AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
+ AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 25840d1b48f6d..4844365f5850c 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1419,8 +1419,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI),
- MMOMetadata(AAInfo,
- LI.getMetadata(LLVMContext::MD_range)),
+ MMOMetadata(AAInfo, LI.getMetadata(LLVMContext::MD_range)),
LI.getSyncScopeID(), LI.getOrdering());
MIRBuilder.buildLoad(Regs[0], Base, *MMO);
return true;
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 9df40851154e8..8528edc6bfc3a 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -3860,10 +3860,9 @@ bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
}
if (expectAndConsume(MIToken::rparen))
return true;
- Dest = MF.getMachineMemOperand(
- Ptr, Flags, MemoryType, Align(BaseAlignment),
- MMOMetadata(AAInfo, Range, MemCacheHint), SSID, Order,
- FailureOrder);
+ Dest = MF.getMachineMemOperand(Ptr, Flags, MemoryType, Align(BaseAlignment),
+ MMOMetadata(AAInfo, Range, MemCacheHint), SSID,
+ Order, FailureOrder);
return false;
}
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 7fe21bd3bc4fd..b664cf0210493 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -566,9 +566,8 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment, MMOMetadata Metadata,
- SyncScope::ID SSID, AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering) {
+ Align BaseAlignment, MMOMetadata Metadata, SyncScope::ID SSID,
+ AtomicOrdering Ordering, AtomicOrdering FailureOrdering) {
assert((!Size.hasValue() ||
Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
"Unexpected an unknown size to be represented using "
@@ -580,9 +579,8 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
- Align BaseAlignment, MMOMetadata Metadata,
- SyncScope::ID SSID, AtomicOrdering Ordering,
- AtomicOrdering FailureOrdering) {
+ Align BaseAlignment, MMOMetadata Metadata, SyncScope::ID SSID,
+ AtomicOrdering Ordering, AtomicOrdering FailureOrdering) {
return new (Allocator)
MachineMemOperand(PtrInfo, F, MemTy, BaseAlignment, Metadata, SSID,
Ordering, FailureOrdering);
@@ -598,8 +596,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
"LocationSize::beforeOrAfter()");
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Size, MMO->getBaseAlign(),
- MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr,
- MMO->getMemCacheHint()),
+ MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -608,8 +605,7 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
return new (Allocator) MachineMemOperand(
PtrInfo, MMO->getFlags(), Ty, MMO->getBaseAlign(),
- MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr,
- MMO->getMemCacheHint()),
+ MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -629,8 +625,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
// are anymore.
return new (Allocator) MachineMemOperand(
PtrInfo.getWithOffset(Offset), MMO->getFlags(), Ty, Alignment,
- MMOMetadata(MMO->getAAInfo(), /*Ranges=*/nullptr,
- MMO->getMemCacheHint()),
+ MMOMetadata(MMO->getAAInfo(), /*Ranges=*/nullptr, MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -644,8 +639,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
return new (Allocator) MachineMemOperand(
MPI, MMO->getFlags(), MMO->getSize(), MMO->getBaseAlign(),
- MMOMetadata(AAInfo, MMO->getRanges(),
- MMO->getMemCacheHint()),
+ MMOMetadata(AAInfo, MMO->getRanges(), MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
@@ -655,8 +649,7 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
MachineMemOperand::Flags Flags) {
return new (Allocator) MachineMemOperand(
MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlign(),
- MMOMetadata(MMO->getAAInfo(), MMO->getRanges(),
- MMO->getMemCacheHint()),
+ MMOMetadata(MMO->getAAInfo(), MMO->getRanges(), MMO->getMemCacheHint()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index f47ae85fb1e38..5d52cf5829fc0 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1170,8 +1170,7 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
}
MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
- LLT Type, Align A,
- MMOMetadata Metadata,
+ LLT Type, Align A, MMOMetadata Metadata,
SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
@@ -1193,8 +1192,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
LocationSize TS, Align BaseAlignment,
- MMOMetadata Metadata,
- SyncScope::ID SSID,
+ MMOMetadata Metadata, SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a189bec7b17ea..0f3e3081754e8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13653,11 +13653,11 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
// FIXME: Can we do this for indexed, expanding, or extending loads?
if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() &&
!MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) {
- SDValue NewLd = DAG.getLoad(
- N->getValueType(0), SDLoc(N), MLD->getChain(), MLD->getBasePtr(),
- MLD->getPointerInfo(), MLD->getBaseAlign(),
- MLD->getMemOperand()->getFlags(),
- MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
+ SDValue NewLd =
+ DAG.getLoad(N->getValueType(0), SDLoc(N), MLD->getChain(),
+ MLD->getBasePtr(), MLD->getPointerInfo(),
+ MLD->getBaseAlign(), MLD->getMemOperand()->getFlags(),
+ MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
return CombineTo(N, NewLd, NewLd.getValue(1));
}
@@ -16781,11 +16781,10 @@ SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
MMOMetadata(LN0->getAAInfo(), NewRanges));
} else
- Load = DAG.getExtLoad(
- ExtType, DL, VT, LN0->getChain(), NewPtr,
- LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
- LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
- LN0->getAAInfo());
+ Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
+ LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
+ LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
+ LN0->getAAInfo());
// Replace the old load's chain with the new load's chain.
WorklistRemover DeadNodes(*this);
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 992e3c7e412a1..09a54c191cef4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -2374,9 +2374,9 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const {
if (IsInvariant)
Flags |= MachineMemOperand::MOInvariant;
- return FuncInfo.MF->getMachineMemOperand(
- MachinePointerInfo(Ptr), Flags, Size, *Alignment,
- MMOMetadata(AAInfo, Ranges));
+ return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
+ *Alignment,
+ MMOMetadata(AAInfo, Ranges));
}
CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 63597140d62b1..80dca88230279 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -2539,8 +2539,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo,
MMO = DAG.getMachineFunction().getMachineMemOperand(
MPI, MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MMOMetadata(LD->getAAInfo(), LD->getRanges()));
+ Alignment, MMOMetadata(LD->getAAInfo(), LD->getRanges()));
Hi = DAG.getLoadVP(LD->getAddressingMode(), ExtType, HiVT, dl, Ch, Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
@@ -2659,8 +2658,7 @@ void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(SLD->getPointerInfo().getAddrSpace()),
MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
+ Alignment, MMOMetadata(SLD->getAAInfo(), SLD->getRanges()));
Hi = DAG.getStridedLoadVP(SLD->getAddressingMode(), SLD->getExtensionType(),
HiVT, DL, SLD->getChain(), Ptr, SLD->getOffset(),
@@ -2720,8 +2718,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MLD->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
+ Alignment, MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
MMO, MLD->getAddressingMode(), ExtType,
@@ -2808,8 +2805,7 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MMOMetadata(N->getAAInfo(), N->getRanges()));
+ Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
SDValue PassThru = MGT->getPassThru();
@@ -4719,8 +4715,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
- Alignment,
- MMOMetadata(N->getAAInfo(), N->getRanges()));
+ Alignment, MMOMetadata(N->getAAInfo(), N->getRanges()));
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 3ff254bf7616b..e75e4fe566c57 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9348,8 +9348,7 @@ getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
Align SrcAlign, bool isVol, bool AlwaysInline,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
- BatchAAResults *BatchAA,
- const MDNode *DstMemCacheHint,
+ BatchAAResults *BatchAA, const MDNode *DstMemCacheHint,
const MDNode *SrcMemCacheHint) {
// Turn a memcpy of undef to nop.
// FIXME: We need to honor volatile even is Src is undef.
@@ -9992,8 +9991,8 @@ SDValue SelectionDAG::getMemcpy(
SDValue Result = getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
- SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, DstMemCacheHint,
- SrcMemCacheHint);
+ SrcAlign, isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
+ DstMemCacheHint, SrcMemCacheHint);
if (Result.getNode())
return Result;
}
@@ -10014,8 +10013,8 @@ SDValue SelectionDAG::getMemcpy(
assert(ConstantSize && "AlwaysInline requires a constant size!");
return getMemcpyLoadsAndStores(
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), DstAlign,
- SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA, DstMemCacheHint,
- SrcMemCacheHint);
+ SrcAlign, isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
+ DstMemCacheHint, SrcMemCacheHint);
}
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
@@ -10586,11 +10585,13 @@ static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
return Info;
}
-SDValue SelectionDAG::getLoad(
- ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
- SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
- EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags,
- MMOMetadata Metadata) {
+SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
+ EVT VT, const SDLoc &dl, SDValue Chain,
+ SDValue Ptr, SDValue Offset,
+ MachinePointerInfo PtrInfo, EVT MemVT,
+ Align Alignment,
+ MachineMemOperand::Flags MMOFlags,
+ MMOMetadata Metadata) {
assert(Chain.getValueType() == MVT::Other &&
"Invalid chain type");
@@ -10603,8 +10604,8 @@ SDValue SelectionDAG::getLoad(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment, Metadata);
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, Metadata);
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
}
@@ -10668,7 +10669,8 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
SDValue Ptr, MachinePointerInfo PtrInfo,
MaybeAlign Alignment,
- MachineMemOperand::Flags MMOFlags, MMOMetadata Metadata) {
+ MachineMemOperand::Flags MMOFlags,
+ MMOMetadata Metadata) {
SDValue Undef = getUNDEF(Ptr.getValueType());
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
PtrInfo, VT, Alignment, MMOFlags, Metadata);
@@ -10709,11 +10711,11 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
auto MMOFlags =
LD->getMemOperand()->getFlags() &
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
- return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
- LD->getChain(), Base, Offset, LD->getPointerInfo(),
- LD->getMemoryVT(), LD->getAlign(), MMOFlags,
- MMOMetadata(LD->getAAInfo(), LD->getRanges(),
- LD->getMemCacheHint()));
+ return getLoad(
+ AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, LD->getChain(),
+ Base, Offset, LD->getPointerInfo(), LD->getMemoryVT(), LD->getAlign(),
+ MMOFlags,
+ MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
}
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
@@ -10732,8 +10734,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
MachineFunction &MF = getMachineFunction();
TypeSize Size = Val.getValueType().getStoreSize();
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, MMOFlags, Size, Alignment, Metadata);
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, Metadata);
return getStore(Chain, dl, Val, Ptr, MMO);
}
@@ -10847,9 +10849,8 @@ SDValue SelectionDAG::getLoadVP(
TypeSize Size = MemVT.getStoreSize();
MachineFunction &MF = getMachineFunction();
- MachineMemOperand *MMO =
- MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment,
- MMOMetadata(AAInfo, Ranges));
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, MMOFlags, Size, Alignment, MMOMetadata(AAInfo, Ranges));
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
MMO, IsExpanding);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 48b728ff1ee3c..78d99119555b4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4789,9 +4789,9 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
: MachinePointerInfo();
SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
- SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
- MMOFlags,
- MMOMetadata(AAInfo, Ranges, MemCacheHint));
+ SDValue L =
+ DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment, MMOFlags,
+ MMOMetadata(AAInfo, Ranges, MemCacheHint));
Chains[ChainI] = L.getValue(1);
if (MemVTs[i] != ValueVTs[i])
@@ -4947,9 +4947,9 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
if (MemVTs[i] != ValueVTs[i])
Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
- SDValue St = DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
- MMOMetadata(AAInfo, /*Ranges=*/nullptr,
- MemCacheHint));
+ SDValue St =
+ DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags,
+ MMOMetadata(AAInfo, /*Ranges=*/nullptr, MemCacheHint));
Chains[ChainI] = St;
}
@@ -5229,8 +5229,7 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MMOMetadata(), SSID, SuccessOrdering,
- FailureOrdering);
+ I.getAlign(), MMOMetadata(), SSID, SuccessOrdering, FailureOrdering);
SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
dl, MemVT, VTs, InChain,
@@ -5348,8 +5347,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
const MDNode *Ranges = getRangeMetadata(I);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
- I.getAlign(), MMOMetadata(AAMDNodes(), Ranges), SSID,
- Order);
+ I.getAlign(), MMOMetadata(AAMDNodes(), Ranges), SSID, Order);
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
>From 8bdae8612b51881785f12d31a135debdd401b817 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 09:06:20 +0000
Subject: [PATCH 24/33] fix build
---
llvm/lib/Target/Hexagon/HexagonISelLowering.cpp | 2 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 16 +++++++++-------
llvm/tools/llvm-reduce/ReducerWorkItem.cpp | 4 ++--
.../CodeGen/GlobalISel/KnownBitsTest.cpp | 4 ++--
.../CodeGen/GlobalISel/KnownBitsVectorTest.cpp | 2 +-
.../Target/AArch64/AArch64SelectionDAGTest.cpp | 4 ++--
6 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index a93af05d151f6..a4938d5494080 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3169,7 +3169,7 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
MachineFunction &MF = DAG.getMachineFunction();
WideMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
- MachineMemOperand::Metadata(MMO->getAAInfo(), MMO->getRanges()),
+ MMOMetadata(MMO->getAAInfo(), MMO->getRanges()),
MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
MMO->getFailureOrdering());
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 1c2b883032b3f..9d969a560b3c9 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8591,7 +8591,8 @@ SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI,
- RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges);
+ RLI.Alignment, RLI.MMOFlags(),
+ MMOMetadata(RLI.AAInfo, RLI.Ranges));
}
// We're trying to insert a regular store, S, and then a load, L. If the
@@ -8920,14 +8921,15 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
// Drop range metadata, as this metadata becomes invalid for f64 bit
// reinterpretation of i64 values.
Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI,
- RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, nullptr);
+ RLI.Alignment, RLI.MMOFlags(),
+ MMOMetadata(RLI.AAInfo));
if (RLI.ResChain)
DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
} else if (Subtarget.hasLFIWAX() &&
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) {
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MMOMetadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8938,7 +8940,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) {
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MMOMetadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other),
@@ -8972,7 +8974,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MMOMetadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ?
PPCISD::LFIWZX : PPCISD::LFIWAX,
@@ -9034,7 +9036,7 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MMOMetadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = { RLI.Chain, RLI.Ptr };
Ld = DAG.getMemIntrinsicNode(IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
DAG.getVTList(MVT::f64, MVT::Other), Ops,
@@ -12076,7 +12078,7 @@ SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
MachineMemOperand *MMO = MF.getMachineMemOperand(
RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment,
- MachineMemOperand::Metadata(RLI.AAInfo, RLI.Ranges));
+ MMOMetadata(RLI.AAInfo, RLI.Ranges));
SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
SDValue Bits = DAG.getMemIntrinsicNode(
PPCISD::LD_SPLAT, dl, DAG.getVTList(MVT::v4i32, MVT::Other), Ops,
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index de4c5d232f03a..b7ee2acbc3fda 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -239,8 +239,8 @@ static void cloneMemOperands(MachineInstr &DstMI, MachineInstr &SrcMI,
MachineMemOperand *NewMMO = DstMF.getMachineMemOperand(
NewPtrInfo, OldMMO->getFlags(), OldMMO->getMemoryType(),
OldMMO->getBaseAlign(),
- MachineMemOperand::Metadata(OldMMO->getAAInfo(), OldMMO->getRanges(),
- /*MemCacheHint=*/nullptr),
+ MMOMetadata(OldMMO->getAAInfo(), OldMMO->getRanges(),
+ /*MemCacheHint=*/nullptr),
OldMMO->getSyncScopeID(), OldMMO->getSuccessOrdering(),
OldMMO->getFailureOrdering());
NewMMOs.push_back(NewMMO);
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index 7eb381c5713d2..448c5030a9063 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -1181,8 +1181,8 @@ static void AddRangeMetadata(LLVMContext &Context, MachineInstr *Load) {
Load->getParent()->getParent()->getMachineMemOperand(
OldMMO->getPointerInfo(), OldMMO->getFlags(), OldMMO->getMemoryType(),
OldMMO->getAlign(),
- MachineMemOperand::Metadata(/*AAInfo=*/OldMMO->getAAInfo(),
- /*Ranges=*/NewMDNode));
+ MMOMetadata(/*AAInfo=*/OldMMO->getAAInfo(),
+ /*Ranges=*/NewMDNode));
MachineIRBuilder MIB(*Load);
MIB.buildLoadInstr(Load->getOpcode(), Load->getOperand(0),
Load->getOperand(1), *NewMMO);
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
index 205ec0c446915..44c0406426587 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
@@ -164,7 +164,7 @@ TEST_F(AArch64GISelMITest, TestVectorMetadata) {
const MachineMemOperand *OldMMO = *Load->memoperands_begin();
MachineMemOperand NewMMO(OldMMO->getPointerInfo(), OldMMO->getFlags(),
OldMMO->getMemoryType(), OldMMO->getAlign(),
- MachineMemOperand::Metadata(
+ MMOMetadata(
/*AAInfo=*/OldMMO->getAAInfo(),
/*Ranges=*/NewMDNode));
MachineIRBuilder MIB(*Load);
diff --git a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
index 37a5854652426..3ac517f2ce6f4 100644
--- a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
@@ -1388,7 +1388,7 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_known01) {
MDNode *Range = MDHelper.createRange(APInt(8, 0), APInt(8, 2));
MachineMemOperand *MMO = DAG->getMachineFunction().getMachineMemOperand(
PtrInfo, MachineMemOperand::MOLoad, 8, Align(8),
- MachineMemOperand::Metadata(/*AAInfo=*/AA, /*Ranges=*/Range));
+ MMOMetadata(/*AAInfo=*/AA, /*Ranges=*/Range));
auto ALoad = DAG->getExtLoad(ISD::EXTLOAD, Loc, Int32VT, DAG->getEntryNode(),
Ptr, Int8VT, MMO);
@@ -1422,7 +1422,7 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_extload_knownnegative) {
MDNode *Range = MDHelper.createRange(APInt(8, 0xf0), APInt(8, 0xff));
MachineMemOperand *MMO = DAG->getMachineFunction().getMachineMemOperand(
PtrInfo, MachineMemOperand::MOLoad, 8, Align(8),
- MachineMemOperand::Metadata(/*AAInfo=*/AA, /*Ranges=*/Range));
+ MMOMetadata(/*AAInfo=*/AA, /*Ranges=*/Range));
auto ALoad = DAG->getExtLoad(ISD::EXTLOAD, Loc, Int32VT, DAG->getEntryNode(),
Ptr, Int8VT, MMO);
>From 6524c33fdd2034919a8b3bb5eefae5d2690319f4 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 09:06:41 +0000
Subject: [PATCH 25/33] format
---
llvm/lib/Target/Hexagon/HexagonISelLowering.cpp | 5 ++---
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 6 +++---
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index a4938d5494080..fba37e0baa8af 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3169,9 +3169,8 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
MachineFunction &MF = DAG.getMachineFunction();
WideMMO = MF.getMachineMemOperand(
MMO->getPointerInfo(), MMO->getFlags(), 2 * LoadLen, Align(LoadLen),
- MMOMetadata(MMO->getAAInfo(), MMO->getRanges()),
- MMO->getSyncScopeID(), MMO->getSuccessOrdering(),
- MMO->getFailureOrdering());
+ MMOMetadata(MMO->getAAInfo(), MMO->getRanges()), MMO->getSyncScopeID(),
+ MMO->getSuccessOrdering(), MMO->getFailureOrdering());
}
SDValue Load0 = DAG.getLoad(LoadTy, dl, Chain, Base0, WideMMO);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 9d969a560b3c9..ae16e49260ec2 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -8920,9 +8920,9 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) {
// Drop range metadata, as this metadata becomes invalid for f64 bit
// reinterpretation of i64 values.
- Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI,
- RLI.Alignment, RLI.MMOFlags(),
- MMOMetadata(RLI.AAInfo));
+ Bits =
+ DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment,
+ RLI.MMOFlags(), MMOMetadata(RLI.AAInfo));
if (RLI.ResChain)
DAG.makeEquivalentMemoryOrdering(RLI.ResChain, Bits.getValue(1));
} else if (Subtarget.hasLFIWAX() &&
>From a5da8adaa0f26f2a942c588d31ef0643f26c7b72 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 9 Jul 2026 09:17:51 +0000
Subject: [PATCH 26/33] more api churn
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 3 +--
llvm/lib/Target/Hexagon/HexagonISelLowering.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b2f91e97fd426..963530fbbb27f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13073,8 +13073,7 @@ SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
SDValue NewLoad = DAG.getLoad(
ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr,
Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(),
- Ld->getMemOperand()->getFlags(), Ld->getAAInfo(),
- nullptr); // Drop ranges
+ Ld->getMemOperand()->getFlags(), Ld->getAAInfo()); // Drop ranges
EVT TruncVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
if (MemVT.isFloatingPoint()) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index fba37e0baa8af..7b6c99a33c703 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3042,7 +3042,7 @@ HexagonTargetLowering::LowerLoad(SDValue Op, SelectionDAG &DAG) const {
LN->getAddressingMode(), ISD::ZEXTLOAD, MVT::i32, dl, LN->getChain(),
LN->getBasePtr(), LN->getOffset(), LN->getPointerInfo(),
/*MemoryVT*/ MVT::i8, LN->getAlign(), LN->getMemOperand()->getFlags(),
- LN->getAAInfo(), LN->getRanges());
+ MMOMetadata(LN->getAAInfo(), LN->getRanges()));
LN = cast<LoadSDNode>(NL.getNode());
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 080a3a007f643..896fe80d9cb67 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13701,7 +13701,7 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
SDValue NewLoad =
DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
MMO->getPointerInfo(), MMO->getBaseAlign(), MMO->getFlags(),
- MMO->getAAInfo(), MMO->getRanges());
+ MMOMetadata(MMO->getAAInfo(), MMO->getRanges()));
SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
}
>From 11d1590fcea3bd2fea1d0a7872ce7fcb7f9f8c3d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 24 Jul 2026 12:34:45 -0700
Subject: [PATCH 27/33] Apply suggestion from @justinfargnoli
Co-authored-by: Justin Fargnoli <jfargnoli at nvidia.com>
---
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 10 ----------
1 file changed, 10 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 6304b3ada26e8..fe6e92ae8b20b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -126,16 +126,6 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
return SmVersion >= 61 && PTXVersion >= 50;
}
// Cache hint SM/PTX version requirements
- //
- // | Cache Hint | SM Requirement | PTX Requirement |
- // |-----------------|----------------|-----------------|
- // | L1::evict_* | SM 70+ | PTX 7.4+ |
- // | L2::evict_* | SM 100+ | PTX 8.8+ |
- // | L2::64B | SM 75+ | PTX 7.4+ |
- // | L2::128B | SM 75+ | PTX 7.4+ |
- // | L2::256B | SM 80+ | PTX 7.4+ |
- // | L2::cache_hint | SM 80+ | PTX 7.4+ |
- //
bool hasL1EvictionHint() const { return SmVersion >= 70 && PTXVersion >= 74; }
bool hasL2EvictionHint() const {
return SmVersion >= 100 && PTXVersion >= 88;
>From 2aec7b9ad8771ff78e5ecad1953598d8d6255eec Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 00:19:25 +0000
Subject: [PATCH 28/33] fix more merge conflicts
---
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 16 ++++++++++++++++
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 12 ++++++------
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 8d218e393c84b..79efea5898091 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -69,6 +69,13 @@ struct NVPTXScopes {
LLVMContext *Context = nullptr;
};
+struct NVPTXMemCacheHintAccess {
+ NVPTX::AddressSpace AddrSpace;
+ bool IsLoad;
+ unsigned NumElts;
+ unsigned EltWidth;
+};
+
class NVPTXDAGToDAGISel : public SelectionDAGISel {
const NVPTXTargetMachine &TM;
@@ -133,6 +140,15 @@ class NVPTXDAGToDAGISel : public SelectionDAGISel {
SDValue getPTXCmpMode(const CondCodeSDNode &CondCode);
SDValue selectPossiblyImm(SDValue V);
+ // Returns the encoded eviction/prefetch hint and cache policy register for a
+ // memory operation. Hints unsupported by the subtarget or address space are
+ // dropped. If L2::cache_hint is active, returns the hint with
+ // L2CacheHintBit set and a register containing the 64-bit cache policy
+ // value. Otherwise returns NOREG for the policy operand.
+ std::pair<unsigned, SDValue>
+ getMemCacheHintOperands(const MemSDNode *N, NVPTXMemCacheHintAccess Access,
+ const SDLoc &DL);
+
// Returns the Memory Order and Scope that the PTX memory instruction should
// use, and inserts appropriate fence instruction before the memory
// instruction, if needed to implement the instructions memory order. Required
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 46a0f91ed8881..e1c10171ae8ed 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -126,14 +126,14 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
return getSmVersion() >= 61 && PTXVersion >= 50;
}
// Cache hint SM/PTX version requirements
- bool hasL1EvictionHint() const { return SmVersion >= 70 && PTXVersion >= 74; }
+ bool hasL1EvictionHint() const { return getSmVersion() >= 70 && PTXVersion >= 74; }
bool hasL2EvictionHint() const {
- return SmVersion >= 100 && PTXVersion >= 88;
+ return getSmVersion() >= 100 && PTXVersion >= 88;
}
- bool hasL2Prefetch64B() const { return SmVersion >= 75 && PTXVersion >= 74; }
- bool hasL2Prefetch128B() const { return SmVersion >= 75 && PTXVersion >= 74; }
- bool hasL2Prefetch256B() const { return SmVersion >= 80 && PTXVersion >= 74; }
- bool hasL2CacheHint() const { return SmVersion >= 80 && PTXVersion >= 74; }
+ bool hasL2Prefetch64B() const { return getSmVersion() >= 75 && PTXVersion >= 74; }
+ bool hasL2Prefetch128B() const { return getSmVersion() >= 75 && PTXVersion >= 74; }
+ bool hasL2Prefetch256B() const { return getSmVersion() >= 80 && PTXVersion >= 74; }
+ bool hasL2CacheHint() const { return getSmVersion() >= 80 && PTXVersion >= 74; }
// Checks following instructions support:
// - tcgen05.ld/st
>From 1e9760d4be45acd2e39c7c879f247a42d6881289 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 02:24:47 +0000
Subject: [PATCH 29/33] Opus review
---
llvm/include/llvm/CodeGen/CodeGenCommonISel.h | 4 +-
llvm/lib/CodeGen/CodeGenCommonISel.cpp | 2 +-
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp | 1 +
llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp | 9 ++-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 56 ++++++++++---------
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 28 ++++------
llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll | 16 ++++++
.../CodeGen/NVPTX/cache-hint-transforms.ll | 48 +++++++++++-----
8 files changed, 102 insertions(+), 62 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
index cc4b92306d797..842cd37535637 100644
--- a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
+++ b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
@@ -229,7 +229,9 @@ findSplitPointForStackProtector(MachineBasicBlock *BB,
LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp);
/// Return the cache hint metadata node for memory operand \p OperandNo on \p I,
-/// or nullptr when the instruction has no hint for that operand.
+/// or nullptr when the instruction has no hint for that operand. For a \c CallBase,
+/// \p OperandNo is an argument index; otherwise it is an instruction operand
+/// index.
LLVM_ABI const MDNode *getMemCacheHintMetadata(const Instruction &I,
unsigned OperandNo = 0);
diff --git a/llvm/lib/CodeGen/CodeGenCommonISel.cpp b/llvm/lib/CodeGen/CodeGenCommonISel.cpp
index c07b6bd3dbd1e..77421253c0b26 100644
--- a/llvm/lib/CodeGen/CodeGenCommonISel.cpp
+++ b/llvm/lib/CodeGen/CodeGenCommonISel.cpp
@@ -34,7 +34,7 @@ const MDNode *llvm::getMemCacheHintMetadata(const Instruction &I,
if (!MD)
return nullptr;
- for (unsigned Idx = 0, E = MD->getNumOperands(); Idx != E; Idx += 2) {
+ for (unsigned Idx = 0; Idx + 1 < MD->getNumOperands(); Idx += 2) {
const auto *OpNoCI = mdconst::extract<ConstantInt>(MD->getOperand(Idx));
const auto *Hint = cast<MDNode>(MD->getOperand(Idx + 1));
if (OpNoCI->getZExtValue() == OperandNo)
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index c2e97ecb6f17f..b9d97598eab50 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -432,6 +432,7 @@ void NVPTXInstPrinter::printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
return;
}
}
+ llvm_unreachable(formatv("Unknown Modifier: {}", Modifier).str().c_str());
}
void NVPTXInstPrinter::printCachePolicy(const MCInst *MI, int OpNum,
diff --git a/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp b/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
index 990a94200eb9f..d7703fbde48fd 100644
--- a/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXForwardParams.cpp
@@ -112,7 +112,14 @@ static bool eliminateMove(MachineInstr &Mov, const MachineRegisterInfo &MRI,
Idx = getNamedOperandIdx(Opc, NVPTX::OpName::policy);
assert(Idx != -1 && "no policy operand");
- LI->getOperand(Idx).ChangeToRegister(NVPTX::NoRegister, false);
+ MachineOperand &Policy = LI->getOperand(Idx);
+ Register PolicyReg = Policy.getReg();
+ MachineInstr *PolicyDef =
+ PolicyReg.isValid() ? MRI.getVRegDef(PolicyReg) : nullptr;
+ Policy.ChangeToRegister(NVPTX::NoRegister, false);
+ // Remove the policy register's definition if it is now dead.
+ if (PolicyDef && PolicyDef->isDead(MRI))
+ RemoveList.push_back(PolicyDef);
}
return true;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 79efea5898091..ae55240b00e7c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1167,22 +1167,23 @@ static std::optional<NVPTX::L2Prefetch> parseL2Prefetch(StringRef Str) {
.Default(std::nullopt);
}
-template <typename T, typename ParseFn>
-static void parseMemCacheHintStringValue(LLVMContext &Ctx, StringRef Key,
- const Metadata *Value, T &Result,
- ParseFn Parse) {
+template <typename T>
+static std::optional<T>
+parseMemCacheHintStringValue(LLVMContext &Ctx, StringRef Key,
+ const Metadata *Value,
+ std::optional<T> (*Parse)(StringRef)) {
const auto *Val = dyn_cast<MDString>(Value);
if (!Val) {
emitInvalidMemCacheHint(Ctx, Twine("'") + Key + "' expects a string value");
- return;
+ return std::nullopt;
}
StringRef ValStr = Val->getString();
- if (auto Parsed = Parse(ValStr))
- Result = *Parsed;
- else
+ auto Parsed = Parse(ValStr);
+ if (!Parsed)
emitInvalidMemCacheHint(Ctx, Twine("unknown value '") + ValStr + "' for '" +
Key + "'");
+ return Parsed;
}
static bool isGlobalOrGeneric(NVPTX::AddressSpace AddrSpace) {
@@ -1243,37 +1244,38 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
const Metadata *Value = Node->getOperand(I + 1).get();
if (KeyStr == "nvvm.l1_eviction") {
- if (Subtarget->hasL1EvictionHint())
- parseMemCacheHintStringValue(Ctx, KeyStr, Value, L1, parseL1Eviction);
+ auto ParsedL1 =
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, parseL1Eviction);
+ if (ParsedL1 && Subtarget->hasL1EvictionHint())
+ L1 = *ParsedL1;
continue;
}
if (KeyStr == "nvvm.l2_eviction") {
- NVPTX::L2Eviction ParsedL2 = NVPTX::L2Eviction::Normal;
- parseMemCacheHintStringValue(Ctx, KeyStr, Value, ParsedL2,
- parseL2Eviction);
- if (isL2EvictionSupported(*Subtarget, Access, ParsedL2))
- L2 = ParsedL2;
+ auto ParsedL2 =
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, parseL2Eviction);
+ if (ParsedL2 && isL2EvictionSupported(*Subtarget, Access, *ParsedL2))
+ L2 = *ParsedL2;
continue;
}
if (KeyStr == "nvvm.l2_prefetch_size") {
- NVPTX::L2Prefetch ParsedPrefetch = NVPTX::L2Prefetch::None;
- parseMemCacheHintStringValue(Ctx, KeyStr, Value, ParsedPrefetch,
- parseL2Prefetch);
- if (isL2PrefetchSupported(*Subtarget, ParsedPrefetch, Access))
- Prefetch = ParsedPrefetch;
+ auto ParsedPrefetch =
+ parseMemCacheHintStringValue(Ctx, KeyStr, Value, parseL2Prefetch);
+ if (ParsedPrefetch &&
+ isL2PrefetchSupported(*Subtarget, *ParsedPrefetch, Access))
+ Prefetch = *ParsedPrefetch;
continue;
}
if (KeyStr == "nvvm.l2_cache_hint") {
- if (isGlobalOrGeneric(Access.AddrSpace) && Subtarget->hasL2CacheHint()) {
- if (auto *ValCI = mdconst::dyn_extract<ConstantInt>(Value))
- CachePolicy = ValCI->getZExtValue();
- else
- emitInvalidMemCacheHint(
- Ctx, "'nvvm.l2_cache_hint' expects an integer value");
- }
+ const auto *ValCI = mdconst::dyn_extract<ConstantInt>(Value);
+ if (!ValCI)
+ emitInvalidMemCacheHint(
+ Ctx, "'nvvm.l2_cache_hint' expects an integer value");
+ else if (isGlobalOrGeneric(Access.AddrSpace) &&
+ Subtarget->hasL2CacheHint())
+ CachePolicy = ValCI->getZExtValue();
continue;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 31d5075970e93..5c0a0764b195a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2697,20 +2697,20 @@ def LDU_GLOBAL_v4i32 : VLDU_G_ELE_V4<B32>;
// Support for ldg on sm_35 or later
//-----------------------------------
+defvar CacheHintQualifiers =
+ "${evictionAndPrefetchHint:l1}${evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}";
+
// Don't annotate ld.global.nc as mayLoad, because these loads go through the
// non-coherent texture cache, and therefore the values read must be read-only
// during the lifetime of the kernel.
-
class LDG_G<NVPTXRegClass regclass>
: NVPTXInst<(outs regclass:$result),
(ins AtomicCode:$Sign, i32imm:$fromWidth,
UsedBytesMask:$usedBytes, ADDR:$src,
EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}"
- "ld.global.nc${evictionAndPrefetchHint:l1}${"
- "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.${"
- "Sign:sign}$fromWidth $result, [$src]${policy};">;
+ "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
+ ".${Sign:sign}$fromWidth \t$result, [$src]${policy};">;
def LD_GLOBAL_NC_i16 : LDG_G<B16>;
def LD_GLOBAL_NC_i32 : LDG_G<B32>;
@@ -2724,10 +2724,8 @@ class VLDG_G_ELE_V2<NVPTXRegClass regclass> :
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}"
- "ld.global.nc${evictionAndPrefetchHint:l1}${"
- "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v2.${"
- "Sign:sign}$fromWidth {{$dst1, $dst2}}, [$src]${policy};">;
+ "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
+ ".v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src]${policy};">;
class VLDG_G_ELE_V4<NVPTXRegClass regclass> :
NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
@@ -2735,10 +2733,8 @@ class VLDG_G_ELE_V4<NVPTXRegClass regclass> :
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}"
- "ld.global.nc${evictionAndPrefetchHint:l1}${"
- "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v4.${"
- "Sign:sign}$fromWidth {{$dst1, $dst2, $dst3, $dst4}}, [$src]${policy};">;
+ "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
+ ".v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src]${policy};">;
class VLDG_G_ELE_V8<NVPTXRegClass regclass> :
NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
@@ -2747,10 +2743,8 @@ class VLDG_G_ELE_V8<NVPTXRegClass regclass> :
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}"
- "ld.global.nc${evictionAndPrefetchHint:l1}${"
- "evictionAndPrefetchHint:l2}${evictionAndPrefetchHint:prefetch}.v8.${"
- "Sign:sign}$fromWidth {{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src]${policy};">;
+ "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
+ ".v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src]${policy};">;
// FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.
def LD_GLOBAL_NC_v2i16 : VLDG_G_ELE_V2<B16>;
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll b/llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll
index e0bfe79b190c4..e17e4b0b496b8 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-invalid.ll
@@ -3,12 +3,15 @@
; RUN: llc < %t/bad-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-KEY
; RUN: llc < %t/bad-other-key.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-OTHER-KEY
; RUN: llc < %t/bad-l1-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-TYPE
+; RUN: llc < %t/bad-l1-value.ll -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx50 2>&1 | FileCheck %s --check-prefix=BAD-L1-VALUE
; RUN: llc < %t/bad-l1-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L1-VALUE
; RUN: llc < %t/bad-l2-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-TYPE
; RUN: llc < %t/bad-l2-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-L2-VALUE
; RUN: llc < %t/bad-prefetch-type.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-TYPE
; RUN: llc < %t/bad-prefetch-value.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-PREFETCH-VALUE
; RUN: llc < %t/bad-policy.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-POLICY
+; RUN: llc < %t/bad-policy.ll -mtriple=nvptx64 -mcpu=sm_60 -mattr=+ptx50 2>&1 | FileCheck %s --check-prefix=BAD-POLICY
+; RUN: llc < %t/bad-policy-shared.ll -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 2>&1 | FileCheck %s --check-prefix=BAD-POLICY-SHARED
;--- bad-empty.ll
@@ -129,3 +132,16 @@ define i32 @bad_cache_policy_value(ptr addrspace(1) %p) {
!0 = !{i32 0, !1}
!1 = !{!"nvvm.l2_cache_hint", !"not_an_integer"}
+
+;--- bad-policy-shared.ll
+
+; nvvm.l2_cache_hint expects an integer cache-policy value even when the
+; address space does not support the hint.
+; BAD-POLICY-SHARED: warning: invalid NVPTX !mem.cache_hint metadata: 'nvvm.l2_cache_hint' expects an integer value
+define i32 @bad_cache_policy_value(ptr addrspace(3) %p) {
+ %v = load i32, ptr addrspace(3) %p, !mem.cache_hint !0
+ ret i32 %v
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"nvvm.l2_cache_hint", !"not_an_integer"}
diff --git a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
index 04b091514d0d4..39d8284616033 100644
--- a/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
+++ b/llvm/test/CodeGen/NVPTX/cache-hint-transforms.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "^\s*(?:mov\.b64|ld(?:\.[A-Za-z0-9_:]+)*\.global|st(?:\.[A-Za-z0-9_:]+)*\.global|atom(?:\.[A-Za-z0-9_:]+)*\.global|ld\.param\.b32)" --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | FileCheck %s --check-prefixes=CHECK,O2
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 -O0 | FileCheck %s --check-prefixes=CHECK,O0
; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx74 | %ptxas-verify %}
; Test cache hint handling across shared pointers, CSE, forwarding, and legalization.
@@ -122,21 +123,33 @@ define void @test_dagcombine_store_concat_trunc_v8i32(ptr addrspace(1) %p, <4 x
;-----------------------------------------------------------------------------
define <16 x i32> @test_legalize_split_load_v16i32(ptr addrspace(1) %p) {
-; CHECK-LABEL: test_legalize_split_load_v16i32(
-; CHECK: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; CHECK: ld.global.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
-; CHECK: ld.global.v4.b32 {%r9, %r10, %r11, %r12}, [%rd1+32];
-; CHECK: ld.global.v4.b32 {%r13, %r14, %r15, %r16}, [%rd1+48];
+; O2-LABEL: test_legalize_split_load_v16i32(
+; O2: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; O2: ld.global.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; O2: ld.global.v4.b32 {%r9, %r10, %r11, %r12}, [%rd1+32];
+; O2: ld.global.v4.b32 {%r13, %r14, %r15, %r16}, [%rd1+48];
+;
+; O0-LABEL: test_legalize_split_load_v16i32(
+; O0: ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1+48];
+; O0: ld.global.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+32];
+; O0: ld.global.v4.b32 {%r9, %r10, %r11, %r12}, [%rd1+16];
+; O0: ld.global.v4.b32 {%r13, %r14, %r15, %r16}, [%rd1];
%v = load <16 x i32>, ptr addrspace(1) %p, align 16, !mem.cache_hint !11
ret <16 x i32> %v
}
define void @test_legalize_split_store_v16i32(ptr addrspace(1) %p, <16 x i32> %v) {
-; CHECK-LABEL: test_legalize_split_store_v16i32(
-; CHECK: st.global.v4.b32 [%rd1+48], {%r1, %r2, %r3, %r4};
-; CHECK: st.global.v4.b32 [%rd1+32], {%r5, %r6, %r7, %r8};
-; CHECK: st.global.v4.b32 [%rd1+16], {%r9, %r10, %r11, %r12};
-; CHECK: st.global.v4.b32 [%rd1], {%r13, %r14, %r15, %r16};
+; O2-LABEL: test_legalize_split_store_v16i32(
+; O2: st.global.v4.b32 [%rd1+48], {%r1, %r2, %r3, %r4};
+; O2: st.global.v4.b32 [%rd1+32], {%r5, %r6, %r7, %r8};
+; O2: st.global.v4.b32 [%rd1+16], {%r9, %r10, %r11, %r12};
+; O2: st.global.v4.b32 [%rd1], {%r13, %r14, %r15, %r16};
+;
+; O0-LABEL: test_legalize_split_store_v16i32(
+; O0: st.global.v4.b32 [%rd1+48], {%r13, %r14, %r15, %r16};
+; O0: st.global.v4.b32 [%rd1+32], {%r9, %r10, %r11, %r12};
+; O0: st.global.v4.b32 [%rd1+16], {%r5, %r6, %r7, %r8};
+; O0: st.global.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
store <16 x i32> %v, ptr addrspace(1) %p, align 16, !mem.cache_hint !12
ret void
}
@@ -151,10 +164,15 @@ define <3 x i64> @test_legalize_scalarize_load_v3i64(ptr addrspace(1) %p) {
}
define void @test_legalize_scalarize_store_v3i64(ptr addrspace(1) %p, <3 x i64> %v) {
-; CHECK-LABEL: test_legalize_scalarize_store_v3i64(
-; CHECK: st.global.b64 [%rd1+16], %rd2;
-; CHECK: st.global.b64 [%rd1+8], %rd4;
-; CHECK: st.global.b64 [%rd1], %rd3;
+; O2-LABEL: test_legalize_scalarize_store_v3i64(
+; O2: st.global.b64 [%rd1+16], %rd2;
+; O2: st.global.b64 [%rd1+8], %rd4;
+; O2: st.global.b64 [%rd1], %rd3;
+;
+; O0-LABEL: test_legalize_scalarize_store_v3i64(
+; O0: st.global.b64 [%rd1+16], %rd4;
+; O0: st.global.b64 [%rd1+8], %rd3;
+; O0: st.global.b64 [%rd1], %rd2;
store <3 x i64> %v, ptr addrspace(1) %p, align 8, !mem.cache_hint !14
ret void
}
>From 9b67b470bd977f5e71edb536ded5bb0742463bd9 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 02:30:26 +0000
Subject: [PATCH 30/33] format
---
llvm/include/llvm/CodeGen/CodeGenCommonISel.h | 3 ++-
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 20 ++++++++++++++-----
3 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
index 842cd37535637..85f964c764923 100644
--- a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
+++ b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
@@ -229,7 +229,8 @@ findSplitPointForStackProtector(MachineBasicBlock *BB,
LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp);
/// Return the cache hint metadata node for memory operand \p OperandNo on \p I,
-/// or nullptr when the instruction has no hint for that operand. For a \c CallBase,
+/// or nullptr when the instruction has no hint for that operand. For a \c
+/// CallBase,
/// \p OperandNo is an argument index; otherwise it is an instruction operand
/// index.
LLVM_ABI const MDNode *getMemCacheHintMetadata(const Instruction &I,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ae55240b00e7c..be7a97f8f30ee 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -17,9 +17,9 @@
#include "NVPTXTargetMachine.h"
#include "NVPTXUtilities.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
-#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAG.h"
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index e1c10171ae8ed..28ac251b8adea 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -126,14 +126,24 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
return getSmVersion() >= 61 && PTXVersion >= 50;
}
// Cache hint SM/PTX version requirements
- bool hasL1EvictionHint() const { return getSmVersion() >= 70 && PTXVersion >= 74; }
+ bool hasL1EvictionHint() const {
+ return getSmVersion() >= 70 && PTXVersion >= 74;
+ }
bool hasL2EvictionHint() const {
return getSmVersion() >= 100 && PTXVersion >= 88;
}
- bool hasL2Prefetch64B() const { return getSmVersion() >= 75 && PTXVersion >= 74; }
- bool hasL2Prefetch128B() const { return getSmVersion() >= 75 && PTXVersion >= 74; }
- bool hasL2Prefetch256B() const { return getSmVersion() >= 80 && PTXVersion >= 74; }
- bool hasL2CacheHint() const { return getSmVersion() >= 80 && PTXVersion >= 74; }
+ bool hasL2Prefetch64B() const {
+ return getSmVersion() >= 75 && PTXVersion >= 74;
+ }
+ bool hasL2Prefetch128B() const {
+ return getSmVersion() >= 75 && PTXVersion >= 74;
+ }
+ bool hasL2Prefetch256B() const {
+ return getSmVersion() >= 80 && PTXVersion >= 74;
+ }
+ bool hasL2CacheHint() const {
+ return getSmVersion() >= 80 && PTXVersion >= 74;
+ }
// Checks following instructions support:
// - tcgen05.ld/st
>From 3b7cf15cf839e631c0c95de267b8b592b24ce94d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 02:48:40 +0000
Subject: [PATCH 31/33] pass MMOMetadata by reference
---
llvm/include/llvm/CodeGen/MachineFunction.h | 8 ++++----
llvm/include/llvm/CodeGen/MachineMemOperand.h | 4 ++--
llvm/lib/CodeGen/MachineFunction.cpp | 4 ++--
llvm/lib/CodeGen/MachineOperand.cpp | 6 ++++--
4 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 9f5de700a5105..d84ce80e10a3c 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -1115,19 +1115,19 @@ class LLVM_ABI MachineFunction {
/// explicitly deallocated.
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
- Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, const MMOMetadata &Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, const MMOMetadata &Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, uint64_t Size,
- Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, const MMOMetadata &Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
@@ -1137,7 +1137,7 @@ class LLVM_ABI MachineFunction {
}
MachineMemOperand *getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, TypeSize Size,
- Align BaseAlignment, MMOMetadata Metadata = MMOMetadata(),
+ Align BaseAlignment, const MMOMetadata &Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic) {
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 242778ad7b523..b275bbae28140 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -203,13 +203,13 @@ class MachineMemOperand {
/// when store does not occur must also be specified.
LLVM_ABI
MachineMemOperand(MachinePointerInfo PtrInfo, Flags Flags, LocationSize TS,
- Align A, MMOMetadata Metadata = MMOMetadata(),
+ Align A, const MMOMetadata &Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
LLVM_ABI
MachineMemOperand(MachinePointerInfo PtrInfo, Flags Flags, LLT Type, Align A,
- MMOMetadata Metadata = MMOMetadata(),
+ const MMOMetadata &Metadata = MMOMetadata(),
SyncScope::ID SSID = SyncScope::System,
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 5a9f5e0f97ba7..144f165e17973 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -566,7 +566,7 @@ void MachineFunction::deleteMachineBasicBlock(MachineBasicBlock *MBB) {
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LocationSize Size,
- Align BaseAlignment, MMOMetadata Metadata, SyncScope::ID SSID,
+ Align BaseAlignment, const MMOMetadata &Metadata, SyncScope::ID SSID,
AtomicOrdering Ordering, AtomicOrdering FailureOrdering) {
assert((!Size.hasValue() ||
Size.getValue().getKnownMinValue() != ~UINT64_C(0)) &&
@@ -579,7 +579,7 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
MachineMemOperand *MachineFunction::getMachineMemOperand(
MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy,
- Align BaseAlignment, MMOMetadata Metadata, SyncScope::ID SSID,
+ Align BaseAlignment, const MMOMetadata &Metadata, SyncScope::ID SSID,
AtomicOrdering Ordering, AtomicOrdering FailureOrdering) {
return new (Allocator)
MachineMemOperand(PtrInfo, F, MemTy, BaseAlignment, Metadata, SSID,
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index 10d4db4d7ab79..3067f6e636130 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -1181,7 +1181,8 @@ MachinePointerInfo MachinePointerInfo::getUnknownStack(MachineFunction &MF) {
}
MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
- LLT Type, Align A, MMOMetadata Metadata,
+ LLT Type, Align A,
+ const MMOMetadata &Metadata,
SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
@@ -1203,7 +1204,8 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
MachineMemOperand::MachineMemOperand(MachinePointerInfo PtrInfo, Flags F,
LocationSize TS, Align BaseAlignment,
- MMOMetadata Metadata, SyncScope::ID SSID,
+ const MMOMetadata &Metadata,
+ SyncScope::ID SSID,
AtomicOrdering Ordering,
AtomicOrdering FailureOrdering)
: MachineMemOperand(
>From 24c521b5ab0bf35ffa201255b1a7816b8a5bd4fa Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 4 Aug 2026 23:43:02 +0000
Subject: [PATCH 32/33] fix error, use strconcat instead of #
---
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 5c0a0764b195a..f1c6e28232950 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2709,8 +2709,8 @@ class LDG_G<NVPTXRegClass regclass>
UsedBytesMask:$usedBytes, ADDR:$src,
EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
- ".${Sign:sign}$fromWidth \t$result, [$src]${policy};">;
+ !strconcat("${usedBytes}", "ld.global.nc", CacheHintQualifiers,
+ ".${Sign:sign}$fromWidth \t$result, [$src]${policy};")>;
def LD_GLOBAL_NC_i16 : LDG_G<B16>;
def LD_GLOBAL_NC_i32 : LDG_G<B32>;
@@ -2724,8 +2724,8 @@ class VLDG_G_ELE_V2<NVPTXRegClass regclass> :
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
- ".v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src]${policy};">;
+ !strconcat("${usedBytes}", "ld.global.nc", CacheHintQualifiers,
+ ".v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src]${policy};")>;
class VLDG_G_ELE_V4<NVPTXRegClass regclass> :
NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
@@ -2733,8 +2733,8 @@ class VLDG_G_ELE_V4<NVPTXRegClass regclass> :
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
- ".v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src]${policy};">;
+ !strconcat("${usedBytes}", "ld.global.nc", CacheHintQualifiers,
+ ".v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src]${policy};")>;
class VLDG_G_ELE_V8<NVPTXRegClass regclass> :
NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3,
@@ -2743,8 +2743,8 @@ class VLDG_G_ELE_V8<NVPTXRegClass regclass> :
(ins AtomicCode:$Sign, i32imm:$fromWidth, UsedBytesMask:$usedBytes,
ADDR:$src, EvictionAndPrefetchHint:$evictionAndPrefetchHint,
CachePolicy:$policy),
- "${usedBytes}" # "ld.global.nc" # CacheHintQualifiers #
- ".v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src]${policy};">;
+ !strconcat("${usedBytes}", "ld.global.nc", CacheHintQualifiers,
+ ".v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src]${policy};")>;
// FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.
def LD_GLOBAL_NC_v2i16 : VLDG_G_ELE_V2<B16>;
>From 4988ac046e6cc8bbdc2c22bd5fcc43f159bd1c33 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 5 Aug 2026 23:55:19 +0000
Subject: [PATCH 33/33] antonio feedback
---
llvm/include/llvm/CodeGen/CodeGenCommonISel.h | 5 ++---
llvm/lib/Target/NVPTX/NVPTX.h | 3 ---
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 6 +++---
3 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
index 85f964c764923..c07ae50793cb2 100644
--- a/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
+++ b/llvm/include/llvm/CodeGen/CodeGenCommonISel.h
@@ -230,9 +230,8 @@ LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp);
/// Return the cache hint metadata node for memory operand \p OperandNo on \p I,
/// or nullptr when the instruction has no hint for that operand. For a \c
-/// CallBase,
-/// \p OperandNo is an argument index; otherwise it is an instruction operand
-/// index.
+/// CallBase, \p OperandNo is an argument index; otherwise it is an instruction
+/// operand index.
LLVM_ABI const MDNode *getMemCacheHintMetadata(const Instruction &I,
unsigned OperandNo = 0);
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 5adb2ca3e6cac..79726c4a8f184 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -23,13 +23,10 @@
#include "llvm/Target/TargetMachine.h"
namespace llvm {
-class Function;
class FunctionPass;
-class MachineMemOperand;
class MachineFunctionPass;
class NVPTXTargetMachine;
class PassRegistry;
-class Value;
namespace NVPTXCC {
enum CondCodes {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index be7a97f8f30ee..72c3e2bea406a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1211,8 +1211,8 @@ static bool isL2PrefetchSupported(const NVPTXSubtarget &Subtarget,
}
static bool isL2EvictionSupported(const NVPTXSubtarget &Subtarget,
- NVPTXMemCacheHintAccess Access,
- NVPTX::L2Eviction Eviction) {
+ NVPTX::L2Eviction Eviction,
+ NVPTXMemCacheHintAccess Access) {
if (Eviction == NVPTX::L2Eviction::Normal)
return true;
@@ -1254,7 +1254,7 @@ std::pair<unsigned, SDValue> NVPTXDAGToDAGISel::getMemCacheHintOperands(
if (KeyStr == "nvvm.l2_eviction") {
auto ParsedL2 =
parseMemCacheHintStringValue(Ctx, KeyStr, Value, parseL2Eviction);
- if (ParsedL2 && isL2EvictionSupported(*Subtarget, Access, *ParsedL2))
+ if (ParsedL2 && isL2EvictionSupported(*Subtarget, *ParsedL2, Access))
L2 = *ParsedL2;
continue;
}
More information about the llvm-commits
mailing list