[llvm] [SelectionDAG] Add an ISD::CLEAR_CACHE node to lower llvm.clear_cache (PR #93795)
Roger Ferrer Ibáñez via llvm-commits
llvm-commits at lists.llvm.org
Thu May 30 04:22:39 PDT 2024
https://github.com/rofirrim updated https://github.com/llvm/llvm-project/pull/93795
>From cea00d83e404fec8279cb69732dc76bd20fa0c5a Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Thu, 30 May 2024 06:34:53 +0000
Subject: [PATCH 1/6] [SelectionDAG] Add ISD::CLEAR_CACHE
The existing way to lower llvm.clear_cache is not ideal. As suggested by
Matt Arsenault, let's turn this into a regular ISD node.
This change adds the node itself and nothing else.
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 5 +++++
.../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 15 +++++++++++----
.../CodeGen/SelectionDAG/SelectionDAGDumper.cpp | 2 ++
3 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index d8af97957e48e..0f87e062e2da6 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1407,6 +1407,11 @@ enum NodeType {
// Output: Output Chain
EXPERIMENTAL_VECTOR_HISTOGRAM,
+ // llvm.clear_cache intrinsic
+ // Operands: Input Chain, Start Addres, End Address
+ // Outputs: Output Chain
+ CLEAR_CACHE,
+
/// BUILTIN_OP_END - This must be the last enum value in this list.
/// The target-specific pre-isel opcode values start here.
BUILTIN_OP_END
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ca352da5d36eb..5d113defd704d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7516,11 +7516,18 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::invariant_end:
// Discard region information.
return;
- case Intrinsic::clear_cache:
- /// FunctionName may be null.
- if (const char *FunctionName = TLI.getClearCacheBuiltinName())
- lowerCallToExternalSymbol(I, FunctionName);
+ case Intrinsic::clear_cache: {
+ SDValue InputChain = DAG.getRoot();
+ Value *Start = I.getArgOperand(0);
+ SDValue StartVal = getValue(Start);
+ Value *End = I.getArgOperand(1);
+ SDValue EndVal = getValue(End);
+ Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
+ {InputChain, StartVal, EndVal});
+ setValue(&I, Res);
+ DAG.setRoot(Res);
return;
+ }
case Intrinsic::donothing:
case Intrinsic::seh_try_begin:
case Intrinsic::seh_scope_begin:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 59742e90c6791..2198c2354483c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -528,6 +528,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
return "stackmap";
case ISD::PATCHPOINT:
return "patchpoint";
+ case ISD::CLEAR_CACHE:
+ return "clear_cache";
case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
return "histogram";
>From de267dce493191ffe6ab742a10603fb9fe10cd00 Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Thu, 30 May 2024 07:00:39 +0000
Subject: [PATCH 2/6] [SelectionDAG] Plumbing lowering of ISD::CLEAR_CACHE as a
libcall in TLI
Lower the newly introduced ISD::CLEAR_CACHE as a runtime function. By
default we lower it to __clear_cache as customary.
---
llvm/include/llvm/IR/RuntimeLibcalls.def | 3 +++
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 16 ++++++++++++++++
llvm/lib/CodeGen/TargetLoweringBase.cpp | 4 ++++
3 files changed, 23 insertions(+)
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index 5e082769fa974..a5a72884a6bf5 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -616,5 +616,8 @@ HANDLE_LIBCALL(DEOPTIMIZE, "__llvm_deoptimize")
// Return address
HANDLE_LIBCALL(RETURN_ADDRESS, nullptr)
+// Clear cache
+HANDLE_LIBCALL(CLEAR_CACHE, "__clear_cache")
+
HANDLE_LIBCALL(UNKNOWN_LIBCALL, nullptr)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 51f2cf9017f85..6b384d01090f3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1105,6 +1105,11 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
if (Action == TargetLowering::Legal)
Action = TargetLowering::Custom;
break;
+ case ISD::CLEAR_CACHE:
+ // This operation is typically going to be LibCall unless the target wants
+ // something differrent.
+ Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
+ break;
case ISD::READCYCLECOUNTER:
case ISD::READSTEADYCOUNTER:
// READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
@@ -4455,6 +4460,17 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
Results.push_back(CallResult.second);
break;
}
+ case ISD::CLEAR_CACHE: {
+ TargetLowering::MakeLibCallOptions CallOptions;
+ SDValue InputChain = Node->getOperand(0);
+ SDValue StartVal = Node->getOperand(1);
+ SDValue EndVal = Node->getOperand(2);
+ std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
+ DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal},
+ CallOptions, SDLoc(Node), InputChain);
+ Results.push_back(Tmp.second);
+ break;
+ }
case ISD::FMINNUM:
case ISD::STRICT_FMINNUM:
ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f2e4632b248f4..3aec7049e0cc8 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1035,6 +1035,10 @@ void TargetLoweringBase::initActions() {
setOperationAction(ISD::SET_FPMODE, VT, Expand);
}
setOperationAction(ISD::RESET_FPMODE, MVT::Other, Expand);
+
+ // This one by default will call __clear_cache unless the target
+ // wants something different.
+ setOperationAction(ISD::CLEAR_CACHE, MVT::Other, LibCall);
}
MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,
>From 1d37dc588e33acd4f40b5cfed7c0f1c4d61f9119 Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Thu, 30 May 2024 08:24:56 +0000
Subject: [PATCH 3/6] [X86][Webassembly] Update targets that need non-default
clear_cache lowering
Some targets do not lower llvm.clear_cache to a call to __clear_cache.
Update those to handle the new ISD::CLEAR_CACHE.
---
llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp | 3 +++
llvm/lib/Target/X86/X86ISelLowering.cpp | 7 +++++++
2 files changed, 10 insertions(+)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 4beab9d091581..a793e59c3d1a7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -111,6 +111,7 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
setOperationAction(ISD::JumpTable, MVTPtr, Custom);
setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
setOperationAction(ISD::BRIND, MVT::Other, Custom);
+ setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
// Take the default expansion for va_arg, va_copy, and va_end. There is no
// default action for va_start, so we do that custom.
@@ -1503,6 +1504,8 @@ SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
case ISD::CTLZ:
case ISD::CTTZ:
return DAG.UnrollVectorOp(Op.getNode());
+ case ISD::CLEAR_CACHE:
+ report_fatal_error("llvm.clear_cache is not supported on wasm");
}
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f5d0e1b15d7a3..929ce3d01b179 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -136,6 +136,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// X86-SSE is even stranger. It uses -1 or 0 for vector masks.
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
+ // X86 does not need to flush the instruction cache in any special way.
+ setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
+
// For 64-bit, since we have so many registers, use the ILP scheduler.
// For 32-bit, use the register pressure specific scheduling.
// For Atom, always use ILP scheduling.
@@ -32387,6 +32390,10 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::ADDRSPACECAST: return LowerADDRSPACECAST(Op, DAG);
case X86ISD::CVTPS2PH: return LowerCVTPS2PH(Op, DAG);
case ISD::PREFETCH: return LowerPREFETCH(Op, Subtarget, DAG);
+ case ISD::CLEAR_CACHE: {
+ // X86 instruction cache is coherent with its data cache so this is a no-op.
+ return Op->getOperand(0);
+ }
// clang-format on
}
}
>From 2c0d86848e8d29d2614c3c40d95dcb0fe237ba63 Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Thu, 30 May 2024 08:25:59 +0000
Subject: [PATCH 4/6] [SelectionDAG] Remove old getClearCacheBuiltinName hook
Now we can remove the old hook.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 6 ------
llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h | 4 ----
llvm/lib/Target/X86/X86ISelLowering.h | 5 -----
3 files changed, 15 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 50a8c7eb75af5..2765caec960c9 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4764,12 +4764,6 @@ class TargetLowering : public TargetLoweringBase {
return false;
}
- /// Return the builtin name for the __builtin___clear_cache intrinsic
- /// Default is to invoke the clear cache library call
- virtual const char * getClearCacheBuiltinName() const {
- return "__clear_cache";
- }
-
/// Return the register ID of the name passed in. Used by named register
/// global variables extension. There is no target-independent behaviour
/// so the default action is to bail.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index 1d1338ab40d0e..7d9cfb7739e43 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -111,10 +111,6 @@ class WebAssemblyTargetLowering final : public TargetLowering {
void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
SelectionDAG &DAG) const override;
- const char *getClearCacheBuiltinName() const override {
- report_fatal_error("llvm.clear_cache is not supported on wasm");
- }
-
bool
shouldSimplifyDemandedVectorElts(SDValue Op,
const TargetLoweringOpt &TLO) const override;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index b0efa074b1978..3c5c903bc0d98 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1473,11 +1473,6 @@ namespace llvm {
const SelectionDAG &DAG,
const MachineMemOperand &MMO) const override;
- /// Intel processors have a unified instruction and data cache
- const char * getClearCacheBuiltinName() const override {
- return nullptr; // nothing to do, move along.
- }
-
Register getRegisterByName(const char* RegName, LLT VT,
const MachineFunction &MF) const override;
>From 0ebe5dce8efc9b545fe3b9e04c5cce942a2afcb2 Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Thu, 30 May 2024 11:06:37 +0000
Subject: [PATCH 5/6] [SelectionDAG] Make the expansion of ISD::CLEAR_CACHE be
a no-op
Some targets like X86 have coherent instruction and data caches
so this expansion is convenient to have.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 5 +++++
llvm/lib/Target/X86/X86ISelLowering.cpp | 9 +++------
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 6b384d01090f3..2c6b737033616 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4303,6 +4303,11 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
Results.push_back(TLI.expandVPCTTZElements(Node, DAG));
break;
+ case ISD::CLEAR_CACHE:
+ // The default expansion of llvm.clear_cache is simply a no-op for those
+ // targets where it is not needed.
+ Results.push_back(Node->getOperand(0));
+ break;
case ISD::GLOBAL_OFFSET_TABLE:
case ISD::GlobalAddress:
case ISD::GlobalTLSAddress:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 929ce3d01b179..839006cbaed4c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -136,8 +136,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// X86-SSE is even stranger. It uses -1 or 0 for vector masks.
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
- // X86 does not need to flush the instruction cache in any special way.
- setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
+ // X86 instruction cache is coherent with its data cache so we can use the
+ // default expansion to a no-op.
+ setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Expand);
// For 64-bit, since we have so many registers, use the ILP scheduler.
// For 32-bit, use the register pressure specific scheduling.
@@ -32390,10 +32391,6 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::ADDRSPACECAST: return LowerADDRSPACECAST(Op, DAG);
case X86ISD::CVTPS2PH: return LowerCVTPS2PH(Op, DAG);
case ISD::PREFETCH: return LowerPREFETCH(Op, Subtarget, DAG);
- case ISD::CLEAR_CACHE: {
- // X86 instruction cache is coherent with its data cache so this is a no-op.
- return Op->getOperand(0);
- }
// clang-format on
}
}
>From cdf83908161f822678902b6341793cef2f0f4b3f Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer at bsc.es>
Date: Thu, 30 May 2024 11:07:57 +0000
Subject: [PATCH 6/6] clang-format
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 2c6b737033616..8cd2bb60d81f2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4471,8 +4471,8 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
SDValue StartVal = Node->getOperand(1);
SDValue EndVal = Node->getOperand(2);
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
- DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal},
- CallOptions, SDLoc(Node), InputChain);
+ DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions,
+ SDLoc(Node), InputChain);
Results.push_back(Tmp.second);
break;
}
More information about the llvm-commits
mailing list