[llvm] [NVPTX] Avoid nonlocal MachineCSE of special register reads. (PR #228303)
Nick Riasanovsky via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 18:58:25 PDT 2026
https://github.com/njriasan created https://github.com/llvm/llvm-project/pull/228303
Special hardware registers for information that remain static throughout a kernels execution (e.g. warpID, gridID) are bad candidates for CSE because they are cheap to recompute and therefore may unnecessarily increase register live ranges. This marks such registers as cheap so the compiler can decided that it is not profitable to extend their live ranges.
>From e7bbe2690c84e6ac15ed5bb4b464f8d9c9acccd3 Mon Sep 17 00:00:00 2001
From: Nick Riasanovsky <njriasan at meta.com>
Date: Thu, 1 Oct 2026 18:45:43 -0700
Subject: [PATCH 1/2] Mark hardware register values as cheap to compute
---
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 14 +-
llvm/test/CodeGen/NVPTX/machine-cse-sreg.mir | 189 +++++++++++++++++++
2 files changed, 202 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/NVPTX/machine-cse-sreg.mir
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index e6ce5fe5be48a..825dd13de3690 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -3883,6 +3883,7 @@ defm isspace_shared_cluster : ISSPACEP<"shared::cluster",
[SM90]>;
// Special register reads
+let isAsCheapAsAMove = true in
def MOV_SPECIAL : BasicNVPTXInst<(outs B32:$d),
(ins SpecialRegs:$r),
"mov.b32", []>;
@@ -5778,6 +5779,11 @@ class PTX_READ_SREG_R64<string regname, Intrinsic intop, list<Predicate> Preds=[
[(set i64:$d, (intop))]>,
Requires<Preds>;
+// Reads of immutable special registers are single moves, so they are cheap to
+// recompute: only CSE them locally or from an immediate predecessor, instead
+// of extending their live ranges across blocks (MachineCSE heuristic #1).
+// Volatile (clock) and memory-modeled (pm) reads below opt back out.
+let isAsCheapAsAMove = true in
class PTX_READ_SREG_R32<string regname, Intrinsic intop, list<Predicate> Preds=[]>
: NVPTXInst<(outs B32:$d), (ins),
"mov.u32 \t$d, %" # regname # ";",
@@ -5844,7 +5850,8 @@ def INT_PTX_SREG_LANEMASK_GE :
def INT_PTX_SREG_LANEMASK_GT :
PTX_READ_SREG_R32<"lanemask_gt", int_nvvm_read_ptx_sreg_lanemask_gt>;
-let hasSideEffects = 1 in {
+// Clock reads are volatile; keep them out of the cheap-as-move set.
+let hasSideEffects = 1, isAsCheapAsAMove = false in {
def SREG_CLOCK : PTX_READ_SREG_R32<"clock", int_nvvm_read_ptx_sreg_clock>;
def SREG_CLOCK64 : PTX_READ_SREG_R64<"clock64", int_nvvm_read_ptx_sreg_clock64>;
def SREG_GLOBALTIMER : PTX_READ_SREG_R64<"globaltimer", int_nvvm_read_ptx_sreg_globaltimer>;
@@ -5855,10 +5862,14 @@ def: Pat <(i64 (readcyclecounter)), (SREG_CLOCK64)>;
def: Pat <(i64 (readsteadycounter)), (SREG_GLOBALTIMER)>;
def: Pat <(i32 (readsteadycounter)), (SREG_GLOBALTIMER_LO)>;
+// Performance-monitor reads are modeled with memory effects; keep them out of
+// the cheap-as-move set.
+let isAsCheapAsAMove = false in {
def INT_PTX_SREG_PM0 : PTX_READ_SREG_R32<"pm0", int_nvvm_read_ptx_sreg_pm0>;
def INT_PTX_SREG_PM1 : PTX_READ_SREG_R32<"pm1", int_nvvm_read_ptx_sreg_pm1>;
def INT_PTX_SREG_PM2 : PTX_READ_SREG_R32<"pm2", int_nvvm_read_ptx_sreg_pm2>;
def INT_PTX_SREG_PM3 : PTX_READ_SREG_R32<"pm3", int_nvvm_read_ptx_sreg_pm3>;
+}
// Reserved shared memory special register reads
foreach suffix = ["begin", "end", "cap", "0", "1"] in {
@@ -5870,6 +5881,7 @@ foreach suffix = ["begin", "end", "cap", "0", "1"] in {
// TODO: It would be nice to use PTX_READ_SREG here, but it doesn't
// handle the constant.
+let isAsCheapAsAMove = true in
def INT_PTX_SREG_WARPSIZE :
NVPTXInst<(outs B32:$dst), (ins), "mov.u32 \t$dst, WARP_SZ;",
[(set i32:$dst, (int_nvvm_read_ptx_sreg_warpsize))]>;
diff --git a/llvm/test/CodeGen/NVPTX/machine-cse-sreg.mir b/llvm/test/CodeGen/NVPTX/machine-cse-sreg.mir
new file mode 100644
index 0000000000000..f469619f323ef
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/machine-cse-sreg.mir
@@ -0,0 +1,189 @@
+# RUN: llc -mtriple=nvptx64 -mcpu=sm_100 -run-pass=machine-cse -o - %s | FileCheck %s
+# Special-register reads are single moves, so MachineCSE must only eliminate
+# them locally (or from an immediate predecessor), not across blocks.
+
+---
+name: keep_ctaid_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_ctaid_reads
+ ; CHECK-COUNT-2: INT_PTX_SREG_CTAID_x
+ bb.0:
+ %0:b32 = INT_PTX_SREG_CTAID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = INT_PTX_SREG_CTAID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_tid_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_tid_reads
+ ; CHECK-COUNT-2: INT_PTX_SREG_TID_x
+ bb.0:
+ %0:b32 = INT_PTX_SREG_TID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = INT_PTX_SREG_TID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_ntid_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_ntid_reads
+ ; CHECK-COUNT-2: INT_PTX_SREG_NTID_x
+ bb.0:
+ %0:b32 = INT_PTX_SREG_NTID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = INT_PTX_SREG_NTID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_nctaid_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_nctaid_reads
+ ; CHECK-COUNT-2: INT_PTX_SREG_NCTAID_x
+ bb.0:
+ %0:b32 = INT_PTX_SREG_NCTAID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = INT_PTX_SREG_NCTAID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_laneid_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_laneid_reads
+ ; CHECK-COUNT-2: SREG_LANEID
+ bb.0:
+ %0:b32 = SREG_LANEID
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = SREG_LANEID
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_clusterid_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_clusterid_reads
+ ; CHECK-COUNT-2: INT_PTX_SREG_CLUSTERID_x
+ bb.0:
+ %0:b32 = INT_PTX_SREG_CLUSTERID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = INT_PTX_SREG_CLUSTERID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_warpsize_reads
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: keep_warpsize_reads
+ ; CHECK-COUNT-2: INT_PTX_SREG_WARPSIZE
+ bb.0:
+ %0:b32 = INT_PTX_SREG_WARPSIZE
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = INT_PTX_SREG_WARPSIZE
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: keep_envreg_reads
+tracksRegLiveness: true
+liveins:
+ - { reg: '$envreg0' }
+body: |
+ ; CHECK-LABEL: name: keep_envreg_reads
+ ; CHECK-COUNT-2: MOV_SPECIAL
+ bb.0:
+ liveins: $envreg0
+ %0:b32 = MOV_SPECIAL $envreg0
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ GOTO %bb.1
+
+ bb.1:
+ INLINEASM &"", sideeffect attdialect
+ GOTO %bb.2
+
+ bb.2:
+ %1:b32 = MOV_SPECIAL $envreg0
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
+---
+name: cse_ctaid_reads_locally
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: cse_ctaid_reads_locally
+ ; CHECK-COUNT-1: INT_PTX_SREG_CTAID_x
+ bb.0:
+ %0:b32 = INT_PTX_SREG_CTAID_x
+ %1:b32 = INT_PTX_SREG_CTAID_x
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %0
+ INLINEASM &"", sideeffect attdialect, reguse:B32, %1
+ Return
+
+...
>From eb2ce377a3e9067ab2a0c87032c7a14cc9f963ab Mon Sep 17 00:00:00 2001
From: Nick Riasanovsky <njriasan at meta.com>
Date: Thu, 1 Oct 2026 18:50:02 -0700
Subject: [PATCH 2/2] Remove unnecessary comments
---
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 95 +++++++++++-------------
1 file changed, 44 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 825dd13de3690..894e9be897b1a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -105,9 +105,9 @@ multiclass BARRIER_ALL<string asmstr, Intrinsic intrinsic, list<Predicate> requi
multiclass BARRIER_COUNT<string asmstr, Intrinsic intrinsic, list<Predicate> requires = []> {
let Predicates = requires in {
- def _rr : BasicNVPTXInst<(outs), (ins B32:$i, B32:$j), asmstr,
+ def _rr : BasicNVPTXInst<(outs), (ins B32:$i, B32:$j), asmstr,
[(intrinsic i32:$i, i32:$j)]>;
- def _ri : BasicNVPTXInst<(outs), (ins B32:$i, i32imm:$j), asmstr,
+ def _ri : BasicNVPTXInst<(outs), (ins B32:$i, i32imm:$j), asmstr,
[(intrinsic i32:$i, imm:$j)]>;
def _ir : BasicNVPTXInst<(outs), (ins i32imm:$i, B32:$j), asmstr,
[(intrinsic imm:$i, i32:$j)]>;
@@ -120,16 +120,16 @@ multiclass BARRIER_RED_ALL<string asmstr, Intrinsic intrinsic, RegTyInfo dst_rt,
let Predicates = requires in {
def _ip : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins i32imm:$i, B1:$pred), asmstr,
[(set dst_rt.Ty:$dest, (intrinsic imm:$i, i1:$pred))]>;
- def _rp : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins B32:$i, B1:$pred), asmstr,
+ def _rp : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins B32:$i, B1:$pred), asmstr,
[(set dst_rt.Ty:$dest, (intrinsic i32:$i, i1:$pred))]>;
}
}
multiclass BARRIER_RED_COUNT<string asmstr, Intrinsic intrinsic, RegTyInfo dst_rt, list<Predicate> requires = []> {
let Predicates = requires in {
- def _rrp : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins B32:$i, B32:$j, B1:$pred), asmstr,
+ def _rrp : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins B32:$i, B32:$j, B1:$pred), asmstr,
[(set dst_rt.Ty:$dest, (intrinsic i32:$i, i32:$j, i1:$pred))]>;
- def _rip : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins B32:$i, i32imm:$j, B1:$pred), asmstr,
+ def _rip : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins B32:$i, i32imm:$j, B1:$pred), asmstr,
[(set dst_rt.Ty:$dest, (intrinsic i32:$i, imm:$j, i1:$pred))]>;
def _irp : BasicNVPTXInst<(outs dst_rt.RC:$dest), (ins i32imm:$i, B32:$j, B1:$pred), asmstr,
[(set dst_rt.Ty:$dest, (intrinsic imm:$i, i32:$j, i1:$pred))]>;
@@ -1630,16 +1630,16 @@ class PREFETCH_INTRS<string InstName, Intrinsic Intr> :
[(Intr addr:$addr)]>,
Requires<[PTX80, SM90]>;
-def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1", int_nvvm_prefetchu_L1>;
+def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1", int_nvvm_prefetchu_L1>;
def PREFETCH_L1 : PREFETCH_INTRS<"prefetch.L1", int_nvvm_prefetch_L1>;
def PREFETCH_L2 : PREFETCH_INTRS<"prefetch.L2", int_nvvm_prefetch_L2>;
def PREFETCH_GLOBAL_L1 : PREFETCH_INTRS<"prefetch.global.L1", int_nvvm_prefetch_global_L1>;
def PREFETCH_LOCAL_L1 : PREFETCH_INTRS<"prefetch.local.L1", int_nvvm_prefetch_local_L1>;
def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2", int_nvvm_prefetch_global_L2>;
def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2", int_nvvm_prefetch_local_L2>;
-def PREFETCH_GLOBAL_L2_EVICT_NORMAL : PREFETCH_INTRS<"prefetch.global.L2::evict_normal",
+def PREFETCH_GLOBAL_L2_EVICT_NORMAL : PREFETCH_INTRS<"prefetch.global.L2::evict_normal",
int_nvvm_prefetch_global_L2_evict_normal>;
-def PREFETCH_GLOBAL_L2_EVICT_LAST : PREFETCH_INTRS<"prefetch.global.L2::evict_last",
+def PREFETCH_GLOBAL_L2_EVICT_LAST : PREFETCH_INTRS<"prefetch.global.L2::evict_last",
int_nvvm_prefetch_global_L2_evict_last>;
//Applypriority intrinsics
@@ -2447,7 +2447,7 @@ let Predicates = [SM100, doNoF32FTZ] in {
def : Pat<(f32 (fma (f32 (fpextend f16:$a)),
(f32 (fpextend f16:$b)), f32:$c)),
(INT_NVVM_MIXED_FMA_rn_f32_f16 B16:$a, B16:$b, B32:$c)>;
- def : Pat<(f32 (fma (f32 (fpextend bf16:$a)),
+ def : Pat<(f32 (fma (f32 (fpextend bf16:$a)),
(f32 (fpextend bf16:$b)), f32:$c)),
(INT_NVVM_MIXED_FMA_rn_f32_bf16 B16:$a, B16:$b, B32:$c)>;
}
@@ -2456,7 +2456,7 @@ foreach ty = [F16RT, F16X2RT, BF16RT, BF16X2RT] in {
foreach relu = ["", "_relu"] in {
defvar Intr = !cast<Intrinsic>("int_nvvm_fma_rn_oob" # relu);
defvar suffix = !subst("_", ".", relu # "_" # ty.PtxType);
- def INT_NVVM_FMA_OOB # relu # ty.PtxType :
+ def INT_NVVM_FMA_OOB # relu # ty.PtxType :
BasicNVPTXInst<(outs ty.RC:$dst), (ins ty.RC:$a, ty.RC:$b, ty.RC:$c),
"fma.rn.oob" # suffix,
[(set ty.Ty:$dst, (Intr ty.Ty:$a, ty.Ty:$b, ty.Ty:$c))]>,
@@ -2726,7 +2726,7 @@ let Predicates = [hasFP16x2MixedPrecisionSupport] in
// These nodes are created by combineFAddWithNeg.
def sub_rn_sat : SDNode<"NVPTXISD::SUB_RN_SAT", SDTFPBinOp>;
-def sub_rn_ftz_sat :
+def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
foreach rnd = FPRoundingModes in
@@ -3167,7 +3167,7 @@ let Predicates = [hasS2F6X2ConversionSupport] in {
(CVT_bf16x2_s2f6x2_scale $a, $scale, CvtRN)>;
def : Pat<(int_nvvm_s2f6x2_to_bf16x2_rn_relu_scale_n2_ue8m0 i16:$a, i16:$scale),
(CVT_bf16x2_s2f6x2_scale $a, $scale, CvtRN_RELU)>;
-
+
def : Pat<(int_nvvm_s2f6x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0 i16:$a, i16:$scale),
(CVT_bf16x2_s2f6x2_sf_scale $a, $scale, CvtRN)>;
def : Pat<(int_nvvm_s2f6x2_to_bf16x2_rn_relu_satfinite_scale_n2_ue8m0 i16:$a, i16:$scale),
@@ -3301,12 +3301,12 @@ let Predicates = [hasNarrowFPConversionSupport] in {
let Predicates = [hasRubinFamilySupport] in {
def : Pat<(int_nvvm_ue5m3x2_to_f16x2_rn i16:$a),
(CVT_f16x2_ue5m3x2 $a, CvtRN)>;
-
+
def : Pat<(int_nvvm_ue5m3x2_to_bf16x2_rn i16:$a),
(CVT_bf16x2_ue5m3x2 $a, CvtRN)>;
def : Pat<(int_nvvm_ue5m3x2_to_bf16x2_rn_satfinite i16:$a),
(CVT_bf16x2_ue5m3x2 $a, CvtRN_SATFINITE)>;
-
+
def : Pat<(int_nvvm_ue5m3x2_to_bf16x2_rn_scale_n2_ue8m0 i16:$a, i16:$b),
(CVT_bf16x2_ue5m3x2_scale_n2 $a, $b, CvtRN)>;
def : Pat<(int_nvvm_ue5m3x2_to_bf16x2_rn_satfinite_scale_n2_ue8m0 i16:$a, i16:$b),
@@ -3349,27 +3349,27 @@ let Predicates = [hasRubinFamilySupport] in {
} // let Predicates = [hasRubinFamilySupport]
def SDT_CVT_F32X4_TO_FPX4_RS_VEC :
- SDTypeProfile<1, 6, [SDTCisVec<0>, SDTCisFP<1>, SDTCisFP<2>, SDTCisFP<3>,
+ SDTypeProfile<1, 6, [SDTCisVec<0>, SDTCisFP<1>, SDTCisFP<2>, SDTCisFP<3>,
SDTCisFP<4>, SDTCisInt<5>, SDTCisInt<6>]>;
def SDT_CVT_F32X4_TO_FPX4_RS_INT :
- SDTypeProfile<1, 6, [SDTCisInt<0>, SDTCisFP<1>, SDTCisFP<2>, SDTCisFP<3>,
+ SDTypeProfile<1, 6, [SDTCisInt<0>, SDTCisFP<1>, SDTCisFP<2>, SDTCisFP<3>,
SDTCisFP<4>, SDTCisInt<5>, SDTCisInt<6>]>;
class CVT_F32X4_TO_FPX4_RS_SF_NODE<string FPName, SDTypeProfile SDT> :
SDNode<"NVPTXISD::CVT_" # FPName # "X4_F32X4_RS_SF", SDT, []>;
-
+
multiclass CVT_F32X4_TO_FPX4_RS_SF_VEC<string FPName, VTVec RetTy> {
def : Pat<(RetTy (CVT_F32X4_TO_FPX4_RS_SF_NODE<!toupper(FPName),
SDT_CVT_F32X4_TO_FPX4_RS_VEC>
f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS)),
- (!cast<NVPTXInst>("CVT_" # FPName # "x4_f32x4_rs_sf")
+ (!cast<NVPTXInst>("CVT_" # FPName # "x4_f32x4_rs_sf")
$f1, $f2, $f3, $f4, $rbits, CvtRS)>;
-
- def : Pat<(RetTy (CVT_F32X4_TO_FPX4_RS_SF_NODE<!toupper(FPName),
+
+ def : Pat<(RetTy (CVT_F32X4_TO_FPX4_RS_SF_NODE<!toupper(FPName),
SDT_CVT_F32X4_TO_FPX4_RS_VEC>
f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS_RELU)),
- (!cast<NVPTXInst>("CVT_" # FPName # "x4_f32x4_rs_sf")
+ (!cast<NVPTXInst>("CVT_" # FPName # "x4_f32x4_rs_sf")
$f1, $f2, $f3, $f4, $rbits, CvtRS_RELU)>;
}
@@ -3384,11 +3384,11 @@ defm : CVT_F32X4_TO_FPX4_RS_SF_VEC<"e2m3", v4i8>;
defm : CVT_F32X4_TO_FPX4_RS_SF_VEC<"e3m2", v4i8>;
// FP4x4 conversions
-def : Pat<(i16 (CVT_F32X4_TO_FPX4_RS_SF_NODE<"E2M1",
+def : Pat<(i16 (CVT_F32X4_TO_FPX4_RS_SF_NODE<"E2M1",
SDT_CVT_F32X4_TO_FPX4_RS_INT>
f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS)),
(CVT_e2m1x4_f32x4_rs_sf $f1, $f2, $f3, $f4, $rbits, CvtRS)>;
-def : Pat<(i16 (CVT_F32X4_TO_FPX4_RS_SF_NODE<"E2M1",
+def : Pat<(i16 (CVT_F32X4_TO_FPX4_RS_SF_NODE<"E2M1",
SDT_CVT_F32X4_TO_FPX4_RS_INT>
f32:$f1, f32:$f2, f32:$f3, f32:$f4, i32:$rbits, CvtRS_RELU)),
(CVT_e2m1x4_f32x4_rs_sf $f1, $f2, $f3, $f4, $rbits, CvtRS_RELU)>;
@@ -5342,7 +5342,7 @@ foreach op = ["clamp", "trap", "zero"] in {
//-----------------------------------
let IsSurfTexQuery = true in {
- foreach query = ["channel_order", "channel_data_type", "width", "height",
+ foreach query = ["channel_order", "channel_data_type", "width", "height",
"depth", "array_size", "num_samples", "num_mipmap_levels"] in {
def TXQ_ # !toupper(query) # _R
: NVPTXInst<(outs B32:$d), (ins B64:$a),
@@ -5779,10 +5779,6 @@ class PTX_READ_SREG_R64<string regname, Intrinsic intop, list<Predicate> Preds=[
[(set i64:$d, (intop))]>,
Requires<Preds>;
-// Reads of immutable special registers are single moves, so they are cheap to
-// recompute: only CSE them locally or from an immediate predecessor, instead
-// of extending their live ranges across blocks (MachineCSE heuristic #1).
-// Volatile (clock) and memory-modeled (pm) reads below opt back out.
let isAsCheapAsAMove = true in
class PTX_READ_SREG_R32<string regname, Intrinsic intop, list<Predicate> Preds=[]>
: NVPTXInst<(outs B32:$d), (ins),
@@ -5850,7 +5846,6 @@ def INT_PTX_SREG_LANEMASK_GE :
def INT_PTX_SREG_LANEMASK_GT :
PTX_READ_SREG_R32<"lanemask_gt", int_nvvm_read_ptx_sreg_lanemask_gt>;
-// Clock reads are volatile; keep them out of the cheap-as-move set.
let hasSideEffects = 1, isAsCheapAsAMove = false in {
def SREG_CLOCK : PTX_READ_SREG_R32<"clock", int_nvvm_read_ptx_sreg_clock>;
def SREG_CLOCK64 : PTX_READ_SREG_R64<"clock64", int_nvvm_read_ptx_sreg_clock64>;
@@ -5862,8 +5857,6 @@ def: Pat <(i64 (readcyclecounter)), (SREG_CLOCK64)>;
def: Pat <(i64 (readsteadycounter)), (SREG_GLOBALTIMER)>;
def: Pat <(i32 (readsteadycounter)), (SREG_GLOBALTIMER_LO)>;
-// Performance-monitor reads are modeled with memory effects; keep them out of
-// the cheap-as-move set.
let isAsCheapAsAMove = false in {
def INT_PTX_SREG_PM0 : PTX_READ_SREG_R32<"pm0", int_nvvm_read_ptx_sreg_pm0>;
def INT_PTX_SREG_PM1 : PTX_READ_SREG_R32<"pm1", int_nvvm_read_ptx_sreg_pm1>;
@@ -7067,8 +7060,8 @@ class ST_ASYNC_MMIO_SYS<string type, NVPTXRegClass reg> :
"st.async.mmio.release.sys.global." # type>;
class ST_ASYNC_RELEASE_B8_BODY<string instr_prefix, string prefix> {
- // The minimum supported register size to pass in arguments is 16-bits, so
- // the 8-bit value is passed in using a 16-bit register and truncated to
+ // The minimum supported register size to pass in arguments is 16-bits, so
+ // the 8-bit value is passed in using a 16-bit register and truncated to
// 8-bits with a cvt instruction.
string body =
"{{ \n\t"
@@ -7722,19 +7715,19 @@ foreach space = ["tensor", "shared"] in {
// tensormap.replace Instructions
//
-class TensormapReplaceInst_2<string state_space, string field_name,
+class TensormapReplaceInst_2<string state_space, string field_name,
string regclass_name, NVPTXRegClass val_RC, ValueType ValTy, Intrinsic Intrin,
NVPTXAddressSpace as> :
- BasicNVPTXInst<(outs),
- (ins ADDR:$addr, val_RC:$val),
+ BasicNVPTXInst<(outs),
+ (ins ADDR:$addr, val_RC:$val),
"tensormap.replace.tile." # field_name # "." # state_space # ".b1024." # regclass_name,
[(IntrinsicInAS<Intrin, as> addr:$addr, ValTy:$val)]>;
-class TensormapReplaceInst_3<string state_space, string field_name,
+class TensormapReplaceInst_3<string state_space, string field_name,
string regclass_name, NVPTXRegClass val_RC, ValueType ValTy, Intrinsic Intrin,
NVPTXAddressSpace as> :
- BasicNVPTXInst<(outs),
- (ins ADDR:$addr, B32:$ord, val_RC:$val),
+ BasicNVPTXInst<(outs),
+ (ins ADDR:$addr, B32:$ord, val_RC:$val),
"tensormap.replace.tile." # field_name # "." # state_space # ".b1024." # regclass_name,
[(IntrinsicInAS<Intrin, as> addr:$addr, i32:$ord, ValTy:$val)]>;
@@ -7742,39 +7735,39 @@ foreach ss = ["GLOBAL", "SHARED_CTA"] in {
defvar as = !if(!eq(ss, "GLOBAL"), AddrSpaceGlobal, AddrSpaceShared);
defvar ss_ptx = !tolower(!subst("_", "::", ss));
let Predicates = [hasTensormapReplaceSupport] in {
- def TENSORMAP_REPLACE_TILE_GLOBAL_ADDRESS_ # ss :
+ def TENSORMAP_REPLACE_TILE_GLOBAL_ADDRESS_ # ss :
TensormapReplaceInst_2<ss_ptx, "global_address", "b64", B64, i64,
int_nvvm_tensormap_replace_global_address, as>;
foreach field_name = ["INTERLEAVE_LAYOUT", "FILL_MODE", "RANK"] in {
defvar intrin = !cast<Intrinsic>("int_nvvm_tensormap_replace_" # !tolower(field_name));
- def TENSORMAP_REPLACE_TILE_ # field_name # _ # ss :
+ def TENSORMAP_REPLACE_TILE_ # field_name # _ # ss :
TensormapReplaceInst_2<ss_ptx, !tolower(field_name), "b32", B32, i32,
intrin, as>;
} // field_name
- def TENSORMAP_REPLACE_TILE_GLOBAL_STRIDE_ # ss :
- TensormapReplaceInst_3<ss_ptx, "global_stride", "b64", B64, i64,
+ def TENSORMAP_REPLACE_TILE_GLOBAL_STRIDE_ # ss :
+ TensormapReplaceInst_3<ss_ptx, "global_stride", "b64", B64, i64,
int_nvvm_tensormap_replace_global_stride, as>;
foreach field_name = ["BOX_DIM", "GLOBAL_DIM", "ELEMENT_STRIDE"] in {
defvar intrin = !cast<Intrinsic>("int_nvvm_tensormap_replace_" # !tolower(field_name));
- def TENSORMAP_REPLACE_TILE_ # field_name # _ # ss :
- TensormapReplaceInst_3<ss_ptx, !tolower(field_name), "b32", B32, i32,
+ def TENSORMAP_REPLACE_TILE_ # field_name # _ # ss :
+ TensormapReplaceInst_3<ss_ptx, !tolower(field_name), "b32", B32, i32,
intrin, as>;
} // field_name
} // hasTensormapReplaceSupport
- def TENSORMAP_REPLACE_TILE_ELEMTYPE_ # ss :
- TensormapReplaceInst_2<ss_ptx, "elemtype", "b32", B32, i32,
+ def TENSORMAP_REPLACE_TILE_ELEMTYPE_ # ss :
+ TensormapReplaceInst_2<ss_ptx, "elemtype", "b32", B32, i32,
int_nvvm_tensormap_replace_elemtype, as>;
- def TENSORMAP_REPLACE_SWIZZLE_ATOMICITY_ # ss :
- TensormapReplaceInst_2<ss_ptx, "swizzle_atomicity", "b32", B32, i32,
+ def TENSORMAP_REPLACE_SWIZZLE_ATOMICITY_ # ss :
+ TensormapReplaceInst_2<ss_ptx, "swizzle_atomicity", "b32", B32, i32,
int_nvvm_tensormap_replace_swizzle_atomicity, as>,
Requires<[hasTensormapReplaceSwizzleAtomicitySupport]>;
- def TENSORMAP_REPLACE_SWIZZLE_MODE_ # ss :
- TensormapReplaceInst_2<ss_ptx, "swizzle_mode", "b32", B32, i32,
+ def TENSORMAP_REPLACE_SWIZZLE_MODE_ # ss :
+ TensormapReplaceInst_2<ss_ptx, "swizzle_mode", "b32", B32, i32,
int_nvvm_tensormap_replace_swizzle_mode, as>;
} // state_space
More information about the llvm-commits
mailing list