[llvm] AMDGPU/GlobalISel: Implement RegBankLegalizeRules for amdgcn_rsq and amdgcn_rsq_clamp. (PR #187672)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 30 05:48:49 PDT 2026
https://github.com/anjenner updated https://github.com/llvm/llvm-project/pull/187672
>From f41d3f996d4f6bb3fd91219e3894c4861863c6b8 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Fri, 20 Mar 2026 06:35:53 -0400
Subject: [PATCH 1/2] AMDGPU/GlobalISel: Implement RegBankLegalizeRules for
amdgcn_rsq and amdgcn_rsq_clamp.
---
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 8 +-
.../AMDGPU/AMDGPUPreLegalizerCombiner.cpp | 42 +++++++++
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 8 ++
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 92 ++++++++-----------
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll | 4 +-
5 files changed, 95 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 7f00eadbf3f3f..5563e1d8dd5f2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -60,6 +60,12 @@ def clamp_i64_to_i16 : GICombineRule<
[{ return matchClampI64ToI16(*${clamp_i64_to_i16}, MRI, MF, ${matchinfo}); }]),
(apply [{ applyClampI64ToI16(*${clamp_i64_to_i16}, ${matchinfo}); }])>;
+def intrinsic_impdef : GICombineRule<
+ (defs root:$intrinsic, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_INTRINSIC):$intrinsic,
+ [{ return matchIntrinsicImpDef(*${intrinsic}, ${matchinfo}); }]),
+ (apply [{ Helper.replaceSingleDefInstWithReg(*${intrinsic}, ${matchinfo}); }])>;
+
def med3_matchdata : GIDefMatchData<"Med3MatchInfo">;
def int_minmax_to_med3 : GICombineRule<
@@ -218,7 +224,7 @@ def gfx8_combines : GICombineGroup<[expand_promoted_fmed3]>;
def AMDGPUPreLegalizerCombiner: GICombiner<
"AMDGPUPreLegalizerCombinerImpl",
[all_combines, combine_fmul_with_select_to_fldexp, clamp_i64_to_i16,
- foldable_fneg, combine_shuffle_vector_to_build_vector,
+ foldable_fneg, intrinsic_impdef, combine_shuffle_vector_to_build_vector,
binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
index 08f78ef820654..2ecd47e9f4886 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
@@ -22,9 +22,11 @@
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Target/TargetMachine.h"
#define GET_GICOMBINER_DEPS
@@ -73,6 +75,8 @@ class AMDGPUPreLegalizerCombinerImpl : public Combiner {
void applyClampI64ToI16(MachineInstr &MI,
const ClampI64ToI16MatchInfo &MatchInfo) const;
+ bool matchIntrinsicImpDef(MachineInstr &MI, Register &MatchInfo) const;
+
private:
#define GET_GICOMBINER_CLASS_MEMBERS
#define AMDGPUSubtarget GCNSubtarget
@@ -206,6 +210,44 @@ void AMDGPUPreLegalizerCombinerImpl::applyClampI64ToI16(
MI.eraseFromParent();
}
+// Match G_INTRINSIC instructions with a G_IMPLICIT_DEF (poison) operand.
+// For certain intrinsics like amdgcn.rsq, amdgcn.sqrt, amdgcn.tanh,
+// if the operand is poison/undef, we can replace the whole intrinsic
+// with poison.
+bool AMDGPUPreLegalizerCombinerImpl::matchIntrinsicImpDef(
+ MachineInstr &MI, Register &MatchInfo) const {
+ if (MI.getOpcode() != TargetOpcode::G_INTRINSIC)
+ return false;
+
+ // Get the intrinsic ID
+ Intrinsic::ID IID = cast<GIntrinsic>(MI).getIntrinsicID();
+
+ // Check if this is one of the intrinsics that propagates poison
+ switch (IID) {
+ case Intrinsic::amdgcn_sqrt:
+ case Intrinsic::amdgcn_rsq:
+ case Intrinsic::amdgcn_tanh:
+ break;
+ default:
+ return false;
+ }
+
+ // For G_INTRINSIC: operand 0 is dest, operand 1 is intrinsic ID,
+ // operand 2+ are the actual arguments.
+ // Check if the first argument (operand 2) is G_IMPLICIT_DEF
+ if (MI.getNumOperands() < 3)
+ return false;
+
+ Register SrcReg = MI.getOperand(2).getReg();
+ MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+ if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
+ return false;
+
+ // Return the poison value to replace with
+ MatchInfo = SrcReg;
+ return true;
+}
+
// Pass boilerplate
// ================
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 8f9f9cc3f2663..dcfe9c16130e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1715,6 +1715,14 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
+ addRulesForIOpcs({amdgcn_rsq, amdgcn_rsq_clamp}, Standard)
+ .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}})
+ .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
+ .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}})
+ .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
+ .Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}})
+ .Div(S64, {{Vgpr64}, {IntrId, Vgpr64}});
+
addRulesForIOpcs({amdgcn_mul_u24, amdgcn_mul_i24}, Standard)
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index efb6b20d6e730..3e1dee32911a8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
declare half @llvm.amdgcn.rsq.f16(half %a)
@@ -11,19 +11,17 @@ define amdgpu_kernel void @rsq_f16(
; GCN-LABEL: rsq_f16:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s10, s6
-; GCN-NEXT: s_mov_b32 s11, s7
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s8, s2
-; GCN-NEXT: s_mov_b32 s9, s3
-; GCN-NEXT: buffer_load_ushort v0, off, s[8:11], 0
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GCN-NEXT: buffer_load_ushort v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_rsq_f16_e32 v0, v0
-; GCN-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT: v_readfirstlane_b32 s2, v0
+; GCN-NEXT: v_rsq_f16_e32 v0, s2
+; GCN-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GCN-NEXT: s_nop 2
+; GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: rsq_f16:
@@ -31,17 +29,13 @@ define amdgpu_kernel void @rsq_f16(
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
+; GFX11-TRUE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-TRUE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: rsq_f16:
@@ -49,53 +43,39 @@ define amdgpu_kernel void @rsq_f16(
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
+; GFX11-FAKE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[4:7], 0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
-; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GFX11-FAKE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_endpgm
;
; GFX12-TRUE16-LABEL: rsq_f16:
; GFX12-TRUE16: ; %bb.0: ; %entry
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: rsq_f16:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
-; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
index f99fe71ac6a4d..08a494716914f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
declare float @llvm.amdgcn.rsq.f32(float) #0
declare double @llvm.amdgcn.rsq.f64(double) #0
>From c261ba82b885ad9f146247f329e1f71f308c7297 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Thu, 30 Apr 2026 08:18:10 -0400
Subject: [PATCH 2/2] Undo combiner changes and improve testing.
---
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 8 +-
.../AMDGPU/AMDGPUPreLegalizerCombiner.cpp | 42 ----
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll | 45 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 238 ++++++++++++------
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll | 12 +-
5 files changed, 207 insertions(+), 138 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 5563e1d8dd5f2..7f00eadbf3f3f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -60,12 +60,6 @@ def clamp_i64_to_i16 : GICombineRule<
[{ return matchClampI64ToI16(*${clamp_i64_to_i16}, MRI, MF, ${matchinfo}); }]),
(apply [{ applyClampI64ToI16(*${clamp_i64_to_i16}, ${matchinfo}); }])>;
-def intrinsic_impdef : GICombineRule<
- (defs root:$intrinsic, register_matchinfo:$matchinfo),
- (match (wip_match_opcode G_INTRINSIC):$intrinsic,
- [{ return matchIntrinsicImpDef(*${intrinsic}, ${matchinfo}); }]),
- (apply [{ Helper.replaceSingleDefInstWithReg(*${intrinsic}, ${matchinfo}); }])>;
-
def med3_matchdata : GIDefMatchData<"Med3MatchInfo">;
def int_minmax_to_med3 : GICombineRule<
@@ -224,7 +218,7 @@ def gfx8_combines : GICombineGroup<[expand_promoted_fmed3]>;
def AMDGPUPreLegalizerCombiner: GICombiner<
"AMDGPUPreLegalizerCombinerImpl",
[all_combines, combine_fmul_with_select_to_fldexp, clamp_i64_to_i16,
- foldable_fneg, intrinsic_impdef, combine_shuffle_vector_to_build_vector,
+ foldable_fneg, combine_shuffle_vector_to_build_vector,
binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
index 2ecd47e9f4886..08f78ef820654 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
@@ -22,11 +22,9 @@
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
-#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Target/TargetMachine.h"
#define GET_GICOMBINER_DEPS
@@ -75,8 +73,6 @@ class AMDGPUPreLegalizerCombinerImpl : public Combiner {
void applyClampI64ToI16(MachineInstr &MI,
const ClampI64ToI16MatchInfo &MatchInfo) const;
- bool matchIntrinsicImpDef(MachineInstr &MI, Register &MatchInfo) const;
-
private:
#define GET_GICOMBINER_CLASS_MEMBERS
#define AMDGPUSubtarget GCNSubtarget
@@ -210,44 +206,6 @@ void AMDGPUPreLegalizerCombinerImpl::applyClampI64ToI16(
MI.eraseFromParent();
}
-// Match G_INTRINSIC instructions with a G_IMPLICIT_DEF (poison) operand.
-// For certain intrinsics like amdgcn.rsq, amdgcn.sqrt, amdgcn.tanh,
-// if the operand is poison/undef, we can replace the whole intrinsic
-// with poison.
-bool AMDGPUPreLegalizerCombinerImpl::matchIntrinsicImpDef(
- MachineInstr &MI, Register &MatchInfo) const {
- if (MI.getOpcode() != TargetOpcode::G_INTRINSIC)
- return false;
-
- // Get the intrinsic ID
- Intrinsic::ID IID = cast<GIntrinsic>(MI).getIntrinsicID();
-
- // Check if this is one of the intrinsics that propagates poison
- switch (IID) {
- case Intrinsic::amdgcn_sqrt:
- case Intrinsic::amdgcn_rsq:
- case Intrinsic::amdgcn_tanh:
- break;
- default:
- return false;
- }
-
- // For G_INTRINSIC: operand 0 is dest, operand 1 is intrinsic ID,
- // operand 2+ are the actual arguments.
- // Check if the first argument (operand 2) is G_IMPLICIT_DEF
- if (MI.getNumOperands() < 3)
- return false;
-
- Register SrcReg = MI.getOperand(2).getReg();
- MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
- if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
- return false;
-
- // Return the poison value to replace with
- MatchInfo = SrcReg;
- return true;
-}
-
// Pass boilerplate
// ================
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
index e879fb2c399f0..aa145f6969334 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
@@ -1,5 +1,7 @@
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI,SDAG,FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI-SDAG,SDAG,FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefixes=SI,SI-GISEL,FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI-GISEL,FUNC %s
declare float @llvm.amdgcn.rsq.clamp.f32(float) #1
declare double @llvm.amdgcn.rsq.clamp.f64(double) #1
@@ -7,11 +9,15 @@ declare double @llvm.amdgcn.rsq.clamp.f64(double) #1
; FUNC-LABEL: {{^}}rsq_clamp_f32:
; SI: v_rsq_clamp_f32_e32
-; VI: s_load_dword [[SRC:s[0-9]+]]
-; VI-DAG: v_rsq_f32_e32 [[RSQ:v[0-9]+]], [[SRC]]
-; VI-DAG: v_min_f32_e32 [[MIN:v[0-9]+]], 0x7f7fffff, [[RSQ]]
-; VI: v_max_f32_e32 [[RESULT:v[0-9]+]], 0xff7fffff, [[MIN]]
-; VI: buffer_store_dword [[RESULT]]
+; VI-SDAG: s_load_dword [[SRC:s[0-9]+]]
+; VI-SDAG-DAG: v_rsq_f32_e32 [[RSQ:v[0-9]+]], [[SRC]]
+; VI-SDAG-DAG: v_min_f32_e32 [[MIN:v[0-9]+]], 0x7f7fffff, [[RSQ]]
+; VI-SDAG: v_max_f32_e32 [[RESULT:v[0-9]+]], 0xff7fffff, [[MIN]]
+; VI-SDAG: buffer_store_dword [[RESULT]]
+
+; VI-GISEL: v_rsq_f32_e32
+; VI-GISEL: v_min_f32_e32
+; VI-GISEL: v_max_f32_e32
define amdgpu_kernel void @rsq_clamp_f32(ptr addrspace(1) %out, float %src) #0 {
%rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %src)
store float %rsq_clamp, ptr addrspace(1) %out
@@ -23,21 +29,30 @@ define amdgpu_kernel void @rsq_clamp_f32(ptr addrspace(1) %out, float %src) #0 {
; SI: v_rsq_clamp_f64_e32
; TODO: this constant should be folded:
-; VI-DAG: s_mov_b32 [[NEG1:s[0-9]+]], -1
-; VI-DAG: s_mov_b32 s[[LOW1:[0-9]+]], [[NEG1]]
-; VI-DAG: s_mov_b32 s[[HIGH1:[0-9]+]], 0x7fefffff
-; VI-DAG: s_mov_b32 s[[HIGH2:[0-9]+]], 0xffefffff
-; VI-DAG: v_rsq_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], s[{{[0-9]+:[0-9]+}}
-; VI-DAG: v_min_f64 v[0:1], [[RSQ]], s[[[LOW1]]:[[HIGH1]]]
-; VI-DAG: v_max_f64 v[0:1], v[0:1], s[[[LOW1]]:[[HIGH2]]]
+; VI-SDAG-DAG: s_mov_b32 [[NEG1:s[0-9]+]], -1
+; VI-SDAG-DAG: s_mov_b32 s[[LOW1:[0-9]+]], [[NEG1]]
+; VI-SDAG-DAG: s_mov_b32 s[[HIGH1:[0-9]+]], 0x7fefffff
+; VI-SDAG-DAG: s_mov_b32 s[[HIGH2:[0-9]+]], 0xffefffff
+; VI-SDAG-DAG: v_rsq_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], s[{{[0-9]+:[0-9]+}}
+; VI-SDAG-DAG: v_min_f64 v[0:1], [[RSQ]], s[[[LOW1]]:[[HIGH1]]]
+; VI-SDAG-DAG: v_max_f64 v[0:1], v[0:1], s[[[LOW1]]:[[HIGH2]]]
+
+; VI-GISEL: v_rsq_f64_e32
+; VI-GISEL: v_min_f64
+; VI-GISEL: v_max_f64
define amdgpu_kernel void @rsq_clamp_f64(ptr addrspace(1) %out, double %src) #0 {
%rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double %src)
store double %rsq_clamp, ptr addrspace(1) %out
ret void
}
+; With SDAG, poison folding happens in target specific combine. With GISel this
+; only happens as part of InstCombine so is not tested here.
+
; FUNC-LABEL: {{^}}rsq_clamp_undef_f32:
-; SI-NOT: v_rsq_clamp_f32
+; SDAG-NOT: v_rsq_clamp_f32
+; SI-GISEL: v_rsq_clamp_f32_e32
+; VI-GISEL: v_rsq_f32_e32
define amdgpu_kernel void @rsq_clamp_undef_f32(ptr addrspace(1) %out) #0 {
%rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float poison)
store float %rsq_clamp, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index 3e1dee32911a8..7ac8caf71af22 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -1,82 +1,177 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=SDAG-GCN -check-prefix=SDAG-VI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=SDAG-GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=SDAG-GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=SDAG-GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=SDAG-GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GISEL-GCN -check-prefix=GISEL-VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GISEL-GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GISEL-GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GISEL-GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GISEL-GFX12-FAKE16 %s
declare half @llvm.amdgcn.rsq.f16(half %a)
define amdgpu_kernel void @rsq_f16(
-; GCN-LABEL: rsq_f16:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: buffer_load_ushort v0, off, s[4:7], 0
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_readfirstlane_b32 s2, v0
-; GCN-NEXT: v_rsq_f16_e32 v0, s2
-; GCN-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; SDAG-GCN-LABEL: rsq_f16:
+; SDAG-GCN: ; %bb.0: ; %entry
+; SDAG-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GCN-NEXT: s_mov_b32 s7, 0xf000
+; SDAG-GCN-NEXT: s_mov_b32 s6, -1
+; SDAG-GCN-NEXT: s_mov_b32 s10, s6
+; SDAG-GCN-NEXT: s_mov_b32 s11, s7
+; SDAG-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GCN-NEXT: s_mov_b32 s8, s2
+; SDAG-GCN-NEXT: s_mov_b32 s9, s3
+; SDAG-GCN-NEXT: buffer_load_ushort v0, off, s[8:11], 0
+; SDAG-GCN-NEXT: s_mov_b32 s4, s0
+; SDAG-GCN-NEXT: s_mov_b32 s5, s1
+; SDAG-GCN-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GCN-NEXT: v_rsq_f16_e32 v0, v0
+; SDAG-GCN-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SDAG-GCN-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: rsq_f16:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX11-TRUE16-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-TRUE16-NEXT: s_endpgm
+; SDAG-GFX11-TRUE16-LABEL: rsq_f16:
+; SDAG-GFX11-TRUE16: ; %bb.0: ; %entry
+; SDAG-GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
+; SDAG-GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; SDAG-GFX11-TRUE16-NEXT: s_endpgm
;
-; GFX11-FAKE16-LABEL: rsq_f16:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[4:7], 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
-; GFX11-FAKE16-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-FAKE16-NEXT: s_endpgm
+; SDAG-GFX11-FAKE16-LABEL: rsq_f16:
+; SDAG-GFX11-FAKE16: ; %bb.0: ; %entry
+; SDAG-GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], 0
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
+; SDAG-GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; SDAG-GFX11-FAKE16-NEXT: s_endpgm
;
-; GFX12-TRUE16-LABEL: rsq_f16:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-TRUE16-NEXT: s_endpgm
+; SDAG-GFX12-TRUE16-LABEL: rsq_f16:
+; SDAG-GFX12-TRUE16: ; %bb.0: ; %entry
+; SDAG-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
+; SDAG-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-TRUE16-NEXT: s_endpgm
;
-; GFX12-FAKE16-LABEL: rsq_f16:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-FAKE16-NEXT: s_endpgm
+; SDAG-GFX12-FAKE16-LABEL: rsq_f16:
+; SDAG-GFX12-FAKE16: ; %bb.0: ; %entry
+; SDAG-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
+; SDAG-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-FAKE16-NEXT: s_endpgm
+;
+; GISEL-GCN-LABEL: rsq_f16:
+; GISEL-GCN: ; %bb.0: ; %entry
+; GISEL-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-GCN-NEXT: s_mov_b32 s6, -1
+; GISEL-GCN-NEXT: s_mov_b32 s7, 0xf000
+; GISEL-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GISEL-GCN-NEXT: buffer_load_ushort v0, off, s[4:7], 0
+; GISEL-GCN-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GCN-NEXT: v_readfirstlane_b32 s2, v0
+; GISEL-GCN-NEXT: v_rsq_f16_e32 v0, s2
+; GISEL-GCN-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GISEL-GCN-NEXT: s_nop 2
+; GISEL-GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GISEL-GCN-NEXT: s_endpgm
+;
+; GISEL-GFX11-TRUE16-LABEL: rsq_f16:
+; GISEL-GFX11-TRUE16: ; %bb.0: ; %entry
+; GISEL-GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GISEL-GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GISEL-GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
+; GISEL-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
+; GISEL-GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GISEL-GFX11-TRUE16-NEXT: s_endpgm
+;
+; GISEL-GFX11-FAKE16-LABEL: rsq_f16:
+; GISEL-GFX11-FAKE16: ; %bb.0: ; %entry
+; GISEL-GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GISEL-GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GISEL-GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[4:7], 0
+; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GISEL-GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GISEL-GFX11-FAKE16-NEXT: s_endpgm
+;
+; GISEL-GFX12-TRUE16-LABEL: rsq_f16:
+; GISEL-GFX12-TRUE16: ; %bb.0: ; %entry
+; GISEL-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GISEL-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, s2
+; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; GISEL-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GISEL-GFX12-TRUE16-NEXT: s_endpgm
+;
+; GISEL-GFX12-FAKE16-LABEL: rsq_f16:
+; GISEL-GFX12-FAKE16: ; %bb.0: ; %entry
+; GISEL-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; GISEL-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GISEL-GFX12-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
entry:
@@ -86,4 +181,5 @@ entry:
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; VI: {{.*}}
+; GISEL-VI: {{.*}}
+; SDAG-VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
index 08a494716914f..3b3e07586f057 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
@@ -1,5 +1,7 @@
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI,SDAG,FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=SI,SDAG,FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefixes=SI,GISEL,FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=SI,GISEL,FUNC %s
declare float @llvm.amdgcn.rsq.f32(float) #0
declare double @llvm.amdgcn.rsq.f64(double) #0
@@ -54,8 +56,12 @@ define amdgpu_kernel void @rsq_f64_constant_100.0(ptr addrspace(1) %out) #1 {
ret void
}
+; With SDAG, poison folding happens in target specific combine. With GISel this
+; only happens as part of InstCombine so is not tested here.
+
; FUNC-LABEL: {{^}}rsq_undef_f32:
-; SI-NOT: v_rsq_f32
+; SDAG-NOT: v_rsq_f32
+; GISEL: v_rsq_f32
define amdgpu_kernel void @rsq_undef_f32(ptr addrspace(1) %out) #1 {
%rsq = call float @llvm.amdgcn.rsq.f32(float poison)
store float %rsq, ptr addrspace(1) %out, align 4
More information about the llvm-commits
mailing list