[llvm] AMDGPU/GlobalISel: Implement RegBankLegalizeRules for amdgcn_rsq and amdgcn_rsq_clamp. (PR #187672)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 10:01:36 PDT 2026
https://github.com/anjenner updated https://github.com/llvm/llvm-project/pull/187672
>From 2b0e01eaa67a662ebf11a65a05b2f25b94721be8 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Fri, 20 Mar 2026 06:35:53 -0400
Subject: [PATCH 1/4] AMDGPU/GlobalISel: Implement RegBankLegalizeRules for
amdgcn_rsq and amdgcn_rsq_clamp.
---
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 8 +-
.../AMDGPU/AMDGPUPreLegalizerCombiner.cpp | 42 +++++++++
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 8 ++
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 92 ++++++++-----------
4 files changed, 93 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 7f00eadbf3f3f..5563e1d8dd5f2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -60,6 +60,12 @@ def clamp_i64_to_i16 : GICombineRule<
[{ return matchClampI64ToI16(*${clamp_i64_to_i16}, MRI, MF, ${matchinfo}); }]),
(apply [{ applyClampI64ToI16(*${clamp_i64_to_i16}, ${matchinfo}); }])>;
+def intrinsic_impdef : GICombineRule<
+ (defs root:$intrinsic, register_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_INTRINSIC):$intrinsic,
+ [{ return matchIntrinsicImpDef(*${intrinsic}, ${matchinfo}); }]),
+ (apply [{ Helper.replaceSingleDefInstWithReg(*${intrinsic}, ${matchinfo}); }])>;
+
def med3_matchdata : GIDefMatchData<"Med3MatchInfo">;
def int_minmax_to_med3 : GICombineRule<
@@ -218,7 +224,7 @@ def gfx8_combines : GICombineGroup<[expand_promoted_fmed3]>;
def AMDGPUPreLegalizerCombiner: GICombiner<
"AMDGPUPreLegalizerCombinerImpl",
[all_combines, combine_fmul_with_select_to_fldexp, clamp_i64_to_i16,
- foldable_fneg, combine_shuffle_vector_to_build_vector,
+ foldable_fneg, intrinsic_impdef, combine_shuffle_vector_to_build_vector,
binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
index 08f78ef820654..2ecd47e9f4886 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
@@ -22,9 +22,11 @@
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Target/TargetMachine.h"
#define GET_GICOMBINER_DEPS
@@ -73,6 +75,8 @@ class AMDGPUPreLegalizerCombinerImpl : public Combiner {
void applyClampI64ToI16(MachineInstr &MI,
const ClampI64ToI16MatchInfo &MatchInfo) const;
+ bool matchIntrinsicImpDef(MachineInstr &MI, Register &MatchInfo) const;
+
private:
#define GET_GICOMBINER_CLASS_MEMBERS
#define AMDGPUSubtarget GCNSubtarget
@@ -206,6 +210,44 @@ void AMDGPUPreLegalizerCombinerImpl::applyClampI64ToI16(
MI.eraseFromParent();
}
+// Match G_INTRINSIC instructions with a G_IMPLICIT_DEF (poison) operand.
+// For certain intrinsics like amdgcn.rsq, amdgcn.sqrt, amdgcn.tanh,
+// if the operand is poison/undef, we can replace the whole intrinsic
+// with poison.
+bool AMDGPUPreLegalizerCombinerImpl::matchIntrinsicImpDef(
+ MachineInstr &MI, Register &MatchInfo) const {
+ if (MI.getOpcode() != TargetOpcode::G_INTRINSIC)
+ return false;
+
+ // Get the intrinsic ID
+ Intrinsic::ID IID = cast<GIntrinsic>(MI).getIntrinsicID();
+
+ // Check if this is one of the intrinsics that propagates poison
+ switch (IID) {
+ case Intrinsic::amdgcn_sqrt:
+ case Intrinsic::amdgcn_rsq:
+ case Intrinsic::amdgcn_tanh:
+ break;
+ default:
+ return false;
+ }
+
+ // For G_INTRINSIC: operand 0 is dest, operand 1 is intrinsic ID,
+ // operand 2+ are the actual arguments.
+ // Check if the first argument (operand 2) is G_IMPLICIT_DEF
+ if (MI.getNumOperands() < 3)
+ return false;
+
+ Register SrcReg = MI.getOperand(2).getReg();
+ MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
+ if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
+ return false;
+
+ // Return the poison value to replace with
+ MatchInfo = SrcReg;
+ return true;
+}
+
// Pass boilerplate
// ================
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 84f51c95da3fd..118d75dd6bbe3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1725,6 +1725,14 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
+ addRulesForIOpcs({amdgcn_rsq, amdgcn_rsq_clamp}, Standard)
+ .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr16}})
+ .Div(S16, {{Vgpr16}, {IntrId, Vgpr16}})
+ .Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32}})
+ .Div(S32, {{Vgpr32}, {IntrId, Vgpr32}})
+ .Uni(S64, {{UniInVgprS64}, {IntrId, Vgpr64}})
+ .Div(S64, {{Vgpr64}, {IntrId, Vgpr64}});
+
addRulesForIOpcs({amdgcn_mul_u24, amdgcn_mul_i24}, Standard)
.Uni(S32, {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index efb6b20d6e730..3e1dee32911a8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
declare half @llvm.amdgcn.rsq.f16(half %a)
@@ -11,19 +11,17 @@ define amdgpu_kernel void @rsq_f16(
; GCN-LABEL: rsq_f16:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s10, s6
-; GCN-NEXT: s_mov_b32 s11, s7
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s8, s2
-; GCN-NEXT: s_mov_b32 s9, s3
-; GCN-NEXT: buffer_load_ushort v0, off, s[8:11], 0
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GCN-NEXT: buffer_load_ushort v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_rsq_f16_e32 v0, v0
-; GCN-NEXT: buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT: v_readfirstlane_b32 s2, v0
+; GCN-NEXT: v_rsq_f16_e32 v0, s2
+; GCN-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GCN-NEXT: s_nop 2
+; GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: rsq_f16:
@@ -31,17 +29,13 @@ define amdgpu_kernel void @rsq_f16(
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
+; GFX11-TRUE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-TRUE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: rsq_f16:
@@ -49,53 +43,39 @@ define amdgpu_kernel void @rsq_f16(
; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX11-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX11-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
+; GFX11-FAKE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[4:7], 0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
-; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GFX11-FAKE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
; GFX11-FAKE16-NEXT: s_endpgm
;
; GFX12-TRUE16-LABEL: rsq_f16:
; GFX12-TRUE16: ; %bb.0: ; %entry
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: rsq_f16:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
-; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
>From a0766717e0394e142aeb8a95d18d6d2c8ea0de05 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Thu, 30 Apr 2026 08:18:10 -0400
Subject: [PATCH 2/4] Undo combiner changes and improve testing.
---
llvm/lib/Target/AMDGPU/AMDGPUCombine.td | 8 +-
.../AMDGPU/AMDGPUPreLegalizerCombiner.cpp | 42 ----
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll | 5 +
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 238 ++++++++++++------
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll | 8 +
5 files changed, 181 insertions(+), 120 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 5563e1d8dd5f2..7f00eadbf3f3f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -60,12 +60,6 @@ def clamp_i64_to_i16 : GICombineRule<
[{ return matchClampI64ToI16(*${clamp_i64_to_i16}, MRI, MF, ${matchinfo}); }]),
(apply [{ applyClampI64ToI16(*${clamp_i64_to_i16}, ${matchinfo}); }])>;
-def intrinsic_impdef : GICombineRule<
- (defs root:$intrinsic, register_matchinfo:$matchinfo),
- (match (wip_match_opcode G_INTRINSIC):$intrinsic,
- [{ return matchIntrinsicImpDef(*${intrinsic}, ${matchinfo}); }]),
- (apply [{ Helper.replaceSingleDefInstWithReg(*${intrinsic}, ${matchinfo}); }])>;
-
def med3_matchdata : GIDefMatchData<"Med3MatchInfo">;
def int_minmax_to_med3 : GICombineRule<
@@ -224,7 +218,7 @@ def gfx8_combines : GICombineGroup<[expand_promoted_fmed3]>;
def AMDGPUPreLegalizerCombiner: GICombiner<
"AMDGPUPreLegalizerCombinerImpl",
[all_combines, combine_fmul_with_select_to_fldexp, clamp_i64_to_i16,
- foldable_fneg, intrinsic_impdef, combine_shuffle_vector_to_build_vector,
+ foldable_fneg, combine_shuffle_vector_to_build_vector,
binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
index 2ecd47e9f4886..08f78ef820654 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreLegalizerCombiner.cpp
@@ -22,11 +22,9 @@
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
-#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Target/TargetMachine.h"
#define GET_GICOMBINER_DEPS
@@ -75,8 +73,6 @@ class AMDGPUPreLegalizerCombinerImpl : public Combiner {
void applyClampI64ToI16(MachineInstr &MI,
const ClampI64ToI16MatchInfo &MatchInfo) const;
- bool matchIntrinsicImpDef(MachineInstr &MI, Register &MatchInfo) const;
-
private:
#define GET_GICOMBINER_CLASS_MEMBERS
#define AMDGPUSubtarget GCNSubtarget
@@ -210,44 +206,6 @@ void AMDGPUPreLegalizerCombinerImpl::applyClampI64ToI16(
MI.eraseFromParent();
}
-// Match G_INTRINSIC instructions with a G_IMPLICIT_DEF (poison) operand.
-// For certain intrinsics like amdgcn.rsq, amdgcn.sqrt, amdgcn.tanh,
-// if the operand is poison/undef, we can replace the whole intrinsic
-// with poison.
-bool AMDGPUPreLegalizerCombinerImpl::matchIntrinsicImpDef(
- MachineInstr &MI, Register &MatchInfo) const {
- if (MI.getOpcode() != TargetOpcode::G_INTRINSIC)
- return false;
-
- // Get the intrinsic ID
- Intrinsic::ID IID = cast<GIntrinsic>(MI).getIntrinsicID();
-
- // Check if this is one of the intrinsics that propagates poison
- switch (IID) {
- case Intrinsic::amdgcn_sqrt:
- case Intrinsic::amdgcn_rsq:
- case Intrinsic::amdgcn_tanh:
- break;
- default:
- return false;
- }
-
- // For G_INTRINSIC: operand 0 is dest, operand 1 is intrinsic ID,
- // operand 2+ are the actual arguments.
- // Check if the first argument (operand 2) is G_IMPLICIT_DEF
- if (MI.getNumOperands() < 3)
- return false;
-
- Register SrcReg = MI.getOperand(2).getReg();
- MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
- if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
- return false;
-
- // Return the poison value to replace with
- MatchInfo = SrcReg;
- return true;
-}
-
// Pass boilerplate
// ================
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
index 0b0cf45b02f3e..e3f0559c738a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefixes=SI,SI-GISEL,FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga < %s | FileCheck -check-prefixes=VI-GISEL,FUNC %s
declare float @llvm.amdgcn.rsq.clamp.f32(float) #1
declare double @llvm.amdgcn.rsq.clamp.f64(double) #1
@@ -68,6 +70,9 @@ define amdgpu_kernel void @rsq_clamp_f64(ptr addrspace(1) %out, double %src) #0
ret void
}
+; With SDAG, poison folding happens in target specific combine. With GISel this
+; only happens as part of InstCombine so is not tested here.
+
define amdgpu_kernel void @rsq_clamp_undef_f32(ptr addrspace(1) %out) #0 {
; SI-LABEL: rsq_clamp_undef_f32:
; SI: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index 3e1dee32911a8..7ac8caf71af22 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -1,82 +1,177 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GFX12-TRUE16 %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=SDAG-GCN -check-prefix=SDAG-VI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=SDAG-GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=SDAG-GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=SDAG-GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=SDAG-GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -check-prefix=GISEL-GCN -check-prefix=GISEL-VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1100 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GISEL-GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1100 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GISEL-GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1200 -mattr=-flat-for-global,+real-true16 < %s | FileCheck -check-prefix=GISEL-GFX12-TRUE16 %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=gfx1200 -mattr=-flat-for-global,-real-true16 < %s | FileCheck -check-prefix=GISEL-GFX12-FAKE16 %s
declare half @llvm.amdgcn.rsq.f16(half %a)
define amdgpu_kernel void @rsq_f16(
-; GCN-LABEL: rsq_f16:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: buffer_load_ushort v0, off, s[4:7], 0
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_readfirstlane_b32 s2, v0
-; GCN-NEXT: v_rsq_f16_e32 v0, s2
-; GCN-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GCN-NEXT: s_nop 2
-; GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GCN-NEXT: s_endpgm
+; SDAG-GCN-LABEL: rsq_f16:
+; SDAG-GCN: ; %bb.0: ; %entry
+; SDAG-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GCN-NEXT: s_mov_b32 s7, 0xf000
+; SDAG-GCN-NEXT: s_mov_b32 s6, -1
+; SDAG-GCN-NEXT: s_mov_b32 s10, s6
+; SDAG-GCN-NEXT: s_mov_b32 s11, s7
+; SDAG-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GCN-NEXT: s_mov_b32 s8, s2
+; SDAG-GCN-NEXT: s_mov_b32 s9, s3
+; SDAG-GCN-NEXT: buffer_load_ushort v0, off, s[8:11], 0
+; SDAG-GCN-NEXT: s_mov_b32 s4, s0
+; SDAG-GCN-NEXT: s_mov_b32 s5, s1
+; SDAG-GCN-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GCN-NEXT: v_rsq_f16_e32 v0, v0
+; SDAG-GCN-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SDAG-GCN-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: rsq_f16:
-; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX11-TRUE16-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-TRUE16-NEXT: s_endpgm
+; SDAG-GFX11-TRUE16-LABEL: rsq_f16:
+; SDAG-GFX11-TRUE16: ; %bb.0: ; %entry
+; SDAG-GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0
+; SDAG-GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
+; SDAG-GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; SDAG-GFX11-TRUE16-NEXT: s_endpgm
;
-; GFX11-FAKE16-LABEL: rsq_f16:
-; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[4:7], 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
-; GFX11-FAKE16-NEXT: s_mov_b64 s[2:3], s[6:7]
-; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
-; GFX11-FAKE16-NEXT: s_endpgm
+; SDAG-GFX11-FAKE16-LABEL: rsq_f16:
+; SDAG-GFX11-FAKE16: ; %bb.0: ; %entry
+; SDAG-GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], 0
+; SDAG-GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; SDAG-GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
+; SDAG-GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; SDAG-GFX11-FAKE16-NEXT: s_endpgm
;
-; GFX12-TRUE16-LABEL: rsq_f16:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-TRUE16-NEXT: s_endpgm
+; SDAG-GFX12-TRUE16-LABEL: rsq_f16:
+; SDAG-GFX12-TRUE16: ; %bb.0: ; %entry
+; SDAG-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
+; SDAG-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-TRUE16-NEXT: s_endpgm
;
-; GFX12-FAKE16-LABEL: rsq_f16:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
-; GFX12-FAKE16-NEXT: s_endpgm
+; SDAG-GFX12-FAKE16-LABEL: rsq_f16:
+; SDAG-GFX12-FAKE16: ; %bb.0: ; %entry
+; SDAG-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
+; SDAG-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
+; SDAG-GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
+; SDAG-GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
+; SDAG-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-FAKE16-NEXT: s_endpgm
+;
+; GISEL-GCN-LABEL: rsq_f16:
+; GISEL-GCN: ; %bb.0: ; %entry
+; GISEL-GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-GCN-NEXT: s_mov_b32 s6, -1
+; GISEL-GCN-NEXT: s_mov_b32 s7, 0xf000
+; GISEL-GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GISEL-GCN-NEXT: buffer_load_ushort v0, off, s[4:7], 0
+; GISEL-GCN-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GCN-NEXT: v_readfirstlane_b32 s2, v0
+; GISEL-GCN-NEXT: v_rsq_f16_e32 v0, s2
+; GISEL-GCN-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GISEL-GCN-NEXT: s_nop 2
+; GISEL-GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GISEL-GCN-NEXT: s_endpgm
+;
+; GISEL-GFX11-TRUE16-LABEL: rsq_f16:
+; GISEL-GFX11-TRUE16: ; %bb.0: ; %entry
+; GISEL-GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GISEL-GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GISEL-GFX11-TRUE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GISEL-GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], 0
+; GISEL-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
+; GISEL-GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GISEL-GFX11-TRUE16-NEXT: s_endpgm
+;
+; GISEL-GFX11-FAKE16-LABEL: rsq_f16:
+; GISEL-GFX11-FAKE16: ; %bb.0: ; %entry
+; GISEL-GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GISEL-GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b64 s[4:5], s[2:3]
+; GISEL-GFX11-FAKE16-NEXT: buffer_load_u16 v0, off, s[4:7], 0
+; GISEL-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GISEL-GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
+; GISEL-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GISEL-GFX11-FAKE16-NEXT: s_mov_b64 s[2:3], s[6:7]
+; GISEL-GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GISEL-GFX11-FAKE16-NEXT: s_endpgm
+;
+; GISEL-GFX12-TRUE16-LABEL: rsq_f16:
+; GISEL-GFX12-TRUE16: ; %bb.0: ; %entry
+; GISEL-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GISEL-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, s2
+; GISEL-GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; GISEL-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GISEL-GFX12-TRUE16-NEXT: s_endpgm
+;
+; GISEL-GFX12-FAKE16-LABEL: rsq_f16:
+; GISEL-GFX12-FAKE16: ; %bb.0: ; %entry
+; GISEL-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GISEL-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GISEL-GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, s2
+; GISEL-GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; GISEL-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GISEL-GFX12-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
entry:
@@ -86,4 +181,5 @@ entry:
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; VI: {{.*}}
+; GISEL-VI: {{.*}}
+; SDAG-VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
index 1224953d15268..2aedeb2adbc15 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefixes=SI,GISEL,FUNC %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga < %s | FileCheck -check-prefixes=VI,GISEL,FUNC %s
declare float @llvm.amdgcn.rsq.f32(float) #0
declare double @llvm.amdgcn.rsq.f64(double) #0
@@ -161,6 +163,12 @@ define amdgpu_kernel void @rsq_f64_constant_100.0(ptr addrspace(1) %out) #1 {
ret void
}
+; With SDAG, poison folding happens in target specific combine. With GISel this
+; only happens as part of InstCombine so is not tested here.
+
+; FUNC-LABEL: {{^}}rsq_undef_f32:
+; SDAG-NOT: v_rsq_f32
+; GISEL: v_rsq_f32
define amdgpu_kernel void @rsq_undef_f32(ptr addrspace(1) %out) #1 {
; SI-LABEL: rsq_undef_f32:
; SI: ; %bb.0:
>From e9dc5a34a7f5405cc721e0de307f73b4b40d6ed5 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Tue, 5 May 2026 06:55:49 -0400
Subject: [PATCH 3/4] Remove -global-isel-abort=1.
---
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll | 8 ++++----
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll | 11 ++++-------
2 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
index e3f0559c738a3..b581e2aaf7d95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefixes=SI,SI-GISEL,FUNC %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga < %s | FileCheck -check-prefixes=VI-GISEL,FUNC %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select < %s | FileCheck -check-prefixes=SI-GISEL %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=tonga < %s | FileCheck -check-prefixes=VI-GISEL %s
declare float @llvm.amdgcn.rsq.clamp.f32(float) #1
declare double @llvm.amdgcn.rsq.clamp.f64(double) #1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
index 2aedeb2adbc15..7e795c1b6876b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 < %s | FileCheck -check-prefixes=SI,GISEL,FUNC %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mcpu=tonga < %s | FileCheck -check-prefixes=VI,GISEL,FUNC %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=VI %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select < %s | FileCheck -check-prefixes=SI-GISEL %s
+; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI-GISEL %s
declare float @llvm.amdgcn.rsq.f32(float) #0
declare double @llvm.amdgcn.rsq.f64(double) #0
@@ -166,9 +166,6 @@ define amdgpu_kernel void @rsq_f64_constant_100.0(ptr addrspace(1) %out) #1 {
; With SDAG, poison folding happens in target specific combine. With GISel this
; only happens as part of InstCombine so is not tested here.
-; FUNC-LABEL: {{^}}rsq_undef_f32:
-; SDAG-NOT: v_rsq_f32
-; GISEL: v_rsq_f32
define amdgpu_kernel void @rsq_undef_f32(ptr addrspace(1) %out) #1 {
; SI-LABEL: rsq_undef_f32:
; SI: ; %bb.0:
>From e7272655db8df96d5398f58552c96a36eb255b2c Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Tue, 5 May 2026 13:08:23 -0400
Subject: [PATCH 4/4] Fix up tests after precommit auto-gen
---
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll | 73 +++++++++
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll | 142 ++++++++++++++++++
2 files changed, 215 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
index b581e2aaf7d95..7c7d899f7de3e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
@@ -32,6 +32,31 @@ define amdgpu_kernel void @rsq_clamp_f32(ptr addrspace(1) %out, float %src) #0 {
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_clamp_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dword s3, s[4:5], 0xb
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_rsq_clamp_f32_e32 v0, s3
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_clamp_f32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_rsq_f32_e32 v0, s0
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_min_f32_e32 v0, 0x7f7fffff, v0
+; VI-GISEL-NEXT: v_max_f32_e32 v2, 0xff7fffff, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
%rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %src)
store float %rsq_clamp, ptr addrspace(1) %out
ret void
@@ -65,6 +90,32 @@ define amdgpu_kernel void @rsq_clamp_f64(ptr addrspace(1) %out, double %src) #0
; VI-NEXT: v_max_f64 v[0:1], v[0:1], s[2:3]
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_clamp_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_rsq_clamp_f64_e32 v[0:1], s[2:3]
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_clamp_f64:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, -1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fefffff
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], s[2:3]
+; VI-GISEL-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, -1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 0xffefffff
+; VI-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT: s_endpgm
%rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double %src)
store double %rsq_clamp, ptr addrspace(1) %out
ret void
@@ -81,6 +132,28 @@ define amdgpu_kernel void @rsq_clamp_undef_f32(ptr addrspace(1) %out) #0 {
; VI-LABEL: rsq_clamp_undef_f32:
; VI: ; %bb.0:
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_clamp_undef_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_rsq_clamp_f32_e32 v0, s0
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_clamp_undef_f32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: v_rsq_f32_e32 v0, s0
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_min_f32_e32 v0, 0x7f7fffff, v0
+; VI-GISEL-NEXT: v_max_f32_e32 v2, 0xff7fffff, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
%rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float poison)
store float %rsq_clamp, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
index 7e795c1b6876b..1cd24e876eec4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
@@ -29,6 +29,28 @@ define amdgpu_kernel void @rsq_f32(ptr addrspace(1) %out, float %src) #1 {
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dword s3, s[4:5], 0xb
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_rsq_f32_e32 v0, s3
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_f32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_rsq_f32_e32 v0, s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call float @llvm.amdgcn.rsq.f32(float %src) #0
store float %rsq, ptr addrspace(1) %out, align 4
ret void
@@ -55,6 +77,26 @@ define amdgpu_kernel void @rsq_f32_constant_4.0(ptr addrspace(1) %out) #1 {
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_f32_constant_4.0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_rsq_f32_e32 v0, 4.0
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_f32_constant_4.0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_rsq_f32_e32 v0, 4.0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call float @llvm.amdgcn.rsq.f32(float 4.0) #0
store float %rsq, ptr addrspace(1) %out, align 4
ret void
@@ -80,6 +122,26 @@ define amdgpu_kernel void @rsq_f32_constant_100.0(ptr addrspace(1) %out) #1 {
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_f32_constant_100.0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_rsq_f32_e32 v0, 0x42c80000
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_f32_constant_100.0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: v_rsq_f32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call float @llvm.amdgcn.rsq.f32(float 100.0) #0
store float %rsq, ptr addrspace(1) %out, align 4
ret void
@@ -107,6 +169,26 @@ define amdgpu_kernel void @rsq_f64(ptr addrspace(1) %out, double %src) #1 {
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_f64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], s[2:3]
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_f64:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], s[2:3]
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call double @llvm.amdgcn.rsq.f64(double %src) #0
store double %rsq, ptr addrspace(1) %out, align 4
ret void
@@ -133,6 +215,26 @@ define amdgpu_kernel void @rsq_f64_constant_4.0(ptr addrspace(1) %out) #1 {
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_f64_constant_4.0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], 4.0
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_f64_constant_4.0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], 4.0
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call double @llvm.amdgcn.rsq.f64(double 4.0) #0
store double %rsq, ptr addrspace(1) %out, align 4
ret void
@@ -158,6 +260,26 @@ define amdgpu_kernel void @rsq_f64_constant_100.0(ptr addrspace(1) %out) #1 {
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_f64_constant_100.0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], 0x40590000
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_f64_constant_100.0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], 0x40590000
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call double @llvm.amdgcn.rsq.f64(double 100.0) #0
store double %rsq, ptr addrspace(1) %out, align 4
ret void
@@ -174,6 +296,26 @@ define amdgpu_kernel void @rsq_undef_f32(ptr addrspace(1) %out) #1 {
; VI-LABEL: rsq_undef_f32:
; VI: ; %bb.0:
; VI-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: rsq_undef_f32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_rsq_f32_e32 v0, s0
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: rsq_undef_f32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_rsq_f32_e32 v0, s0
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-GISEL-NEXT: s_endpgm
%rsq = call float @llvm.amdgcn.rsq.f32(float poison)
store float %rsq, ptr addrspace(1) %out, align 4
ret void
More information about the llvm-commits
mailing list