[llvm] GlobalISel: Don't use GISelMatchGenericTypes for floating point opcodes (PR #212814)
Petar Avramovic via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 05:27:52 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/212814
>From edff43101cfa9d3e147ac0ca05209fecf27020a8 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Thu, 30 Jul 2026 14:20:56 +0200
Subject: [PATCH] GlobalISel: Don't use GISelMatchGenericTypes for floating
point opcodes
With GISelMatchGenericTypes, AMDGPU had the same input pattern for
G_ATOMICRMW_FADD that checks for v2s16, so the one that is first
in the tablegen table always fires (it was v2f16 in this case).
Should explicitly check for v2f16/v2bf16.
---
llvm/include/llvm/Target/GenericOpcodes.td | 22 ++++--
.../inst-select-atomicrmw-fadd-local.mir | 76 ++++++++++---------
.../inst-select-atomicrmw-fadd-region.mir | 76 ++++++++++---------
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 60 +++++----------
4 files changed, 116 insertions(+), 118 deletions(-)
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 8270ae7690953..b7b216b552a04 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1478,19 +1478,25 @@ def G_ATOMICRMW_MAX : G_ATOMICRMW_OP;
def G_ATOMICRMW_MIN : G_ATOMICRMW_OP;
def G_ATOMICRMW_UMAX : G_ATOMICRMW_OP;
def G_ATOMICRMW_UMIN : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FADD : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FSUB : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FMAX : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FMIN : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FMAXIMUM : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FMINIMUM : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FMAXIMUMNUM : G_ATOMICRMW_OP;
-def G_ATOMICRMW_FMINIMUMNUM : G_ATOMICRMW_OP;
def G_ATOMICRMW_UINC_WRAP : G_ATOMICRMW_OP;
def G_ATOMICRMW_UDEC_WRAP : G_ATOMICRMW_OP;
def G_ATOMICRMW_USUB_COND : G_ATOMICRMW_OP;
def G_ATOMICRMW_USUB_SAT : G_ATOMICRMW_OP;
+// FP atomicrmw: match exact FP type, not generic sN, f16 and bf16 are both s16.
+class G_ATOMICRMW_FP_OP : G_ATOMICRMW_OP {
+ let GISelMatchGenericTypes = 0;
+}
+
+def G_ATOMICRMW_FADD : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FSUB : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FMAX : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FMIN : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FMAXIMUM : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FMINIMUM : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FMAXIMUMNUM : G_ATOMICRMW_FP_OP;
+def G_ATOMICRMW_FMINIMUMNUM : G_ATOMICRMW_FP_OP;
+
def G_FENCE : GenericInstruction {
let OutOperandList = (outs);
let InOperandList = (ins i32imm:$ordering, i32imm:$scope);
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir
index 45a121f3a48aa..4b860dab62b02 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir
@@ -9,7 +9,7 @@
# RUN: llc -mtriple=amdgpu7.01 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s
---
-name: atomicrmw_fadd_s32_local
+name: atomicrmw_fadd_f32_local
legalized: true
regBankSelected: true
tracksRegLiveness: true
@@ -17,38 +17,40 @@ body: |
bb.0:
liveins: $vgpr0, $vgpr1
- ; GFX8-LABEL: name: atomicrmw_fadd_s32_local
+ ; GFX8-LABEL: name: atomicrmw_fadd_f32_local
; GFX8: liveins: $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: $m0 = S_MOV_B32 -1
- ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3)
+ ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 3)
; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
- ; GFX9-LABEL: name: atomicrmw_fadd_s32_local
+ ;
+ ; GFX9-LABEL: name: atomicrmw_fadd_f32_local
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX9-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3)
+ ; GFX9-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (f32), addrspace 3)
; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]]
- ; GFX6-LABEL: name: atomicrmw_fadd_s32_local
+ ;
+ ; GFX6-LABEL: name: atomicrmw_fadd_f32_local
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(f32) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3)
- ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+ ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(f32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (f32), addrspace 3)
+ ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](f32)
%0:vgpr(p3) = COPY $vgpr0
- %1:vgpr(s32) = COPY $vgpr1
- %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p3), %1 :: (load store seq_cst (s32), addrspace 3)
+ %1:vgpr(f32) = COPY $vgpr1
+ %2:vgpr(f32) = G_ATOMICRMW_FADD %0(p3), %1 :: (load store seq_cst (f32), addrspace 3)
$vgpr0 = COPY %2
...
---
-name: atomicrmw_fadd_s32_local_noret
+name: atomicrmw_fadd_f32_local_noret
legalized: true
regBankSelected: true
tracksRegLiveness: true
@@ -56,34 +58,36 @@ body: |
bb.0:
liveins: $vgpr0, $vgpr1
- ; GFX8-LABEL: name: atomicrmw_fadd_s32_local_noret
+ ; GFX8-LABEL: name: atomicrmw_fadd_f32_local_noret
; GFX8: liveins: $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: $m0 = S_MOV_B32 -1
- ; GFX8-NEXT: DS_ADD_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3)
- ; GFX9-LABEL: name: atomicrmw_fadd_s32_local_noret
+ ; GFX8-NEXT: DS_ADD_F32 [[COPY]], [[COPY1]], 0, 0, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 3)
+ ;
+ ; GFX9-LABEL: name: atomicrmw_fadd_f32_local_noret
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX9-NEXT: DS_ADD_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3)
- ; GFX6-LABEL: name: atomicrmw_fadd_s32_local_noret
+ ; GFX9-NEXT: DS_ADD_F32_gfx9 [[COPY]], [[COPY1]], 0, 0, implicit $exec :: (load store seq_cst (f32), addrspace 3)
+ ;
+ ; GFX6-LABEL: name: atomicrmw_fadd_f32_local_noret
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(f32) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3)
+ ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(f32) = G_ATOMICRMW_FADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst (f32), addrspace 3)
%0:vgpr(p3) = COPY $vgpr0
- %1:vgpr(s32) = COPY $vgpr1
- %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p3), %1 :: (load store seq_cst (s32), addrspace 3)
+ %1:vgpr(f32) = COPY $vgpr1
+ %2:vgpr(f32) = G_ATOMICRMW_FADD %0(p3), %1 :: (load store seq_cst (f32), addrspace 3)
...
---
-name: atomicrmw_fadd_s32_local_gep4
+name: atomicrmw_fadd_f32_local_gep4
legalized: true
regBankSelected: true
tracksRegLiveness: true
@@ -91,36 +95,38 @@ body: |
bb.0:
liveins: $vgpr0, $vgpr1
- ; GFX8-LABEL: name: atomicrmw_fadd_s32_local_gep4
+ ; GFX8-LABEL: name: atomicrmw_fadd_f32_local_gep4
; GFX8: liveins: $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: $m0 = S_MOV_B32 -1
- ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 0, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 3)
+ ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 0, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 3)
; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
- ; GFX9-LABEL: name: atomicrmw_fadd_s32_local_gep4
+ ;
+ ; GFX9-LABEL: name: atomicrmw_fadd_f32_local_gep4
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX9-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 4, 0, implicit $exec :: (load store seq_cst (s32), addrspace 3)
+ ; GFX9-NEXT: [[DS_ADD_RTN_F32_gfx9_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32_gfx9 [[COPY]], [[COPY1]], 4, 0, implicit $exec :: (load store seq_cst (f32), addrspace 3)
; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_gfx9_]]
- ; GFX6-LABEL: name: atomicrmw_fadd_s32_local_gep4
+ ;
+ ; GFX6-LABEL: name: atomicrmw_fadd_f32_local_gep4
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4
- ; GFX6-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(f32) = COPY $vgpr1
+ ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(f32) = G_CONSTANT i32 4
+ ; GFX6-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[C]](f32)
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p3), [[COPY1]] :: (load store seq_cst (s32), addrspace 3)
- ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+ ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(f32) = G_ATOMICRMW_FADD [[PTR_ADD]](p3), [[COPY1]] :: (load store seq_cst (f32), addrspace 3)
+ ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](f32)
%0:vgpr(p3) = COPY $vgpr0
- %1:vgpr(s32) = COPY $vgpr1
- %2:vgpr(s32) = G_CONSTANT i32 4
+ %1:vgpr(f32) = COPY $vgpr1
+ %2:vgpr(f32) = G_CONSTANT i32 4
%3:vgpr(p3) = G_PTR_ADD %0, %2
- %4:vgpr(s32) = G_ATOMICRMW_FADD %3(p3), %1 :: (load store seq_cst (s32), addrspace 3)
+ %4:vgpr(f32) = G_ATOMICRMW_FADD %3(p3), %1 :: (load store seq_cst (f32), addrspace 3)
$vgpr0 = COPY %4
...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir
index 6bc57348c8ee1..fbaae37a0017d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-region.mir
@@ -9,7 +9,7 @@
# RUN: llc -mtriple=amdgpu7.01 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -disable-gisel-legality-check -o - %s | FileCheck -check-prefix=GFX6 %s
---
-name: atomicrmw_fadd_s32_region
+name: atomicrmw_fadd_f32_region
legalized: true
regBankSelected: true
tracksRegLiveness: true
@@ -17,38 +17,40 @@ body: |
bb.0:
liveins: $vgpr0, $vgpr1
- ; GFX8-LABEL: name: atomicrmw_fadd_s32_region
+ ; GFX8-LABEL: name: atomicrmw_fadd_f32_region
; GFX8: liveins: $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: $m0 = S_MOV_B32 -1
- ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2)
+ ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 2)
; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
- ; GFX9-LABEL: name: atomicrmw_fadd_s32_region
+ ;
+ ; GFX9-LABEL: name: atomicrmw_fadd_f32_region
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX9-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2)
+ ; GFX9-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 2)
; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
- ; GFX6-LABEL: name: atomicrmw_fadd_s32_region
+ ;
+ ; GFX6-LABEL: name: atomicrmw_fadd_f32_region
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(f32) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2)
- ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+ ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(f32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (f32), addrspace 2)
+ ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](f32)
%0:vgpr(p2) = COPY $vgpr0
- %1:vgpr(s32) = COPY $vgpr1
- %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst (s32), addrspace 2)
+ %1:vgpr(f32) = COPY $vgpr1
+ %2:vgpr(f32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst (f32), addrspace 2)
$vgpr0 = COPY %2
...
---
-name: atomicrmw_fadd_s32_region_noret
+name: atomicrmw_fadd_f32_region_noret
legalized: true
regBankSelected: true
tracksRegLiveness: true
@@ -56,34 +58,36 @@ body: |
bb.0:
liveins: $vgpr0, $vgpr1
- ; GFX8-LABEL: name: atomicrmw_fadd_s32_region_noret
+ ; GFX8-LABEL: name: atomicrmw_fadd_f32_region_noret
; GFX8: liveins: $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: $m0 = S_MOV_B32 -1
- ; GFX8-NEXT: DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2)
- ; GFX9-LABEL: name: atomicrmw_fadd_s32_region_noret
+ ; GFX8-NEXT: DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 2)
+ ;
+ ; GFX9-LABEL: name: atomicrmw_fadd_f32_region_noret
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX9-NEXT: DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2)
- ; GFX6-LABEL: name: atomicrmw_fadd_s32_region_noret
+ ; GFX9-NEXT: DS_ADD_F32 [[COPY]], [[COPY1]], 0, 1, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 2)
+ ;
+ ; GFX6-LABEL: name: atomicrmw_fadd_f32_region_noret
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(f32) = COPY $vgpr1
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2)
+ ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr(f32) = G_ATOMICRMW_FADD [[COPY]](p2), [[COPY1]] :: (load store seq_cst (f32), addrspace 2)
%0:vgpr(p2) = COPY $vgpr0
- %1:vgpr(s32) = COPY $vgpr1
- %2:vgpr(s32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst (s32), addrspace 2)
+ %1:vgpr(f32) = COPY $vgpr1
+ %2:vgpr(f32) = G_ATOMICRMW_FADD %0(p2), %1 :: (load store seq_cst (f32), addrspace 2)
...
---
-name: atomicrmw_fadd_s32_region_gep4
+name: atomicrmw_fadd_f32_region_gep4
legalized: true
regBankSelected: true
tracksRegLiveness: true
@@ -91,36 +95,38 @@ body: |
bb.0:
liveins: $vgpr0, $vgpr1
- ; GFX8-LABEL: name: atomicrmw_fadd_s32_region_gep4
+ ; GFX8-LABEL: name: atomicrmw_fadd_f32_region_gep4
; GFX8: liveins: $vgpr0, $vgpr1
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX8-NEXT: $m0 = S_MOV_B32 -1
- ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2)
+ ; GFX8-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 2)
; GFX8-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
- ; GFX9-LABEL: name: atomicrmw_fadd_s32_region_gep4
+ ;
+ ; GFX9-LABEL: name: atomicrmw_fadd_f32_region_gep4
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX9-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (s32), addrspace 2)
+ ; GFX9-NEXT: [[DS_ADD_RTN_F32_:%[0-9]+]]:vgpr_32 = DS_ADD_RTN_F32 [[COPY]], [[COPY1]], 4, 1, implicit $m0, implicit $exec :: (load store seq_cst (f32), addrspace 2)
; GFX9-NEXT: $vgpr0 = COPY [[DS_ADD_RTN_F32_]]
- ; GFX6-LABEL: name: atomicrmw_fadd_s32_region_gep4
+ ;
+ ; GFX6-LABEL: name: atomicrmw_fadd_f32_region_gep4
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
- ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
- ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 4
- ; GFX6-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p2) = G_PTR_ADD [[COPY]], [[C]](s32)
+ ; GFX6-NEXT: [[COPY1:%[0-9]+]]:vgpr(f32) = COPY $vgpr1
+ ; GFX6-NEXT: [[C:%[0-9]+]]:vgpr(f32) = G_CONSTANT i32 4
+ ; GFX6-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p2) = G_PTR_ADD [[COPY]], [[C]](f32)
; GFX6-NEXT: $m0 = S_MOV_B32 -1
- ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(s32) = G_ATOMICRMW_FADD [[PTR_ADD]](p2), [[COPY1]] :: (load store seq_cst (s32), addrspace 2)
- ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](s32)
+ ; GFX6-NEXT: [[ATOMICRMW_FADD:%[0-9]+]]:vgpr_32(f32) = G_ATOMICRMW_FADD [[PTR_ADD]](p2), [[COPY1]] :: (load store seq_cst (f32), addrspace 2)
+ ; GFX6-NEXT: $vgpr0 = COPY [[ATOMICRMW_FADD]](f32)
%0:vgpr(p2) = COPY $vgpr0
- %1:vgpr(s32) = COPY $vgpr1
- %2:vgpr(s32) = G_CONSTANT i32 4
+ %1:vgpr(f32) = COPY $vgpr1
+ %2:vgpr(f32) = G_CONSTANT i32 4
%3:vgpr(p2) = G_PTR_ADD %0, %2
- %4:vgpr(s32) = G_ATOMICRMW_FADD %3(p2), %1 :: (load store seq_cst (s32), addrspace 2)
+ %4:vgpr(f32) = G_ATOMICRMW_FADD %3(p2), %1 :: (load store seq_cst (f32), addrspace 2)
$vgpr0 = COPY %4
...
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index ccdd6535614aa..78aefe033755a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -13826,25 +13826,15 @@ define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
}
define <2 x bfloat> @flat_atomic_fadd_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bfloat> %data) {
-; GFX1250-SDAG-LABEL: flat_atomic_fadd_v2bf16_saddr_rtn:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_pk_add_bf16 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-GISEL-LABEL: flat_atomic_fadd_v2bf16_saddr_rtn:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_pk_add_f16 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: flat_atomic_fadd_v2bf16_saddr_rtn:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: flat_atomic_pk_add_bf16 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-SDAG-LABEL: flat_atomic_fadd_v2bf16_saddr_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -13859,7 +13849,7 @@ define <2 x bfloat> @flat_atomic_fadd_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bflo
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-GISEL-NEXT: flat_atomic_pk_add_f16 v0, v[2:3], v0 offset:40 sc0
+; GFX950-GISEL-NEXT: flat_atomic_pk_add_bf16 v0, v[2:3], v0 offset:40 sc0
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 10
@@ -13868,25 +13858,15 @@ define <2 x bfloat> @flat_atomic_fadd_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bflo
}
define void @flat_atomic_fadd_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %data) {
-; GFX1250-SDAG-LABEL: flat_atomic_fadd_v2bf16_saddr_nortn:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_pk_add_bf16 v1, v0, s[0:1] offset:40
-; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-GISEL-LABEL: flat_atomic_fadd_v2bf16_saddr_nortn:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_pk_add_f16 v1, v0, s[0:1] offset:40
-; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: flat_atomic_fadd_v2bf16_saddr_nortn:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: flat_atomic_pk_add_bf16 v1, v0, s[0:1] offset:40
+; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-SDAG-LABEL: flat_atomic_fadd_v2bf16_saddr_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -13901,7 +13881,7 @@ define void @flat_atomic_fadd_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %d
; GFX950-GISEL: ; %bb.0:
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-GISEL-NEXT: flat_atomic_pk_add_f16 v[2:3], v0 offset:40
+; GFX950-GISEL-NEXT: flat_atomic_pk_add_bf16 v[2:3], v0 offset:40
; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 10
More information about the llvm-commits
mailing list