[llvm] [AMDGPU][GlobalISel] Add register bank legalize rules for amdgcn_icmp, amdgcn_fcmp (PR #172017)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 02:45:41 PDT 2026
https://github.com/anjenner updated https://github.com/llvm/llvm-project/pull/172017
>From 7e402be7948dcb8f64ed173cb0f397e7b71f8c09 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Fri, 12 Dec 2025 08:53:17 -0500
Subject: [PATCH 1/7] AMDGPU: Add register bank legalize rules for amdgcn_icmp
and amdgcn_fcmp.
---
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 26 +++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index fdcbdc8712f01..eb2f3bb568d50 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1566,6 +1566,32 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForIOpcs({returnaddress}).Any({{UniP0}, {{SgprP0}, {}}});
+ addRulesForIOpcs({amdgcn_icmp})
+ .Any({{UniS64, _, UniS1},
+ {{Sgpr64}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
+ .Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
+ .Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
+ .Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
+
+ .Any({{UniS32, _, UniS1},
+ {{Sgpr32}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
+ .Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
+ .Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
+ .Any({{UniS32, _, S64}, {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
+
+ addRulesForIOpcs({amdgcn_fcmp})
+ .Any({{UniS64, _, UniS1},
+ {{Sgpr64}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
+ .Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
+ .Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
+ .Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
+
+ .Any({{UniS32, _, UniS1},
+ {{Sgpr32}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
+ .Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
+ .Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
+ .Any({{UniS32, _, S64}, {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
+
addRulesForIOpcs({amdgcn_s_getpc}).Any({{UniS64, _}, {{Sgpr64}, {None}}});
addRulesForIOpcs({amdgcn_s_getreg}).Any({{}, {{Sgpr32}, {IntrId, Imm}}});
>From 5e7254fbc51d369bedb3ca6712278dbc27604be2 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Wed, 17 Dec 2025 05:33:23 -0500
Subject: [PATCH 2/7] Update tests.
---
.../GlobalISel/regbankselect-amdgcn.fcmp.mir | 4 +-
.../GlobalISel/regbankselect-amdgcn.icmp.mir | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll | 170 ++++++++++++------
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll | 12 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 15 +-
6 files changed, 122 insertions(+), 87 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
index be59d356af05e..00c2a6102de38 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
---
name: fcmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
index 434cc138f3704..2ba4a3c6cf043 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
---
name: icmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index 87a9ba30490a0..edda79b813378 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX11 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
declare i32 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i32 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 9e4824694e76a..67a973d3b0e07 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-GISEL %s
declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i64 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
@@ -26,17 +26,29 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_oeq_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f32_oeq_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_oeq_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], v0, |s3|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_oeq_with_fabs:
; VI-SDAG: ; %bb.0:
@@ -55,8 +67,8 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |v0|
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], v0, |s3|
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
@@ -82,17 +94,29 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], |v0|, |s3|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
; VI-SDAG: ; %bb.0:
@@ -111,8 +135,8 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], |v0|, |s3|
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
@@ -1838,19 +1862,33 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oeq_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], v0, |s3|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
; VI-SDAG: ; %bb.0:
@@ -1873,8 +1911,8 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_lshr_b32 s3, s2, 16
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], v0, |s3|
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
@@ -1905,19 +1943,33 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |v0|, |s3|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
; VI-SDAG: ; %bb.0:
@@ -1940,8 +1992,8 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; VI-GISEL-NEXT: s_lshr_b32 s3, s2, 16
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
-; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |v0|, |s3|
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index e2b068e2e9105..f8ccd40d37bcc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -2,16 +2,8 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,SDAG-GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s 2>%t | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: FileCheck --check-prefix=ERR %s < %t
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s 2>%t | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
-; RUN: FileCheck --check-prefix=ERR %s < %t
-
-; Note: GlobalISel abort is disabled so we don't crash on i1 inputs.
-; They are allowed in DAGISel but we (intentionally) don't support them
-; in GlobalISel.
-
-; ERR: warning: Instruction selection used fallback path for v_icmp_i1_ne0
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
declare i32 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
declare i32 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index 366b71bae75c9..e6278cf2a52ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -3,18 +3,9 @@
; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,SDAG-VI %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,SDAG-GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s 2>%t | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: FileCheck --check-prefix=ERR %s < %t
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=fiji < %s 2>%t | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
-; RUN: FileCheck --check-prefix=ERR %s < %t
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx900 < %s 2>%t | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
-; RUN: FileCheck --check-prefix=ERR %s < %t
-
-; Note: GlobalISel abort is disabled so we don't crash on i1 inputs.
-; They are allowed in DAGISel but we (intentionally) don't support them
-; in GlobalISel.
-
-; ERR: warning: Instruction selection used fallback path for v_icmp_i1_ne0
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
>From f416e012d216a5454de8f0c4f0085d3b2e10dbba Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Tue, 6 Jan 2026 07:03:49 -0500
Subject: [PATCH 3/7] Make various changes requested in review feedback.
---
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 15 ++
.../AMDGPU/AMDGPURegBankLegalizeRules.h | 1 +
.../GlobalISel/regbankselect-amdgcn.fcmp.mir | 3 +-
.../GlobalISel/regbankselect-amdgcn.icmp.mir | 3 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll | 92 +++++++----
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 148 ++++++++++++------
8 files changed, 186 insertions(+), 86 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index db5573cb08cf8..6601f312f5246 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1468,6 +1468,7 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
case Sgpr32Trunc:
case Sgpr32AExt:
case Sgpr32AExtBoolInReg:
+ case Vgpr32AExtBoolInReg:
case Sgpr32SExt:
case Sgpr32ZExt:
case UniInVgprS32:
@@ -1675,6 +1676,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
case Sgpr32Trunc:
case Sgpr32AExt:
case Sgpr32AExtBoolInReg:
+ case Vgpr32AExtBoolInReg:
case Sgpr32SExt:
case Sgpr32ZExt:
return SgprRB;
@@ -2096,6 +2098,19 @@ bool RegBankLegalizeHelper::applyMappingSrc(
Op.setReg(BoolInReg.getReg(0));
break;
}
+ case Vgpr32AExtBoolInReg: {
+ // Note: this ext allows S1, and it is meant to be combined away.
+ assert(Ty.getSizeInBits() == 1);
+ assert(RB == SgprRB);
+ auto Aext = B.buildAnyExt(SgprRB_S32, Reg);
+ // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
+ // most of times meant to be combined away in AMDGPURegBankCombiner.
+ auto Cst1 = B.buildConstant(SgprRB_S32, 1);
+ auto BoolInReg = B.buildAnd(SgprRB_S32, Aext, Cst1);
+
+ Op.setReg(B.buildCopy(VgprRB_S32, BoolInReg).getReg(0));
+ break;
+ }
case Sgpr32SExt: {
assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
assert(RB == SgprRB);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index d7684d16676a0..e4801323863f0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -258,6 +258,7 @@ enum RegBankLLTMappingApplyID {
// Src only modifiers: extends
Sgpr32AExt,
Sgpr32AExtBoolInReg,
+ Vgpr32AExtBoolInReg,
Sgpr32SExt,
Sgpr32ZExt,
Vgpr32AExt,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
index 00c2a6102de38..685a712bbcd18 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.fcmp.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" -verify-machineinstrs -o - %s | FileCheck %s
---
name: fcmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
index 2ba4a3c6cf043..5c181562954d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.icmp.mir
@@ -1,6 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" -verify-machineinstrs -o - %s | FileCheck %s
---
name: icmp_ss
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index edda79b813378..678ee5d9e95c6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX11 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX10 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
declare i32 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i32 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 67a973d3b0e07..80d77c27b3ec2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-SDAG %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-GISEL %s
declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i64 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index f8ccd40d37bcc..7f79e60857d96 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,SDAG-GFX10 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
declare i32 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
declare i32 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
@@ -1609,33 +1609,65 @@ define amdgpu_kernel void @v_icmp_i16_sle(ptr addrspace(1) %out, i16 %src) {
}
define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b) {
-; GFX11-LABEL: v_icmp_i1_ne0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_gt_u32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_cmp_gt_u32 s3, 2
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s2, s2, s3
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
-;
-; GFX10-LABEL: v_icmp_i1_ne0:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cmp_gt_u32 s2, 1
-; GFX10-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10-NEXT: s_cmp_gt_u32 s3, 2
-; GFX10-NEXT: s_cselect_b32 s3, -1, 0
-; GFX10-NEXT: s_and_b32 s2, s2, s3
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
-; GFX10-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i1_ne0:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: s_cmp_gt_u32 s2, 1
+; SDAG-GFX11-NEXT: s_cselect_b32 s2, -1, 0
+; SDAG-GFX11-NEXT: s_cmp_gt_u32 s3, 2
+; SDAG-GFX11-NEXT: s_cselect_b32 s3, -1, 0
+; SDAG-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; SDAG-GFX11-NEXT: s_and_b32 s2, s2, s3
+; SDAG-GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; SDAG-GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
+;
+; SDAG-GFX10-LABEL: v_icmp_i1_ne0:
+; SDAG-GFX10: ; %bb.0:
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX10-NEXT: s_cmp_gt_u32 s2, 1
+; SDAG-GFX10-NEXT: s_cselect_b32 s2, -1, 0
+; SDAG-GFX10-NEXT: s_cmp_gt_u32 s3, 2
+; SDAG-GFX10-NEXT: s_cselect_b32 s3, -1, 0
+; SDAG-GFX10-NEXT: s_and_b32 s2, s2, s3
+; SDAG-GFX10-NEXT: v_mov_b32_e32 v1, s2
+; SDAG-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; SDAG-GFX10-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i1_ne0:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_cmp_gt_u32 s2, 1
+; GISEL-GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-GFX11-NEXT: s_cmp_gt_u32 s3, 2
+; GISEL-GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_and_b32 s2, s2, s3
+; GISEL-GFX11-NEXT: v_cmp_ne_u32_e64 s2, s2, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-GFX10-LABEL: v_icmp_i1_ne0:
+; GISEL-GFX10: ; %bb.0:
+; GISEL-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX10-NEXT: s_cmp_gt_u32 s2, 1
+; GISEL-GFX10-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-GFX10-NEXT: s_cmp_gt_u32 s3, 2
+; GISEL-GFX10-NEXT: s_cselect_b32 s3, 1, 0
+; GISEL-GFX10-NEXT: s_and_b32 s2, s2, s3
+; GISEL-GFX10-NEXT: v_cmp_ne_u32_e64 s2, s2, 0
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-GFX10-NEXT: s_endpgm
%c0 = icmp ugt i32 %a, 1
%c1 = icmp ugt i32 %b, 2
%src = and i1 %c0, %c1
@@ -1670,3 +1702,5 @@ define amdgpu_ps void @test_intr_icmp_i32_invalid_cc(ptr addrspace(1) %out, i32
attributes #0 = { nounwind readnone convergent }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
+; GFX10: {{.*}}
+; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index e6278cf2a52ee..aca943eb6f839 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -3,9 +3,9 @@
; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,SDAG-VI %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,SDAG-GFX9 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=0 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
@@ -1877,52 +1877,103 @@ define amdgpu_kernel void @v_icmp_i16_sle(ptr addrspace(1) %out, i16 %src) {
}
define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b) {
-; GFX11-LABEL: v_icmp_i1_ne0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_gt_u32 s2, 1
-; GFX11-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX11-NEXT: s_cmp_gt_u32 s3, 2
-; GFX11-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i1_ne0:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: s_cmp_gt_u32 s2, 1
+; SDAG-GFX11-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SDAG-GFX11-NEXT: s_cmp_gt_u32 s3, 2
+; SDAG-GFX11-NEXT: s_cselect_b64 s[2:3], -1, 0
+; SDAG-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; SDAG-GFX11-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
-; VI-LABEL: v_icmp_i1_ne0:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_gt_u32 s2, 1
-; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-NEXT: s_cmp_gt_u32 s3, 2
-; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
-; VI-NEXT: s_endpgm
-;
-; GFX9-LABEL: v_icmp_i1_ne0:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_gt_u32 s2, 1
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: s_cmp_gt_u32 s3, 2
-; GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX9-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-VI-LABEL: v_icmp_i1_ne0:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-VI-NEXT: s_cmp_gt_u32 s2, 1
+; SDAG-VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SDAG-VI-NEXT: s_cmp_gt_u32 s3, 2
+; SDAG-VI-NEXT: s_cselect_b64 s[2:3], -1, 0
+; SDAG-VI-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
+; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
+; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; SDAG-VI-NEXT: s_endpgm
+;
+; SDAG-GFX9-LABEL: v_icmp_i1_ne0:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: s_cmp_gt_u32 s2, 1
+; SDAG-GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SDAG-GFX9-NEXT: s_cmp_gt_u32 s3, 2
+; SDAG-GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
+; SDAG-GFX9-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i1_ne0:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: s_cmp_gt_u32 s2, 1
+; GISEL-GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-GFX11-NEXT: s_cmp_gt_u32 s3, 2
+; GISEL-GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_and_b32 s2, s2, s3
+; GISEL-GFX11-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
+;
+; GISEL-VI-LABEL: v_icmp_i1_ne0:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-VI-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-VI-NEXT: s_cmp_gt_u32 s2, 1
+; GISEL-VI-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-VI-NEXT: s_cmp_gt_u32 s3, 2
+; GISEL-VI-NEXT: s_cselect_b32 s3, 1, 0
+; GISEL-VI-NEXT: s_and_b32 s2, s2, s3
+; GISEL-VI-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, 0
+; GISEL-VI-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-VI-NEXT: v_mov_b32_e32 v3, s1
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
+; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i1_ne0:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_cmp_gt_u32 s2, 1
+; GISEL-GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GISEL-GFX9-NEXT: s_cmp_gt_u32 s3, 2
+; GISEL-GFX9-NEXT: s_cselect_b32 s3, 1, 0
+; GISEL-GFX9-NEXT: s_and_b32 s2, s2, s3
+; GISEL-GFX9-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, 0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%c0 = icmp ugt i32 %a, 1
%c1 = icmp ugt i32 %b, 2
%src = and i1 %c0, %c1
@@ -1966,3 +2017,4 @@ define amdgpu_ps void @test_intr_icmp_i32_invalid_cc(ptr addrspace(1) %out, i32
attributes #0 = { nounwind readnone convergent }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
+; VI: {{.*}}
>From 37f1b398a96ded7949f9166de31d5d31979390d0 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Thu, 28 May 2026 10:08:54 -0400
Subject: [PATCH 4/7] Lower the i1 case to ballot explicitly.
---
.../AMDGPU/AMDGPUInstructionSelector.cpp | 4 ++
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 27 ++++----
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 12 ++--
.../AMDGPU/AMDGPURegBankLegalizeRules.h | 4 +-
.../GlobalISel/llvm.amdgcn.ballot.i32.ll | 3 +-
.../GlobalISel/llvm.amdgcn.ballot.i64.ll | 7 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll | 4 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll | 6 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll | 23 +++++--
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 66 +++++++++++++++++--
10 files changed, 110 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 4ca2de216f487..ad2d81a557d47 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1634,6 +1634,10 @@ static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
return true;
}
+ // Lane mask generated by SCC to VCC copy (GlobalISel).
+ if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
+ return true;
+
// Lane mask generated using compare with same exec.
if (isa<GAnyCmp>(MI))
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 6601f312f5246..4c75753941853 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1449,6 +1449,18 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerAbsToNegMax(MI);
case AbsToS32:
return lowerAbsToS32(MI);
+ case IcmpI1ToBallot: {
+ // amdgcn.icmp(i1 src, i1 0, NE) -> ballot(src)
+ // This implements the "hack ballot" pattern from SIInstructions.td.
+ // For uniform i1: ballot returns -1 if true, 0 if false.
+ // The src operand has been converted to VCC by the Vcc mapping.
+ assert(cast<GIntrinsic>(&MI)->is(Intrinsic::amdgcn_icmp));
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(2).getReg();
+ B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src);
+ MI.eraseFromParent();
+ return true;
+ }
}
return true;
@@ -1468,7 +1480,6 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
case Sgpr32Trunc:
case Sgpr32AExt:
case Sgpr32AExtBoolInReg:
- case Vgpr32AExtBoolInReg:
case Sgpr32SExt:
case Sgpr32ZExt:
case UniInVgprS32:
@@ -1676,7 +1687,6 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
case Sgpr32Trunc:
case Sgpr32AExt:
case Sgpr32AExtBoolInReg:
- case Vgpr32AExtBoolInReg:
case Sgpr32SExt:
case Sgpr32ZExt:
return SgprRB;
@@ -2098,19 +2108,6 @@ bool RegBankLegalizeHelper::applyMappingSrc(
Op.setReg(BoolInReg.getReg(0));
break;
}
- case Vgpr32AExtBoolInReg: {
- // Note: this ext allows S1, and it is meant to be combined away.
- assert(Ty.getSizeInBits() == 1);
- assert(RB == SgprRB);
- auto Aext = B.buildAnyExt(SgprRB_S32, Reg);
- // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
- // most of times meant to be combined away in AMDGPURegBankCombiner.
- auto Cst1 = B.buildConstant(SgprRB_S32, 1);
- auto BoolInReg = B.buildAnd(SgprRB_S32, Aext, Cst1);
-
- Op.setReg(B.buildCopy(VgprRB_S32, BoolInReg).getReg(0));
- break;
- }
case Sgpr32SExt: {
assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
assert(RB == SgprRB);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index eb2f3bb568d50..03c8c2b5e9795 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1568,26 +1568,26 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForIOpcs({amdgcn_icmp})
.Any({{UniS64, _, UniS1},
- {{Sgpr64}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
+ {{Sgpr64}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
+ .Any({{UniS64, _, DivS1},
+ {{Sgpr64}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
.Any({{UniS32, _, UniS1},
- {{Sgpr32}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
+ {{Sgpr32}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
+ .Any({{UniS32, _, DivS1},
+ {{Sgpr32}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS32, _, S64}, {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
addRulesForIOpcs({amdgcn_fcmp})
- .Any({{UniS64, _, UniS1},
- {{Sgpr64}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
.Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
- .Any({{UniS32, _, UniS1},
- {{Sgpr32}, {IntrId, Vgpr32AExtBoolInReg, Vgpr32AExtBoolInReg}}})
.Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS32, _, S64}, {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index e4801323863f0..f3e5a24cf0b9f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -258,7 +258,6 @@ enum RegBankLLTMappingApplyID {
// Src only modifiers: extends
Sgpr32AExt,
Sgpr32AExtBoolInReg,
- Vgpr32AExtBoolInReg,
Sgpr32SExt,
Sgpr32ZExt,
Vgpr32AExt,
@@ -313,7 +312,8 @@ enum LoweringMethodID {
InsVecEltToSel,
InsVecEltTo32,
AbsToNegMax,
- AbsToS32
+ AbsToS32,
+ IcmpI1ToBallot
};
enum FastRulesTypes {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
index ad88b6030004b..91a6ca417d410 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
@@ -21,7 +21,7 @@ define amdgpu_cs i32 @constant_false() {
define amdgpu_cs i32 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_and_b32 s0, exec_lo, exec_lo
+; CHECK-NEXT: s_mov_b32 s0, exec_lo
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
ret i32 %ballot
@@ -383,7 +383,6 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_cmp_lt_u32 s0, 12
; CHECK-NEXT: s_cselect_b32 s0, exec_lo, 0
-; CHECK-NEXT: s_and_b32 s0, s0, exec_lo
; CHECK-NEXT: s_cmp_le_i32 s0, 22
; CHECK-NEXT: s_cbranch_scc1 .LBB18_2
; CHECK-NEXT: ; %bb.1: ; %true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
index 9df5a811b85e6..a62355dbd2a78 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
@@ -9,7 +9,8 @@ declare i64 @llvm.ctpop.i64(i64)
define amdgpu_cs i64 @constant_false() {
; CHECK-LABEL: constant_false:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_and_b64 s[0:1], 0, exec
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 0)
ret i64 %ballot
@@ -20,7 +21,8 @@ define amdgpu_cs i64 @constant_false() {
define amdgpu_cs i64 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_and_b64 s[0:1], exec, exec
+; CHECK-NEXT: s_mov_b32 s0, exec_lo
+; CHECK-NEXT: s_mov_b32 s1, exec_hi
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1)
ret i64 %ballot
@@ -383,7 +385,6 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_cmp_lt_u32 s0, 12
; CHECK-NEXT: s_cselect_b64 s[0:1], exec, 0
-; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
; CHECK-NEXT: v_cmp_le_i64_e64 s[0:1], s[0:1], 22
; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: s_cbranch_scc1 .LBB18_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index 678ee5d9e95c6..555e27357a825 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX11 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=SDAG-GFX10 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GISEL-GFX10 %s
declare i32 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i32 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 80d77c27b3ec2..a87c002ad72ff 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-SDAG %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-GISEL %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=VI-GISEL %s
declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
declare i64 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index 7f79e60857d96..358685c056a95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,SDAG-GFX11 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,SDAG-GFX10 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 -mattr="+wavefrontsize32" < %s | FileCheck -check-prefixes=GCN,GFX10,GISEL-GFX10 %s
declare i32 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
declare i32 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
@@ -1648,8 +1648,9 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX11-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GISEL-GFX11-NEXT: s_and_b32 s2, s2, s3
-; GISEL-GFX11-NEXT: v_cmp_ne_u32_e64 s2, s2, 0
-; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GISEL-GFX11-NEXT: s_cselect_b32 s2, exec_lo, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
@@ -1664,7 +1665,8 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX10-NEXT: s_cmp_gt_u32 s3, 2
; GISEL-GFX10-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX10-NEXT: s_and_b32 s2, s2, s3
-; GISEL-GFX10-NEXT: v_cmp_ne_u32_e64 s2, s2, 0
+; GISEL-GFX10-NEXT: s_cmp_lg_u32 s2, 0
+; GISEL-GFX10-NEXT: s_cselect_b32 s2, exec_lo, 0
; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
@@ -1676,6 +1678,16 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
ret void
}
+define amdgpu_ps i32 @v_icmp_i1_ne0_divergent(i32 %a) {
+; GCN-LABEL: v_icmp_i1_ne0_divergent:
+; GCN: ; %bb.0:
+; GCN-NEXT: v_cmp_eq_u32_e64 s0, 0, v0
+; GCN-NEXT: ; return to shader part epilog
+ %cond = icmp eq i32 %a, 0
+ %result = call i32 @llvm.amdgcn.icmp.i1(i1 %cond, i1 false, i32 33)
+ ret i32 %result
+}
+
define amdgpu_ps void @test_intr_icmp_i32_invalid_cc(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: test_intr_icmp_i32_invalid_cc:
; SDAG-GFX11: ; %bb.0:
@@ -1701,6 +1713,5 @@ define amdgpu_ps void @test_intr_icmp_i32_invalid_cc(ptr addrspace(1) %out, i32
attributes #0 = { nounwind readnone convergent }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GCN: {{.*}}
; GFX10: {{.*}}
; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index aca943eb6f839..f928e9ab082db 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -3,9 +3,9 @@
; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,SDAG-VI %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,SDAG-GFX9 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
-; RUN: llc -global-isel=1 -new-reg-bank-select -global-isel-abort=1 -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
@@ -1935,8 +1935,9 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX11-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GISEL-GFX11-NEXT: s_and_b32 s2, s2, s3
-; GISEL-GFX11-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, 0
-; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GISEL-GFX11-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -1951,7 +1952,8 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-VI-NEXT: s_cmp_gt_u32 s3, 2
; GISEL-VI-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-VI-NEXT: s_and_b32 s2, s2, s3
-; GISEL-VI-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, 0
+; GISEL-VI-NEXT: s_cmp_lg_u32 s2, 0
+; GISEL-VI-NEXT: s_cselect_b64 s[2:3], exec, 0
; GISEL-VI-NEXT: v_mov_b32_e32 v0, s2
; GISEL-VI-NEXT: v_mov_b32_e32 v3, s1
; GISEL-VI-NEXT: v_mov_b32_e32 v1, s3
@@ -1969,7 +1971,8 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX9-NEXT: s_cmp_gt_u32 s3, 2
; GISEL-GFX9-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX9-NEXT: s_and_b32 s2, s2, s3
-; GISEL-GFX9-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, 0
+; GISEL-GFX9-NEXT: s_cmp_lg_u32 s2, 0
+; GISEL-GFX9-NEXT: s_cselect_b64 s[2:3], exec, 0
; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -1982,6 +1985,55 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
ret void
}
+define amdgpu_ps i64 @v_icmp_i1_ne0_divergent(i32 %a) {
+; SDAG-GFX11-LABEL: v_icmp_i1_ne0_divergent:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s1
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-GFX11-NEXT: ; return to shader part epilog
+;
+; SDAG-VI-LABEL: v_icmp_i1_ne0_divergent:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
+; SDAG-VI-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-VI-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-VI-NEXT: ; return to shader part epilog
+;
+; SDAG-GFX9-LABEL: v_icmp_i1_ne0_divergent:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s1
+; SDAG-GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-GFX9-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: v_icmp_i1_ne0_divergent:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-VI-LABEL: v_icmp_i1_ne0_divergent:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; GISEL-VI-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX9-LABEL: v_icmp_i1_ne0_divergent:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v0
+; GISEL-GFX9-NEXT: ; return to shader part epilog
+ %cond = icmp eq i32 %a, 0
+ %result = call i64 @llvm.amdgcn.icmp.i1(i1 %cond, i1 false, i32 33)
+ ret i64 %result
+}
+
define amdgpu_ps void @test_intr_icmp_i32_invalid_cc(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: test_intr_icmp_i32_invalid_cc:
; SDAG-GFX11: ; %bb.0:
>From dc26da76b10f6b04edb5620c8dbaf997c063acf7 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Wed, 10 Jun 2026 11:32:37 -0400
Subject: [PATCH 5/7] Address review feedback.
---
.../Target/AMDGPU/AMDGPUInstructionSelector.cpp | 4 ----
.../Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 15 +++++++++++++--
.../Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp | 8 ++------
.../AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll | 3 ++-
.../AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll | 7 +++----
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll | 4 +++-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 5 ++++-
7 files changed, 27 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index ad2d81a557d47..4ca2de216f487 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1634,10 +1634,6 @@ static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
return true;
}
- // Lane mask generated by SCC to VCC copy (GlobalISel).
- if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
- return true;
-
// Lane mask generated using compare with same exec.
if (isa<GAnyCmp>(MI))
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 4c75753941853..b3977cdbc417c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1451,10 +1451,21 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerAbsToS32(MI);
case IcmpI1ToBallot: {
// amdgcn.icmp(i1 src, i1 0, NE) -> ballot(src)
- // This implements the "hack ballot" pattern from SIInstructions.td.
- // For uniform i1: ballot returns -1 if true, 0 if false.
// The src operand has been converted to VCC by the Vcc mapping.
assert(cast<GIntrinsic>(&MI)->is(Intrinsic::amdgcn_icmp));
+
+ Register Src1 = MI.getOperand(3).getReg();
+ auto Src1Const =
+ getAnyConstantVRegValWithLookThrough(Src1, MRI,
+ /*LookThroughInstrs=*/true,
+ /*LookThroughAnyExt=*/true);
+ if (!Src1Const || Src1Const->Value != 0)
+ return false;
+
+ int64_t Pred = MI.getOperand(4).getImm();
+ if (Pred != CmpInst::ICMP_NE)
+ return false;
+
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(2).getReg();
B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 03c8c2b5e9795..acca1335320ad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1567,17 +1567,13 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForIOpcs({returnaddress}).Any({{UniP0}, {{SgprP0}, {}}});
addRulesForIOpcs({amdgcn_icmp})
- .Any({{UniS64, _, UniS1},
- {{Sgpr64}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
- .Any({{UniS64, _, DivS1},
+ .Any({{UniS64, _, S1},
{{Sgpr64}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
- .Any({{UniS32, _, UniS1},
- {{Sgpr32}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
- .Any({{UniS32, _, DivS1},
+ .Any({{UniS32, _, S1},
{{Sgpr32}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
index 91a6ca417d410..ad88b6030004b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
@@ -21,7 +21,7 @@ define amdgpu_cs i32 @constant_false() {
define amdgpu_cs i32 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_mov_b32 s0, exec_lo
+; CHECK-NEXT: s_and_b32 s0, exec_lo, exec_lo
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
ret i32 %ballot
@@ -383,6 +383,7 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_cmp_lt_u32 s0, 12
; CHECK-NEXT: s_cselect_b32 s0, exec_lo, 0
+; CHECK-NEXT: s_and_b32 s0, s0, exec_lo
; CHECK-NEXT: s_cmp_le_i32 s0, 22
; CHECK-NEXT: s_cbranch_scc1 .LBB18_2
; CHECK-NEXT: ; %bb.1: ; %true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
index a62355dbd2a78..9df5a811b85e6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
@@ -9,8 +9,7 @@ declare i64 @llvm.ctpop.i64(i64)
define amdgpu_cs i64 @constant_false() {
; CHECK-LABEL: constant_false:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_mov_b32 s0, 0
-; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: s_and_b64 s[0:1], 0, exec
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 0)
ret i64 %ballot
@@ -21,8 +20,7 @@ define amdgpu_cs i64 @constant_false() {
define amdgpu_cs i64 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_mov_b32 s0, exec_lo
-; CHECK-NEXT: s_mov_b32 s1, exec_hi
+; CHECK-NEXT: s_and_b64 s[0:1], exec, exec
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1)
ret i64 %ballot
@@ -385,6 +383,7 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_cmp_lt_u32 s0, 12
; CHECK-NEXT: s_cselect_b64 s[0:1], exec, 0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
; CHECK-NEXT: v_cmp_le_i64_e64 s[0:1], s[0:1], 22
; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: s_cbranch_scc1 .LBB18_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index 358685c056a95..b9be4b667eabc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -1650,7 +1650,8 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX11-NEXT: s_and_b32 s2, s2, s3
; GISEL-GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GISEL-GFX11-NEXT: s_cselect_b32 s2, exec_lo, 0
-; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_and_b32 s2, s2, exec_lo
; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GISEL-GFX11-NEXT: s_endpgm
@@ -1667,6 +1668,7 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX10-NEXT: s_and_b32 s2, s2, s3
; GISEL-GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GISEL-GFX10-NEXT: s_cselect_b32 s2, exec_lo, 0
+; GISEL-GFX10-NEXT: s_and_b32 s2, s2, exec_lo
; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GISEL-GFX10-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index f928e9ab082db..97f6050525eaa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -1937,7 +1937,8 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX11-NEXT: s_and_b32 s2, s2, s3
; GISEL-GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GISEL-GFX11-NEXT: s_cselect_b64 s[2:3], exec, 0
-; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GISEL-GFX11-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -1954,6 +1955,7 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-VI-NEXT: s_and_b32 s2, s2, s3
; GISEL-VI-NEXT: s_cmp_lg_u32 s2, 0
; GISEL-VI-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GISEL-VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GISEL-VI-NEXT: v_mov_b32_e32 v0, s2
; GISEL-VI-NEXT: v_mov_b32_e32 v3, s1
; GISEL-VI-NEXT: v_mov_b32_e32 v1, s3
@@ -1973,6 +1975,7 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; GISEL-GFX9-NEXT: s_and_b32 s2, s2, s3
; GISEL-GFX9-NEXT: s_cmp_lg_u32 s2, 0
; GISEL-GFX9-NEXT: s_cselect_b64 s[2:3], exec, 0
+; GISEL-GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
>From f3fb7316e30c8ad71238a045af73dc7973e8a69d Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Wed, 10 Jun 2026 11:57:15 -0400
Subject: [PATCH 6/7] Use a cpp predicate for checking IcmpI1ToBallot compares.
---
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 14 +-------------
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 19 +++++++++++++++++--
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index b3977cdbc417c..2478c91933df4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1451,21 +1451,9 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerAbsToS32(MI);
case IcmpI1ToBallot: {
// amdgcn.icmp(i1 src, i1 0, NE) -> ballot(src)
+ // The rule predicate verified src1 == 0 and predicate == NE.
// The src operand has been converted to VCC by the Vcc mapping.
assert(cast<GIntrinsic>(&MI)->is(Intrinsic::amdgcn_icmp));
-
- Register Src1 = MI.getOperand(3).getReg();
- auto Src1Const =
- getAnyConstantVRegValWithLookThrough(Src1, MRI,
- /*LookThroughInstrs=*/true,
- /*LookThroughAnyExt=*/true);
- if (!Src1Const || Src1Const->Value != 0)
- return false;
-
- int64_t Pred = MI.getOperand(4).getImm();
- if (Pred != CmpInst::ICMP_NE)
- return false;
-
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(2).getReg();
B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index acca1335320ad..91d7b00c58073 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1566,14 +1566,29 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForIOpcs({returnaddress}).Any({{UniP0}, {{SgprP0}, {}}});
+ // amdgcn.icmp(i1 src, i1 0, NE) is the only valid form with i1 inputs.
+ // Check that src1 is constant 0 and predicate is NE.
+ Predicate isIcmpI1NeZero([](const MachineInstr &MI) -> bool {
+ // Check predicate is ICMP_NE (operand 4 for intrinsic).
+ int64_t Pred = MI.getOperand(4).getImm();
+ if (Pred != CmpInst::ICMP_NE)
+ return false;
+
+ // Check src1 (operand 3) is constant 0.
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ Register Src1 = MI.getOperand(3).getReg();
+ auto Src1Const = getAnyConstantVRegValWithLookThrough(Src1, MRI);
+ return Src1Const && Src1Const->Value == 0;
+ });
+
addRulesForIOpcs({amdgcn_icmp})
- .Any({{UniS64, _, S1},
+ .Any({{{UniS64, _, S1}, isIcmpI1NeZero},
{{Sgpr64}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
- .Any({{UniS32, _, S1},
+ .Any({{{UniS32, _, S1}, isIcmpI1NeZero},
{{Sgpr32}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
>From 0a79ad1d5df75a387778ea0585758ae8f129890a Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Thu, 11 Jun 2026 05:52:21 -0400
Subject: [PATCH 7/7] Move icmp i1 to ballot lowering to legalizeIntrinsic.
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 25 +++++++++++++++++++
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 11 --------
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 21 ++--------------
.../AMDGPU/AMDGPURegBankLegalizeRules.h | 3 +--
4 files changed, 28 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 8b5076cdd7712..b1e1ce930ff55 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -8069,6 +8069,31 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
// Replace the use G_BRCOND with the exec manipulate and branch pseudos.
auto IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
switch (IntrID) {
+ case Intrinsic::amdgcn_icmp: {
+ // amdgcn.icmp(i1 src0, i1 0, NE) -> ballot(src0)
+ // This is the only valid form of amdgcn.icmp with i1 inputs.
+ Register Src0 = MI.getOperand(2).getReg();
+ LLT SrcTy = MRI.getType(Src0);
+ if (SrcTy != LLT::scalar(1))
+ return true; // Not i1, leave for default handling.
+
+ // Check that src1 is constant 0.
+ Register Src1 = MI.getOperand(3).getReg();
+ auto Src1Const = getIConstantVRegValWithLookThrough(Src1, MRI);
+ if (!Src1Const || Src1Const->Value != 0)
+ return false; // Invalid i1 icmp form.
+
+ // Check that predicate is ICMP_NE.
+ int64_t Pred = MI.getOperand(4).getImm();
+ if (Pred != CmpInst::ICMP_NE)
+ return false; // Invalid i1 icmp form.
+
+ // Convert to ballot.
+ Register Dst = MI.getOperand(0).getReg();
+ B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src0);
+ MI.eraseFromParent();
+ return true;
+ }
case Intrinsic::sponentry:
if (B.getMF().getInfo<SIMachineFunctionInfo>()->isBottomOfStack()) {
// FIXME: The imported pattern checks for i32 instead of p5; if we fix
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 2478c91933df4..db5573cb08cf8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1449,17 +1449,6 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerAbsToNegMax(MI);
case AbsToS32:
return lowerAbsToS32(MI);
- case IcmpI1ToBallot: {
- // amdgcn.icmp(i1 src, i1 0, NE) -> ballot(src)
- // The rule predicate verified src1 == 0 and predicate == NE.
- // The src operand has been converted to VCC by the Vcc mapping.
- assert(cast<GIntrinsic>(&MI)->is(Intrinsic::amdgcn_icmp));
- Register Dst = MI.getOperand(0).getReg();
- Register Src = MI.getOperand(2).getReg();
- B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src);
- MI.eraseFromParent();
- return true;
- }
}
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 91d7b00c58073..474208b37c36e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1566,30 +1566,13 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
addRulesForIOpcs({returnaddress}).Any({{UniP0}, {{SgprP0}, {}}});
- // amdgcn.icmp(i1 src, i1 0, NE) is the only valid form with i1 inputs.
- // Check that src1 is constant 0 and predicate is NE.
- Predicate isIcmpI1NeZero([](const MachineInstr &MI) -> bool {
- // Check predicate is ICMP_NE (operand 4 for intrinsic).
- int64_t Pred = MI.getOperand(4).getImm();
- if (Pred != CmpInst::ICMP_NE)
- return false;
-
- // Check src1 (operand 3) is constant 0.
- const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
- Register Src1 = MI.getOperand(3).getReg();
- auto Src1Const = getAnyConstantVRegValWithLookThrough(Src1, MRI);
- return Src1Const && Src1Const->Value == 0;
- });
-
+ // Note: amdgcn.icmp with i1 inputs is legalized to ballot in the legalizer,
+ // so no S1 rules are needed here.
addRulesForIOpcs({amdgcn_icmp})
- .Any({{{UniS64, _, S1}, isIcmpI1NeZero},
- {{Sgpr64}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS64, _, S16}, {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS64, _, S32}, {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS64, _, S64}, {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
- .Any({{{UniS32, _, S1}, isIcmpI1NeZero},
- {{Sgpr32}, {IntrId, Vcc, Sgpr32AExt}, IcmpI1ToBallot}})
.Any({{UniS32, _, S16}, {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
.Any({{UniS32, _, S32}, {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
.Any({{UniS32, _, S64}, {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index f3e5a24cf0b9f..d7684d16676a0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -312,8 +312,7 @@ enum LoweringMethodID {
InsVecEltToSel,
InsVecEltTo32,
AbsToNegMax,
- AbsToS32,
- IcmpI1ToBallot
+ AbsToS32
};
enum FastRulesTypes {
More information about the llvm-commits
mailing list