[llvm] AMDGPU: Preserve carry-out dead flag in V_ADD_CO_U32 custom inserter (PR #225736)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 06:20:51 PDT 2026
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/225736
>From b24530ec8aa2b4837e1e831a04bace97894a7066 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 23 Sep 2026 12:15:53 +0200
Subject: [PATCH] AMDGPU: Preserve carry-out dead flag in V_ADD_CO_U32 custom
inserter
EmitInstrWithCustomInserter rebuilds V_ADD/SUB_CO_U32_e32 but did
not carry over the dead flag. Preserve the flag to reduce implicitly
depending on dead flags recomputed by LiveVariables.
Co-authored-by: Claude (Opus 4.8) <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 12 +++-
.../finalize-isel-vadd-co-u32-dead-carry.mir | 72 +++++++++++++++++++
llvm/test/CodeGen/AMDGPU/sdiv64.ll | 10 ++-
3 files changed, 85 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/finalize-isel-vadd-co-u32-dead-carry.mir
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f9a4ba4c1bd1d8..43cb9580e80e9a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7334,14 +7334,20 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
NeedClampOperand = true;
}
+ bool VCCDead = MI.getOperand(3).isDead();
+
auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg());
- if (TII->isVOP3(*I)) {
- I.addReg(TRI->getVCC(), RegState::Define);
- }
+ bool IsVOP3 = TII->isVOP3(*I);
+ if (IsVOP3)
+ I.addReg(TRI->getVCC(), RegState::Define | getDeadRegState(VCCDead));
+
I.add(MI.getOperand(1)).add(MI.getOperand(2));
if (NeedClampOperand)
I.addImm(0); // clamp bit for e64 encoding
+ if (!IsVOP3 && VCCDead)
+ I.setOperandDead(3);
+
TII->legalizeOperands(*I);
MI.eraseFromParent();
diff --git a/llvm/test/CodeGen/AMDGPU/finalize-isel-vadd-co-u32-dead-carry.mir b/llvm/test/CodeGen/AMDGPU/finalize-isel-vadd-co-u32-dead-carry.mir
new file mode 100644
index 00000000000000..bf6a1fd777d948
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/finalize-isel-vadd-co-u32-dead-carry.mir
@@ -0,0 +1,72 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu8.03 -passes=finalize-isel -o - %s | FileCheck %s
+
+# The V_ADD_CO/V_SUB_CO_U32_e32 custom inserter rebuilds the
+# instruction; the carry-out ($vcc) dead flag should be preserved
+
+---
+name: add_co_u32_e32_dead_vcc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: add_co_u32_e32_dead_vcc
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[COPY]], [[COPY1]], implicit-def dead $vcc, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e32_]]
+ ; CHECK-NEXT: SI_RETURN_TO_EPILOG $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_ADD_CO_U32_e32 %0, %1, implicit-def dead $vcc, implicit $exec
+ $vgpr0 = COPY %2
+ SI_RETURN_TO_EPILOG $vgpr0
+...
+
+---
+name: sub_co_u32_e32_dead_vcc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: sub_co_u32_e32_dead_vcc
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_SUB_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_SUB_CO_U32_e32 [[COPY]], [[COPY1]], implicit-def dead $vcc, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = COPY [[V_SUB_CO_U32_e32_]]
+ ; CHECK-NEXT: SI_RETURN_TO_EPILOG $vgpr0
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_SUB_CO_U32_e32 %0, %1, implicit-def dead $vcc, implicit $exec
+ $vgpr0 = COPY %2
+ SI_RETURN_TO_EPILOG $vgpr0
+...
+
+---
+name: add_co_u32_e32_live_vcc
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1
+ ; CHECK-LABEL: name: add_co_u32_e32_live_vcc
+ ; CHECK: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[COPY]], [[COPY1]], implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: [[V_ADDC_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADDC_U32_e32 [[COPY]], [[COPY1]], implicit-def dead $vcc, implicit $vcc, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = COPY [[V_ADD_CO_U32_e32_]]
+ ; CHECK-NEXT: $vgpr1 = COPY [[V_ADDC_U32_e32_]]
+ ; CHECK-NEXT: SI_RETURN_TO_EPILOG $vgpr0, implicit $vgpr1
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:vgpr_32 = V_ADD_CO_U32_e32 %0, %1, implicit-def $vcc, implicit $exec
+ %3:vgpr_32 = V_ADDC_U32_e32 %0, %1, implicit-def dead $vcc, implicit $vcc, implicit $exec
+ $vgpr0 = COPY %2
+ $vgpr1 = COPY %3
+ SI_RETURN_TO_EPILOG $vgpr0, implicit $vgpr1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 9018259618f824..74ee61b676f0a7 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -2369,9 +2369,8 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) {
; GCN-NEXT: v_max_i32_e32 v0, v2, v0
; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0
; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0
-; GCN-NEXT: s_mov_b32 s4, 0x8000
-; GCN-NEXT: v_rcp_f32_e32 v2, v2
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GCN-NEXT: v_rcp_f32_e32 v2, v2
; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2
; GCN-NEXT: v_mul_lo_u32 v3, v3, v2
@@ -2380,7 +2379,7 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) {
; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2
; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0
; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2
-; GCN-NEXT: v_sub_i32_e32 v3, vcc, s4, v3
+; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0x8000, v3
; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0
; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0
@@ -2401,9 +2400,8 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0
; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0
; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0
-; GCN-IR-NEXT: s_mov_b32 s4, 0x8000
-; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2
; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2
; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2
; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2
@@ -2412,7 +2410,7 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) {
; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 17, v2
; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0
; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2
-; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, s4, v3
+; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0x8000, v3
; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0
; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0
More information about the llvm-commits
mailing list