[llvm] [AMDGPU] Keep i64 carry chains on VCC when feeding VALU users (PR #204502)
Shilei Tian via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 18 10:20:54 PDT 2026
https://github.com/shiltian updated https://github.com/llvm/llvm-project/pull/204502
>From 872f3f02280f2dd569d9a5f1116dd943e27cd344 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i at tianshilei.me>
Date: Wed, 17 Jun 2026 22:01:14 -0400
Subject: [PATCH] [AMDGPU] Keep i64 carry chains on VCC when feeding VALU users
This PR fixes an issue where ISel could mix scalar and vector carry chains when
lowering widened integer add/sub operations. A scalar-looking i64 carry producer
may feed a divergent carry consumer, so ISel now keeps that carry chain on VCC
to avoid invalid MIR.
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 38 ++++++++++-
.../CodeGen/AMDGPU/i128-add-carry-chain.ll | 65 +++++++++++++++++++
2 files changed, 101 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/i128-add-carry-chain.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 76f6f091eeeec..d9765e6ca2377 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -1080,8 +1080,42 @@ void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
{{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
{AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
- unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
- unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
+ // The carry is passed between the low and high halves of this i64 add/sub,
+ // and between separate add/sub nodes chained through the carry glue (as
+ // happens when a wider integer add/sub is expanded into several i64
+ // ADDC/ADDE nodes), via either SCC or VCC. Every node in such a glue chain
+ // must agree on which register holds the carry; otherwise the carry crosses
+ // the SALU/VALU boundary with no copy, producing a use of an undefined VCC.
+ // Select the VALU variant if this node is divergent or if any later
+ // carry-in node in this glue chain needs the VALU carry path.
+ unsigned ScalarCarryOpc = OpcMap[1][0][IsAdd];
+ unsigned VectorCarryOpc = OpcMap[1][1][IsAdd];
+ unsigned CarryInOpc = IsAdd ? ISD::ADDE : ISD::SUBE;
+
+ bool IsVALU = N->isDivergent();
+ if (ProduceCarry && !IsVALU) {
+ for (SDNode *User = N->getGluedUser(); User; User = User->getGluedUser()) {
+ if (!User->isMachineOpcode()) {
+ assert(User->getOpcode() == CarryInOpc && "unexpected carry user");
+ if (User->isDivergent()) {
+ IsVALU = true;
+ break;
+ }
+ continue;
+ }
+
+ unsigned UserOpc = User->getMachineOpcode();
+ assert((UserOpc == ScalarCarryOpc || UserOpc == VectorCarryOpc) &&
+ "unexpected selected carry user");
+ if (UserOpc == VectorCarryOpc) {
+ IsVALU = true;
+ break;
+ }
+ }
+ }
+
+ unsigned Opc = OpcMap[0][IsVALU][IsAdd];
+ unsigned CarryOpc = OpcMap[1][IsVALU][IsAdd];
SDNode *AddLo;
if (!ConsumeCarry) {
diff --git a/llvm/test/CodeGen/AMDGPU/i128-add-carry-chain.ll b/llvm/test/CodeGen/AMDGPU/i128-add-carry-chain.ll
new file mode 100644
index 0000000000000..d226b953829e2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/i128-add-carry-chain.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s
+
+define i128 @i128_add_uniform_carry_to_divergent(i128 %x) #0 {
+; CHECK-LABEL: i128_add_uniform_carry_to_divergent:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s16, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b64 s[18:19], -1
+; CHECK-NEXT: buffer_store_dword v42, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[18:19]
+; CHECK-NEXT: v_writelane_b32 v42, s16, 3
+; CHECK-NEXT: s_add_i32 s32, s32, 0x400
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v41, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: v_writelane_b32 v42, s34, 0
+; CHECK-NEXT: v_writelane_b32 v42, s30, 1
+; CHECK-NEXT: v_writelane_b32 v42, s31, 2
+; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: s_mov_b32 s34, 32
+; CHECK-NEXT: v_lshlrev_b64 v[40:41], s34, v[0:1]
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, external at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, external at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; CHECK-NEXT: s_mov_b64 s[4:5], 1
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
+; CHECK-NEXT: v_mov_b32_e32 v0, s4
+; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, s6, v0
+; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: v_mov_b32_e32 v2, s7
+; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v1, vcc
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_addc_co_u32_e32 v2, vcc, v40, v2, vcc
+; CHECK-NEXT: v_mov_b32_e32 v3, s7
+; CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, v41, v3, vcc
+; CHECK-NEXT: v_lshrrev_b64 v[4:5], s34, v[0:1]
+; CHECK-NEXT: v_lshrrev_b64 v[6:7], s34, v[2:3]
+; CHECK-NEXT: v_mov_b32_e32 v1, v4
+; CHECK-NEXT: v_mov_b32_e32 v3, v6
+; CHECK-NEXT: v_readlane_b32 s30, v42, 1
+; CHECK-NEXT: v_readlane_b32 s31, v42, 2
+; CHECK-NEXT: v_readlane_b32 s34, v42, 0
+; CHECK-NEXT: buffer_load_dword v41, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s4, v42, 3
+; CHECK-NEXT: s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[6:7]
+; CHECK-NEXT: s_mov_b32 s33, s4
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %shift = shl i128 %x, 96
+ %call = call i32 @external()
+ %add = add i128 %shift, 1
+ ret i128 %add
+}
+
+declare i32 @external()
+
+attributes #0 = { noinline optnone }
More information about the llvm-commits
mailing list