[llvm] 11ebc0e - [AMDGPU][GlobalISel] Select s16 G_MERGE_VALUES into wider scalars (#207999)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 21:51:13 PDT 2026
Author: Keshav Vinayak Jha
Date: 2026-07-15T10:21:08+05:30
New Revision: 11ebc0e6645c8c53c75bca93e9fddf0053507cc1
URL: https://github.com/llvm/llvm-project/commit/11ebc0e6645c8c53c75bca93e9fddf0053507cc1
DIFF: https://github.com/llvm/llvm-project/commit/11ebc0e6645c8c53c75bca93e9fddf0053507cc1.diff
LOG: [AMDGPU][GlobalISel] Select s16 G_MERGE_VALUES into wider scalars (#207999)
With real-true16 (default on `gfx11`/`gfx12`), a scalar `s16` is a
register type, so the `G_MERGE_VALUES` legality rule started accepting
scalar merges built from `s16` pieces, e.g. `s64 = G_MERGE_VALUES(4 x
s16)`. The instruction selector has no pattern for that shape. So this
aborts with `cannot select` on something like:
```mlir
define amdgpu_kernel void @k(ptr %p) {
store i136 0, ptr %p, align 8
ret void
}
```
Make it selectable rather than restricting legalization: Pack pairs of
`s16` with `S_PACK_LL_B32_B16` then `REG_SEQUENCE`.
Assisted-by: Claude
Signed-off-by: Keshav Vinayak Jha <keshavvinayakjha at gmail.com>
Added:
llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
llvm/test/CodeGen/AMDGPU/GlobalISel/select-merge-values-build-vector-s16.mir
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 9d44d4c5424a3..173d8a0c2b8ef 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -775,6 +775,41 @@ bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
return true;
}
+// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
+// s32 pieces into the destination with a REG_SEQUENCE.
+bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
+ MachineBasicBlock *BB = MI.getParent();
+ const DebugLoc &DL = MI.getDebugLoc();
+ Register DstReg = MI.getOperand(0).getReg();
+ const unsigned DstSize = MRI->getType(DstReg).getSizeInBits();
+ const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
+ const unsigned NumSrc = MI.getNumOperands() - 1;
+
+ // Pack each pair of s16 sources into an s32.
+ SmallVector<Register, 8> S32Regs;
+ for (unsigned I = 0; I != NumSrc; I += 2) {
+ Register S32 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ auto Pack = BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_PACK_LL_B32_B16), S32)
+ .addReg(MI.getOperand(I + 1).getReg())
+ .addReg(MI.getOperand(I + 2).getReg());
+ constrainSelectedInstRegOperands(*Pack, TII, TRI, RBI);
+ S32Regs.push_back(S32);
+ }
+
+ // Combine the s32 pieces into the destination with a REG_SEQUENCE.
+ const TargetRegisterClass *DstRC =
+ TRI.getRegClassForSizeOnBank(DstSize, *DstBank);
+ if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
+ return false;
+ ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, /*EltSize=*/4);
+ auto MIB = BuildMI(*BB, MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg);
+ for (unsigned I = 0, E = S32Regs.size(); I != E; ++I)
+ MIB.addReg(S32Regs[I]).addImm(SubRegs[I]);
+
+ MI.eraseFromParent();
+ return true;
+}
+
bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
MachineBasicBlock *BB = MI.getParent();
Register DstReg = MI.getOperand(0).getReg();
@@ -788,7 +823,19 @@ bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
MI.getNumOperands() == 3) {
return selectS16MergeToS32(MI);
}
- return selectImpl(MI, *CoverageInfo);
+ // With true16 a scalar s16 is a register type, so a scalar wider than 32
+ // bits can be built from s16 pieces.
+ bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
+ DstTy.getSizeInBits() % 32 == 0;
+
+ // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
+ if (IsWideS16Merge &&
+ RBI.getRegBank(DstReg, *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
+ return selectS16MergeToWide(MI);
+
+ // A VGPR wide s16 merge falls through to the generic path below.
+ if (!IsWideS16Merge)
+ return selectImpl(MI, *CoverageInfo);
}
const DebugLoc &DL = MI.getDebugLoc();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 19028ef58a166..bd8ec0a769398 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -102,6 +102,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;
bool selectG_EXTRACT(MachineInstr &I) const;
bool selectS16MergeToS32(MachineInstr &MI) const;
+ bool selectS16MergeToWide(MachineInstr &MI) const;
bool selectG_MERGE_VALUES(MachineInstr &I) const;
bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
bool selectG_BUILD_VECTOR(MachineInstr &I) const;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
new file mode 100644
index 0000000000000..fc0df6181b7d9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1150 < %s | FileCheck -check-prefix=GFX1150 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+define amdgpu_kernel void @store_i136_zero(ptr %p) {
+; GFX1150-LABEL: store_i136_zero:
+; GFX1150: ; %bb.0:
+; GFX1150-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX1150-NEXT: s_pack_ll_b32_b16 s0, 0, 0
+; GFX1150-NEXT: v_mov_b32_e32 v6, 0
+; GFX1150-NEXT: s_mov_b32 s1, s0
+; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1150-NEXT: s_mov_b64 s[2:3], s[0:1]
+; GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX1150-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1150-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX1150-NEXT: s_clause 0x1
+; GFX1150-NEXT: flat_store_b128 v[4:5], v[0:3]
+; GFX1150-NEXT: flat_store_b8 v[4:5], v6 offset:16
+; GFX1150-NEXT: s_endpgm
+ store i136 0, ptr %p, align 8
+ ret void
+}
+
+define amdgpu_kernel void @store_i136_divergent(ptr %p) {
+; GFX1150-LABEL: store_i136_divergent:
+; GFX1150: ; %bb.0:
+; GFX1150-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1150-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1150-NEXT: v_mov_b32_e32 v2, 0
+; GFX1150-NEXT: v_mov_b32_e32 v6, 0
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1150-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX1150-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1150-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX1150-NEXT: v_lshlrev_b16 v4.l, 8, v1.l
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1150-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX1150-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX1150-NEXT: v_or_b16 v0.l, v0.l, v4.l
+; GFX1150-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1150-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
+; GFX1150-NEXT: s_clause 0x1
+; GFX1150-NEXT: flat_store_b128 v[4:5], v[0:3]
+; GFX1150-NEXT: flat_store_b8 v[4:5], v6 offset:16
+; GFX1150-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %v = trunc i32 %tid to i16
+ %z = zext i16 %v to i136
+ store i136 %z, ptr %p, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-merge-values-build-vector-s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-merge-values-build-vector-s16.mir
index e607a84507902..6e2b8cf4db8d9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/select-merge-values-build-vector-s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/select-merge-values-build-vector-s16.mir
@@ -137,3 +137,95 @@ body: |
%4:vgpr_32(<2 x s16>) = G_BUILD_VECTOR %2:vgpr_16(s16), %3:vgpr_16(s16)
$vgpr0 = COPY %4:vgpr_32(<2 x s16>)
...
+
+---
+name: test_merge_values_s16_into_s64_sgpr
+legalized: true
+regBankSelected: true
+
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+
+ ; GFX11-LABEL: name: test_merge_values_s16_into_s64_sgpr
+ ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX11-NEXT: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[COPY]], [[COPY1]]
+ ; GFX11-NEXT: [[S_PACK_LL_B32_B16_1:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[COPY2]], [[COPY3]]
+ ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_PACK_LL_B32_B16_]], %subreg.sub0, [[S_PACK_LL_B32_B16_1]], %subreg.sub1
+ ; GFX11-NEXT: $sgpr0_sgpr1 = COPY [[REG_SEQUENCE]]
+ ;
+ ; GFX12-LABEL: name: test_merge_values_s16_into_s64_sgpr
+ ; GFX12: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[S_PACK_LL_B32_B16_:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[COPY]], [[COPY1]]
+ ; GFX12-NEXT: [[S_PACK_LL_B32_B16_1:%[0-9]+]]:sreg_32 = S_PACK_LL_B32_B16 [[COPY2]], [[COPY3]]
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_PACK_LL_B32_B16_]], %subreg.sub0, [[S_PACK_LL_B32_B16_1]], %subreg.sub1
+ ; GFX12-NEXT: $sgpr0_sgpr1 = COPY [[REG_SEQUENCE]]
+ %0:sreg_32(s32) = COPY $sgpr0
+ %1:sreg_32(s32) = COPY $sgpr1
+ %2:sreg_32(s32) = COPY $sgpr2
+ %3:sreg_32(s32) = COPY $sgpr3
+ %4:sreg_32(s16) = G_TRUNC %0:sreg_32(s32)
+ %5:sreg_32(s16) = G_TRUNC %1:sreg_32(s32)
+ %6:sreg_32(s16) = G_TRUNC %2:sreg_32(s32)
+ %7:sreg_32(s16) = G_TRUNC %3:sreg_32(s32)
+ %8:sreg_64(s64) = G_MERGE_VALUES %4:sreg_32(s16), %5:sreg_32(s16), %6:sreg_32(s16), %7:sreg_32(s16)
+ $sgpr0_sgpr1 = COPY %8:sreg_64(s64)
+...
+
+---
+name: test_merge_values_s16_into_s64_vgpr
+legalized: true
+regBankSelected: true
+
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+ ; GFX11-LABEL: name: test_merge_values_s16_into_s64_vgpr
+ ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: {{ $}}
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+ ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
+ ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[COPY3]].lo16
+ ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[COPY5]], %subreg.hi16, [[COPY6]], %subreg.sub1_lo16, [[COPY7]], %subreg.sub1_hi16
+ ; GFX11-NEXT: $vgpr0_vgpr1 = COPY [[REG_SEQUENCE]]
+ ;
+ ; GFX12-LABEL: name: test_merge_values_s16_into_s64_vgpr
+ ; GFX12: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[COPY3]].lo16
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[COPY5]], %subreg.hi16, [[COPY6]], %subreg.sub1_lo16, [[COPY7]], %subreg.sub1_hi16
+ ; GFX12-NEXT: $vgpr0_vgpr1 = COPY [[REG_SEQUENCE]]
+ %0:vgpr_32(s32) = COPY $vgpr0
+ %1:vgpr_32(s32) = COPY $vgpr1
+ %2:vgpr_32(s32) = COPY $vgpr2
+ %3:vgpr_32(s32) = COPY $vgpr3
+ %4:vgpr_16(s16) = G_TRUNC %0:vgpr_32(s32)
+ %5:vgpr_16(s16) = G_TRUNC %1:vgpr_32(s32)
+ %6:vgpr_16(s16) = G_TRUNC %2:vgpr_32(s32)
+ %7:vgpr_16(s16) = G_TRUNC %3:vgpr_32(s32)
+ %8:vreg_64(s64) = G_MERGE_VALUES %4:vgpr_16(s16), %5:vgpr_16(s16), %6:vgpr_16(s16), %7:vgpr_16(s16)
+ $vgpr0_vgpr1 = COPY %8:vreg_64(s64)
+...
More information about the llvm-commits
mailing list