[llvm] [AMDGPU][GlobalISel] Narrow 64-bit shifts when KnownBits proves the amount is >= 32 (PR #215500)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 02:06:02 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
Mirror SDAG performShlCombine/performSraCombine/performSrlCombine
---
Full diff: https://github.com/llvm/llvm-project/pull/215500.diff
6 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUCombine.td (+8-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp (+55)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h (+3)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll (+22)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll (+22)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll (+29)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 0c348bb647c0c..45df2875d2e3a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -203,6 +203,13 @@ def binop_s64_with_s32_mask_combines : GICombineGroup<[
combine_or_s64_with_s32_mask, combine_and_s64_with_s32_mask
]>;
+// Narrow a 64-bit shift to 32-bit when the amount is KnownBits->=32 but not constant.
+def shift_known_ge_half_width : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_SHL, G_LSHR, G_ASHR):$root,
+ [{ return Helper.matchShiftKnownGeHalfWidth(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
// (or i64:x, (zext i32:y)) -> i64:(merge (or lo_32(x), i32:y), hi_32(x))
// (or (zext i32:y), i64:x) -> i64:(merge (or lo_32(x), i32:y), hi_32(x))
def or_s64_zext_s32_frag : GICombinePatFrag<(outs root:$dst), (ins $src_s64, $src_s32),
@@ -256,7 +263,7 @@ def AMDGPUPostLegalizerCombiner: GICombiner<
[all_combines, gfx6gfx7_combines, gfx8_combines, combine_fmul_with_select_to_fldexp,
uchar_to_float, cvt_f32_ubyteN, remove_fcanonicalize, foldable_fneg,
fold_fabs_fptrunc, rcp_sqrt_to_rsq, fdiv_by_sqrt_to_rsq_f16,
- sign_extension_in_reg, smulu64,
+ sign_extension_in_reg, smulu64, shift_known_ge_half_width,
binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
index a325170e89e9b..aa11b4cef2d75 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
@@ -9,6 +9,7 @@
#include "AMDGPUCombinerHelper.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
@@ -557,3 +558,57 @@ bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const {
// Check if low 32 bits or high 32 bits are all ones.
return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen));
}
+
+// 64-bit shift is quarter rate on some subtargets, so splitting into a move
+// plus 32-bit shift is a win. Constant amounts are handled by the generic
+// matchCombineShiftToUnmerge combine already.
+bool AMDGPUCombinerHelper::matchShiftKnownGeHalfWidth(
+ MachineInstr &MI, BuildFnTy &MatchInfo) const {
+ unsigned Opc = MI.getOpcode();
+ assert(Opc == TargetOpcode::G_SHL || Opc == TargetOpcode::G_LSHR ||
+ Opc == TargetOpcode::G_ASHR);
+
+ Register Dst = MI.getOperand(0).getReg();
+ if (MRI.getType(Dst) != LLT::scalar(64))
+ return false;
+
+ Register ShiftAmt = MI.getOperand(2).getReg();
+ if (getIConstantVRegValWithLookThrough(ShiftAmt, MRI))
+ return false;
+
+ if (VT->getKnownBits(ShiftAmt).getMinValue().getZExtValue() < 32)
+ return false;
+
+ Register Src = MI.getOperand(1).getReg();
+ MatchInfo = [=, &MI](MachineIRBuilder &B) {
+ const LLT S32 = LLT::integer(32);
+ auto Unmerge = B.buildUnmerge(S32, Src);
+ Register Lo = Unmerge.getReg(0);
+ Register Hi = Unmerge.getReg(1);
+ auto MaskedAmt = B.buildAnd(S32, ShiftAmt, B.buildConstant(S32, 31));
+
+ switch (Opc) {
+ case TargetOpcode::G_SHL: {
+ auto Shl = B.buildShl(S32, Lo, MaskedAmt, MI.getFlags());
+ B.buildMergeLikeInstr(Dst, {B.buildConstant(S32, 0), Shl});
+ break;
+ }
+ case TargetOpcode::G_LSHR: {
+ auto Shr = B.buildLShr(S32, Hi, MaskedAmt, MI.getFlags());
+ B.buildMergeLikeInstr(Dst, {Shr, B.buildConstant(S32, 0)});
+ break;
+ }
+ case TargetOpcode::G_ASHR: {
+ auto FrozenHi = B.buildFreeze(S32, Hi);
+ auto NewLo = B.buildAShr(S32, FrozenHi, MaskedAmt, MI.getFlags());
+ auto NewHi = B.buildAShr(S32, FrozenHi, B.buildConstant(S32, 31));
+ B.buildMergeLikeInstr(Dst, {NewLo, NewHi});
+ break;
+ }
+ default:
+ llvm_unreachable("Unexpected opcode");
+ }
+ };
+
+ return true;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
index ffbb3a4e69305..d88c2151e955a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
@@ -48,6 +48,9 @@ class AMDGPUCombinerHelper : public CombinerHelper {
std::function<void(MachineIRBuilder &)> &MatchInfo) const;
bool matchConstantIs32BitMask(Register Reg) const;
+
+ // Narrow a 64-bit shift when the amount is KnownBits->=32 but not constant.
+ bool matchShiftKnownGeHalfWidth(MachineInstr &MI, BuildFnTy &MatchInfo) const;
};
} // namespace llvm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index 5a8027bf75881..19cf64121a29a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -1534,6 +1534,28 @@ define i64 @v_ashr_i64_31(i64 %value) {
ret i64 %result
}
+define i64 @v_ashr_i64_or32(i64 %value, i32 %amount) {
+; GCN-LABEL: v_ashr_i64_or32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v0, 31, v2
+; GCN-NEXT: v_ashrrev_i32_e32 v0, v0, v1
+; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10PLUS-LABEL: v_ashr_i64_or32:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, 31, v2
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v0, v0, v1
+; GFX10PLUS-NEXT: v_ashrrev_i32_e32 v1, 31, v1
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+ %amount.or = or i32 %amount, 32
+ %amount.ext = zext i32 %amount.or to i64
+ %result = ashr i64 %value, %amount.ext
+ ret i64 %result
+}
+
define amdgpu_ps i64 @s_ashr_i64(i64 inreg %value, i64 inreg %amount) {
; GCN-LABEL: s_ashr_i64:
; GCN: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index 344e16addf8a9..0aa9004ab6341 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -1502,6 +1502,28 @@ define i64 @v_lshr_i64_31(i64 %value) {
ret i64 %result
}
+define i64 @v_lshr_i64_or32(i64 %value, i32 %amount) {
+; GCN-LABEL: v_lshr_i64_or32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v0, 31, v2
+; GCN-NEXT: v_lshrrev_b32_e32 v0, v0, v1
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10PLUS-LABEL: v_lshr_i64_or32:
+; GFX10PLUS: ; %bb.0:
+; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10PLUS-NEXT: v_and_b32_e32 v0, 31, v2
+; GFX10PLUS-NEXT: v_lshrrev_b32_e32 v0, v0, v1
+; GFX10PLUS-NEXT: v_mov_b32_e32 v1, 0
+; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+ %amount.or = or i32 %amount, 32
+ %amount.ext = zext i32 %amount.or to i64
+ %result = lshr i64 %value, %amount.ext
+ ret i64 %result
+}
+
define amdgpu_ps i64 @s_lshr_i64(i64 inreg %value, i64 inreg %amount) {
; GCN-LABEL: s_lshr_i64:
; GCN: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index e4035042e9ca0..9a627a3bde9c9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -1458,6 +1458,35 @@ define i64 @v_shl_i64_31(i64 %value) {
ret i64 %result
}
+define i64 @v_shl_i64_or32(i64 %value, i32 %amount) {
+; GCN-LABEL: v_shl_i64_or32:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v1, 31, v2
+; GCN-NEXT: v_lshlrev_b32_e32 v1, v1, v0
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_shl_i64_or32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v1, 31, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_shl_i64_or32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v1, 31, v2
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_lshlrev_b32 v1, v1, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+ %amount.or = or i32 %amount, 32
+ %amount.ext = zext i32 %amount.or to i64
+ %result = shl i64 %value, %amount.ext
+ ret i64 %result
+}
+
define amdgpu_ps i64 @s_shl_i64(i64 inreg %value, i64 inreg %amount) {
; GCN-LABEL: s_shl_i64:
; GCN: ; %bb.0:
``````````
</details>
https://github.com/llvm/llvm-project/pull/215500
More information about the llvm-commits
mailing list