[llvm-branch-commits] [llvm] [AMDGPU] Fold fmul with zero checks to fmul.legacy (PR #229357)
Harrison Hao via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 21 02:16:39 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/229357
>From 7d5adbdd46ffb1ec2a6879bcb1bd9ca21f2547bc Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 6 Oct 2026 17:10:24 +0800
Subject: [PATCH 1/3] [AMDGPU] Fold fmul with zero checks to fmul.legacy
Recognize f32 multiplication with zero checks in AMDGPUCodeGenPrepare,
including fabs/fneg wrappers.
Example:
```
(x == 0 || y == 0) ? +0.0 : x * y
==>
llvm.amdgcn.fmul.legacy(x, y)
```
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 77 ++++++++++++
.../amdgpu-codegenprepare-fmul-legacy.ll | 114 +++---------------
2 files changed, 96 insertions(+), 95 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 4b669756d83f6..7fc17075a15cb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -177,6 +177,9 @@ class AMDGPUCodeGenPrepareImpl
/// we expand some divisions here, we need to perform this before obscuring.
bool foldBinOpIntoSelect(BinaryOperator &I) const;
+ /// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y).
+ bool foldSelectToFMulLegacy(SelectInst &I);
+
bool divHasSpecialOptimization(BinaryOperator &I,
Value *Num, Value *Den) const;
unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
@@ -1578,11 +1581,85 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
return false;
}
+// select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y)
+bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
+ if (!I.getType()->isFloatTy())
+ return false;
+
+ // Inverted form: select (X != 0 && Y != 0), X * Y, +0.0.
+ bool Inverted = match(I.getFalseValue(), m_PosZeroFP());
+ if (!Inverted && !match(I.getTrueValue(), m_PosZeroFP()))
+ return false;
+
+ Value *X, *Y;
+ if (!match(Inverted ? I.getTrueValue() : I.getFalseValue(),
+ m_OneUse(m_FMul(m_Value(X), m_Value(Y)))))
+ return false;
+
+ SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}};
+ SmallPtrSet<Value *, 4> Leaves[2];
+ for (auto [Worklist, Found] : zip(Roots, Leaves)) {
+ SmallPtrSet<Value *, 8> Visited;
+
+ while (!Worklist.empty()) {
+ Value *Cur = Worklist.pop_back_val();
+ if (!Visited.insert(Cur).second)
+ continue;
+
+ Value *Src, *LHS, *RHS;
+ if (Cur->getType()->isIntegerTy(1)) {
+ if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS)))
+ : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) {
+ Worklist.append({LHS, RHS});
+ continue;
+ }
+
+ if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
+ : FCmpInst::FCMP_OEQ,
+ m_Value(Src), m_AnyZeroFP())))
+ return false;
+
+ Worklist.push_back(Src);
+ continue;
+ }
+
+ // Peeling a nnan/ninf op could expose poison that the select hides.
+ auto *FPOp = dyn_cast<FPMathOperator>(Cur);
+ if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) {
+ Found.insert(Cur);
+ continue;
+ }
+
+ // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero.
+ if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) {
+ Worklist.push_back(Src);
+ continue;
+ }
+
+ Found.insert(Cur);
+ }
+ }
+
+ if (Leaves[0] != Leaves[1])
+ return false;
+
+ IRBuilder<> Builder(&I);
+ Value *LegacyMul =
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y});
+ LegacyMul->takeName(&I);
+ I.replaceAllUsesWith(LegacyMul);
+ DeadVals.push_back(&I);
+ return true;
+}
+
bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
FPMathOperator *FPOp = dyn_cast<FPMathOperator>(&I);
if (!FPOp)
return false;
+ if (foldSelectToFMulLegacy(I))
+ return true;
+
Value *X;
Value *Fract = nullptr;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
index 05b603a449c3f..09a1233cd1833 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
@@ -4,11 +4,7 @@
define float @select_fmul_legacy(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -22,11 +18,7 @@ define float @select_fmul_legacy(float %x, float %y) {
define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) {
; CHECK-LABEL: define float @select_fmul_legacy_uniform(
; CHECK-SAME: float inreg [[X:%.*]], float inreg [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -40,11 +32,7 @@ define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) {
define float @select_fmul_legacy_commute_fmul(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_commute_fmul(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[Y]], [[X]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[Y]], float [[X]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -58,11 +46,7 @@ define float @select_fmul_legacy_commute_fmul(float %x, float %y) {
define float @select_fmul_legacy_commute_or(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_commute_or(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[YZ]], [[XZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -76,11 +60,7 @@ define float @select_fmul_legacy_commute_or(float %x, float %y) {
define float @select_fmul_legacy_cmp_neg_zero(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_cmp_neg_zero(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], -0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], -0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, -0.0
@@ -189,11 +169,7 @@ define float @select_fmul_legacy_logical_or(float %x, float %y) {
define float @select_fmul_legacy_inverted(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_inverted(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xnz = fcmp une float %x, 0.0
@@ -438,9 +414,7 @@ define <2 x float> @select_fmul_legacy_v2f32(<2 x float> %x, <2 x float> %y) {
define float @select_fmul_legacy_square(float %x) {
; CHECK-LABEL: define float @select_fmul_legacy_square(
; CHECK-SAME: float [[X:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[XZ]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -452,11 +426,7 @@ define float @select_fmul_legacy_square(float %x) {
define float @select_fmul_legacy_square_or(float %x) {
; CHECK-LABEL: define float @select_fmul_legacy_square_or(
; CHECK-SAME: float [[X:%.*]]) {
-; CHECK-NEXT: [[XZ0:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[XZ1:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ0]], [[XZ1]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]])
; CHECK-NEXT: ret float [[R]]
;
%xz0 = fcmp oeq float %x, 0.0
@@ -542,12 +512,8 @@ define float @select_fmul_legacy_const_operand(float %x) {
define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fabs_mul_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FABS_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FABS_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -562,12 +528,7 @@ define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) {
define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fabs_cmp_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -582,12 +543,8 @@ define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) {
define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fneg_mul_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -602,12 +559,7 @@ define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) {
define float @select_fmul_legacy_fneg_cmp_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fneg_cmp_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]]
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FNEG_X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fneg.x = fneg float %x
@@ -624,11 +576,7 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) {
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
; CHECK-NEXT: [[FNEG_FABS_X:%.*]] = fneg float [[FABS_X]]
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_FABS_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_FABS_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -644,12 +592,8 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) {
define float @select_fmul_legacy_fneg_nsz(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fneg_nsz(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
; CHECK-NEXT: [[FNEG_X:%.*]] = fneg nsz float [[X]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -748,11 +692,7 @@ define float @select_fmul_legacy_copysign_mul_operand(float %x, float %y, float
define float @select_fmul_legacy_select_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_select_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select nsz i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -767,11 +707,7 @@ define float @select_fmul_legacy_select_flags(float %x, float %y) {
define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_select_nnan_ninf(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select nnan ninf i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -785,11 +721,7 @@ define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) {
define float @select_fmul_legacy_fmul_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fmul_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul nnan ninf float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -821,11 +753,7 @@ define float @select_fmul_legacy_fmul_fast(float %x, float %y) {
define float @select_fmul_legacy_fcmp_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fcmp_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp nnan oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp nnan oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp nnan oeq float %x, 0.0
@@ -839,11 +767,7 @@ define float @select_fmul_legacy_fcmp_flags(float %x, float %y) {
define float @select_fmul_legacy_daz(float %x, float %y) #0 {
; CHECK-LABEL: define float @select_fmul_legacy_daz(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
>From 2220b5b3fada3f461369731104927cc129819745 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Wed, 21 Oct 2026 11:00:52 +0800
Subject: [PATCH 2/3] Update for comments
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 49 +++++++++++--------
.../amdgpu-codegenprepare-fmul-legacy.ll | 7 +--
2 files changed, 29 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 7fc17075a15cb..2216c8b1acb96 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -178,7 +178,7 @@ class AMDGPUCodeGenPrepareImpl
bool foldBinOpIntoSelect(BinaryOperator &I) const;
/// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y).
- bool foldSelectToFMulLegacy(SelectInst &I);
+ bool foldSelectToFMulLegacy(SelectInst &I) const;
bool divHasSpecialOptimization(BinaryOperator &I,
Value *Num, Value *Den) const;
@@ -1582,7 +1582,7 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
}
// select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y)
-bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
+bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) const {
if (!I.getType()->isFloatTy())
return false;
@@ -1596,7 +1596,30 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
m_OneUse(m_FMul(m_Value(X), m_Value(Y)))))
return false;
- SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}};
+ // Collect the values compared against zero in the condition.
+ SmallVector<Value *, 4> CondWorklist = {I.getCondition()};
+ SmallVector<Value *, 4> CmpSrcs;
+ SmallPtrSet<Value *, 8> VisitedConds;
+ while (!CondWorklist.empty()) {
+ Value *Cond = CondWorklist.pop_back_val();
+ if (!VisitedConds.insert(Cond).second)
+ continue;
+
+ Value *LHS, *RHS, *Src;
+ if (Inverted ? match(Cond, m_And(m_Value(LHS), m_Value(RHS)))
+ : match(Cond, m_Or(m_Value(LHS), m_Value(RHS)))) {
+ CondWorklist.append({LHS, RHS});
+ continue;
+ }
+
+ if (!match(Cond, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
+ : FCmpInst::FCMP_OEQ,
+ m_Value(Src), m_AnyZeroFP())))
+ return false;
+ CmpSrcs.push_back(Src);
+ }
+
+ SmallVector<Value *, 4> Roots[2] = {std::move(CmpSrcs), {X, Y}};
SmallPtrSet<Value *, 4> Leaves[2];
for (auto [Worklist, Found] : zip(Roots, Leaves)) {
SmallPtrSet<Value *, 8> Visited;
@@ -1606,23 +1629,6 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
if (!Visited.insert(Cur).second)
continue;
- Value *Src, *LHS, *RHS;
- if (Cur->getType()->isIntegerTy(1)) {
- if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS)))
- : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) {
- Worklist.append({LHS, RHS});
- continue;
- }
-
- if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
- : FCmpInst::FCMP_OEQ,
- m_Value(Src), m_AnyZeroFP())))
- return false;
-
- Worklist.push_back(Src);
- continue;
- }
-
// Peeling a nnan/ninf op could expose poison that the select hides.
auto *FPOp = dyn_cast<FPMathOperator>(Cur);
if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) {
@@ -1630,6 +1636,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
continue;
}
+ Value *Src;
// m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero.
if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) {
Worklist.push_back(Src);
@@ -1645,7 +1652,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
IRBuilder<> Builder(&I);
Value *LegacyMul =
- Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y});
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y});
LegacyMul->takeName(&I);
I.replaceAllUsesWith(LegacyMul);
DeadVals.push_back(&I);
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
index 09a1233cd1833..a577113adc640 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
@@ -240,12 +240,7 @@ define float @select_fmul_legacy_inverted_logical_and(float %x, float %y) {
define float @select_fmul_legacy_inverted_fabs(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_inverted_fabs(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[FABS_X]], 0.000000e+00
-; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
>From 74fea14fa792211d1af8ea168f4ca7c5452222c7 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Wed, 21 Oct 2026 12:33:27 +0800
Subject: [PATCH 3/3] Remove worklist
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 79 +++++++------------
.../amdgpu-codegenprepare-fmul-legacy.ll | 8 +-
2 files changed, 30 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 2216c8b1acb96..7dd9e41c7cfa4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -32,6 +32,7 @@
#include "llvm/Pass.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/KnownFPClass.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/IntegerDivision.h"
#include "llvm/Transforms/Utils/Local.h"
@@ -1591,68 +1592,44 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) const {
if (!Inverted && !match(I.getTrueValue(), m_PosZeroFP()))
return false;
+ Value *Mul = Inverted ? I.getTrueValue() : I.getFalseValue();
Value *X, *Y;
- if (!match(Inverted ? I.getTrueValue() : I.getFalseValue(),
- m_OneUse(m_FMul(m_Value(X), m_Value(Y)))))
+ if (!match(Mul, m_OneUse(m_FMul(m_Value(X), m_Value(Y)))))
return false;
- // Collect the values compared against zero in the condition.
- SmallVector<Value *, 4> CondWorklist = {I.getCondition()};
- SmallVector<Value *, 4> CmpSrcs;
- SmallPtrSet<Value *, 8> VisitedConds;
- while (!CondWorklist.empty()) {
- Value *Cond = CondWorklist.pop_back_val();
- if (!VisitedConds.insert(Cond).second)
- continue;
-
- Value *LHS, *RHS, *Src;
- if (Inverted ? match(Cond, m_And(m_Value(LHS), m_Value(RHS)))
- : match(Cond, m_Or(m_Value(LHS), m_Value(RHS)))) {
- CondWorklist.append({LHS, RHS});
- continue;
- }
-
- if (!match(Cond, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
- : FCmpInst::FCMP_OEQ,
- m_Value(Src), m_AnyZeroFP())))
+ FCmpInst::Predicate ZeroCmpPred =
+ Inverted ? FCmpInst::FCMP_UNE : FCmpInst::FCMP_OEQ;
+ Value *A, *B;
+ auto ZeroCmpA = m_SpecificFCmp(ZeroCmpPred, m_Value(A), m_AnyZeroFP());
+ auto ZeroCmpB = m_SpecificFCmp(ZeroCmpPred, m_Value(B), m_AnyZeroFP());
+ Value *Cond = I.getCondition();
+ if (Inverted ? !match(Cond, m_And(ZeroCmpA, ZeroCmpB))
+ : !match(Cond, m_Or(ZeroCmpA, ZeroCmpB))) {
+ if (!match(Cond, ZeroCmpA))
return false;
- CmpSrcs.push_back(Src);
+ B = A;
}
- SmallVector<Value *, 4> Roots[2] = {std::move(CmpSrcs), {X, Y}};
- SmallPtrSet<Value *, 4> Leaves[2];
- for (auto [Worklist, Found] : zip(Roots, Leaves)) {
- SmallPtrSet<Value *, 8> Visited;
-
- while (!Worklist.empty()) {
- Value *Cur = Worklist.pop_back_val();
- if (!Visited.insert(Cur).second)
- continue;
-
- // Peeling a nnan/ninf op could expose poison that the select hides.
- auto *FPOp = dyn_cast<FPMathOperator>(Cur);
- if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) {
- Found.insert(Cur);
- continue;
- }
-
- Value *Src;
- // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero.
- if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) {
- Worklist.push_back(Src);
- continue;
- }
-
- Found.insert(Cur);
- }
- }
+ Value *SA = InstCombiner::stripSignOnlyFPOps(A);
+ Value *SB = InstCombiner::stripSignOnlyFPOps(B);
+ Value *SX = InstCombiner::stripSignOnlyFPOps(X);
+ Value *SY = InstCombiner::stripSignOnlyFPOps(Y);
+ if (!((SA == SX && SB == SY) || (SA == SY && SB == SX)))
+ return false;
- if (Leaves[0] != Leaves[1])
+ // A stripped nnan/ninf op could create poison that the select hides.
+ if (!impliesPoison(X, SX) || !impliesPoison(Y, SY))
return false;
+ // The select may hide NaN, Inf or a signed zero from the fmul.
+ FastMathFlags FMF = cast<Instruction>(Mul)->getFastMathFlags();
+ FMF.setNoNaNs(false);
+ FMF.setNoInfs(false);
+ FMF.setNoSignedZeros(I.hasNoSignedZeros());
+
IRBuilder<> Builder(&I);
Value *LegacyMul =
- Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y});
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y}, FMF);
LegacyMul->takeName(&I);
I.replaceAllUsesWith(LegacyMul);
DeadVals.push_back(&I);
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
index a577113adc640..224269bf3dfce 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
@@ -687,7 +687,7 @@ define float @select_fmul_legacy_copysign_mul_operand(float %x, float %y, float
define float @select_fmul_legacy_select_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_select_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
+; CHECK-NEXT: [[R:%.*]] = call nsz float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -730,11 +730,7 @@ define float @select_fmul_legacy_fmul_flags(float %x, float %y) {
define float @select_fmul_legacy_fmul_fast(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fmul_fast(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul fast float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call reassoc arcp contract afn float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
More information about the llvm-branch-commits
mailing list