[llvm-branch-commits] [llvm] [AMDGPU] Fold fmul with zero checks to fmul.legacy (PR #229357)
Harrison Hao via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Oct 20 20:14:05 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/229357
>From e3b261a4c71295ea8988e64e1db410b7013e4ac8 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Tue, 6 Oct 2026 17:10:24 +0800
Subject: [PATCH 1/2] [AMDGPU] Fold fmul with zero checks to fmul.legacy
Recognize f32 multiplication with zero checks in AMDGPUCodeGenPrepare,
including fabs/fneg wrappers.
Example:
```
(x == 0 || y == 0) ? +0.0 : x * y
==>
llvm.amdgcn.fmul.legacy(x, y)
```
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 77 +++++++++++
.../amdgpu-codegenprepare-fmul-legacy.ll | 128 +++---------------
2 files changed, 98 insertions(+), 107 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 4b669756d83f63..7fc17075a15cba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -177,6 +177,9 @@ class AMDGPUCodeGenPrepareImpl
/// we expand some divisions here, we need to perform this before obscuring.
bool foldBinOpIntoSelect(BinaryOperator &I) const;
+ /// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y).
+ bool foldSelectToFMulLegacy(SelectInst &I);
+
bool divHasSpecialOptimization(BinaryOperator &I,
Value *Num, Value *Den) const;
unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
@@ -1578,11 +1581,85 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
return false;
}
+// select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y)
+bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
+ if (!I.getType()->isFloatTy())
+ return false;
+
+ // Inverted form: select (X != 0 && Y != 0), X * Y, +0.0.
+ bool Inverted = match(I.getFalseValue(), m_PosZeroFP());
+ if (!Inverted && !match(I.getTrueValue(), m_PosZeroFP()))
+ return false;
+
+ Value *X, *Y;
+ if (!match(Inverted ? I.getTrueValue() : I.getFalseValue(),
+ m_OneUse(m_FMul(m_Value(X), m_Value(Y)))))
+ return false;
+
+ SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}};
+ SmallPtrSet<Value *, 4> Leaves[2];
+ for (auto [Worklist, Found] : zip(Roots, Leaves)) {
+ SmallPtrSet<Value *, 8> Visited;
+
+ while (!Worklist.empty()) {
+ Value *Cur = Worklist.pop_back_val();
+ if (!Visited.insert(Cur).second)
+ continue;
+
+ Value *Src, *LHS, *RHS;
+ if (Cur->getType()->isIntegerTy(1)) {
+ if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS)))
+ : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) {
+ Worklist.append({LHS, RHS});
+ continue;
+ }
+
+ if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
+ : FCmpInst::FCMP_OEQ,
+ m_Value(Src), m_AnyZeroFP())))
+ return false;
+
+ Worklist.push_back(Src);
+ continue;
+ }
+
+ // Peeling a nnan/ninf op could expose poison that the select hides.
+ auto *FPOp = dyn_cast<FPMathOperator>(Cur);
+ if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) {
+ Found.insert(Cur);
+ continue;
+ }
+
+ // m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero.
+ if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) {
+ Worklist.push_back(Src);
+ continue;
+ }
+
+ Found.insert(Cur);
+ }
+ }
+
+ if (Leaves[0] != Leaves[1])
+ return false;
+
+ IRBuilder<> Builder(&I);
+ Value *LegacyMul =
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y});
+ LegacyMul->takeName(&I);
+ I.replaceAllUsesWith(LegacyMul);
+ DeadVals.push_back(&I);
+ return true;
+}
+
bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
FPMathOperator *FPOp = dyn_cast<FPMathOperator>(&I);
if (!FPOp)
return false;
+ if (foldSelectToFMulLegacy(I))
+ return true;
+
Value *X;
Value *Fract = nullptr;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
index 8731f24dbd3aae..c0a5d20503f7a1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
@@ -4,11 +4,7 @@
define float @select_fmul_legacy(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -22,11 +18,7 @@ define float @select_fmul_legacy(float %x, float %y) {
define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) {
; CHECK-LABEL: define float @select_fmul_legacy_uniform(
; CHECK-SAME: float inreg [[X:%.*]], float inreg [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -40,11 +32,7 @@ define float @select_fmul_legacy_uniform(float inreg %x, float inreg %y) {
define float @select_fmul_legacy_commute_fmul(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_commute_fmul(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[Y]], [[X]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[Y]], float [[X]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -58,11 +46,7 @@ define float @select_fmul_legacy_commute_fmul(float %x, float %y) {
define float @select_fmul_legacy_commute_or(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_commute_or(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[YZ]], [[XZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -76,11 +60,7 @@ define float @select_fmul_legacy_commute_or(float %x, float %y) {
define float @select_fmul_legacy_cmp_neg_zero(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_cmp_neg_zero(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], -0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], -0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, -0.0
@@ -189,11 +169,7 @@ define float @select_fmul_legacy_logical_or(float %x, float %y) {
define float @select_fmul_legacy_inverted(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_inverted(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xnz = fcmp une float %x, 0.0
@@ -438,9 +414,7 @@ define <2 x float> @select_fmul_legacy_v2f32(<2 x float> %x, <2 x float> %y) {
define float @select_fmul_legacy_square(float %x) {
; CHECK-LABEL: define float @select_fmul_legacy_square(
; CHECK-SAME: float [[X:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[XZ]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -452,11 +426,7 @@ define float @select_fmul_legacy_square(float %x) {
define float @select_fmul_legacy_square_or(float %x) {
; CHECK-LABEL: define float @select_fmul_legacy_square_or(
; CHECK-SAME: float [[X:%.*]]) {
-; CHECK-NEXT: [[XZ0:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[XZ1:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ0]], [[XZ1]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[X]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[X]])
; CHECK-NEXT: ret float [[R]]
;
%xz0 = fcmp oeq float %x, 0.0
@@ -470,12 +440,7 @@ define float @select_fmul_legacy_square_or(float %x) {
define float @select_fmul_legacy_redundant_cmp(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_redundant_cmp(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C0:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[C:%.*]] = or i1 [[C0]], [[XZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -490,12 +455,7 @@ define float @select_fmul_legacy_redundant_cmp(float %x, float %y) {
define float @select_fmul_legacy_shared_or(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_shared_or(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C0:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[C:%.*]] = or i1 [[C0]], [[C0]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -582,12 +542,8 @@ define float @select_fmul_legacy_const_operand(float %x) {
define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fabs_mul_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FABS_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FABS_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -602,12 +558,7 @@ define float @select_fmul_legacy_fabs_mul_operand(float %x, float %y) {
define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fabs_cmp_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -622,12 +573,8 @@ define float @select_fmul_legacy_fabs_cmp_operand(float %x, float %y) {
define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fneg_mul_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -642,12 +589,7 @@ define float @select_fmul_legacy_fneg_mul_operand(float %x, float %y) {
define float @select_fmul_legacy_fneg_cmp_operand(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fneg_cmp_operand(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[FNEG_X:%.*]] = fneg float [[X]]
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FNEG_X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fneg.x = fneg float %x
@@ -664,11 +606,7 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) {
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
; CHECK-NEXT: [[FNEG_FABS_X:%.*]] = fneg float [[FABS_X]]
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[FABS_X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_FABS_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_FABS_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -684,12 +622,8 @@ define float @select_fmul_legacy_fneg_fabs(float %x, float %y) {
define float @select_fmul_legacy_fneg_nsz(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fneg_nsz(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
; CHECK-NEXT: [[FNEG_X:%.*]] = fneg nsz float [[X]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[FNEG_X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[FNEG_X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -788,11 +722,7 @@ define float @select_fmul_legacy_canonicalize(float %x, float %y) {
define float @select_fmul_legacy_select_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_select_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select nsz i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -807,11 +737,7 @@ define float @select_fmul_legacy_select_flags(float %x, float %y) {
define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_select_nnan_ninf(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select nnan ninf i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -825,11 +751,7 @@ define float @select_fmul_legacy_select_nnan_ninf(float %x, float %y) {
define float @select_fmul_legacy_fmul_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fmul_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul nnan ninf float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
@@ -843,11 +765,7 @@ define float @select_fmul_legacy_fmul_flags(float %x, float %y) {
define float @select_fmul_legacy_fcmp_flags(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_fcmp_flags(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp nnan oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp nnan oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp nnan oeq float %x, 0.0
@@ -861,11 +779,7 @@ define float @select_fmul_legacy_fcmp_flags(float %x, float %y) {
define float @select_fmul_legacy_daz(float %x, float %y) #0 {
; CHECK-LABEL: define float @select_fmul_legacy_daz(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[XZ:%.*]] = fcmp oeq float [[X]], 0.000000e+00
-; CHECK-NEXT: [[YZ:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float 0.000000e+00, float [[MUL]]
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%xz = fcmp oeq float %x, 0.0
>From fe93783ea4c1291e6b534896eb858cbed259ba02 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Wed, 21 Oct 2026 11:00:52 +0800
Subject: [PATCH 2/2] Update for comments
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 49 +++++++++++--------
.../amdgpu-codegenprepare-fmul-legacy.ll | 7 +--
2 files changed, 29 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 7fc17075a15cba..2216c8b1acb96a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -178,7 +178,7 @@ class AMDGPUCodeGenPrepareImpl
bool foldBinOpIntoSelect(BinaryOperator &I) const;
/// Fold select (X == 0 || Y == 0), +0.0, X * Y into fmul_legacy(X, Y).
- bool foldSelectToFMulLegacy(SelectInst &I);
+ bool foldSelectToFMulLegacy(SelectInst &I) const;
bool divHasSpecialOptimization(BinaryOperator &I,
Value *Num, Value *Den) const;
@@ -1582,7 +1582,7 @@ bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
}
// select (X == 0 || Y == 0), +0.0, (X * Y) -> fmul_legacy(X, Y)
-bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
+bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) const {
if (!I.getType()->isFloatTy())
return false;
@@ -1596,7 +1596,30 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
m_OneUse(m_FMul(m_Value(X), m_Value(Y)))))
return false;
- SmallVector<Value *, 4> Roots[2] = {{I.getCondition()}, {X, Y}};
+ // Collect the values compared against zero in the condition.
+ SmallVector<Value *, 4> CondWorklist = {I.getCondition()};
+ SmallVector<Value *, 4> CmpSrcs;
+ SmallPtrSet<Value *, 8> VisitedConds;
+ while (!CondWorklist.empty()) {
+ Value *Cond = CondWorklist.pop_back_val();
+ if (!VisitedConds.insert(Cond).second)
+ continue;
+
+ Value *LHS, *RHS, *Src;
+ if (Inverted ? match(Cond, m_And(m_Value(LHS), m_Value(RHS)))
+ : match(Cond, m_Or(m_Value(LHS), m_Value(RHS)))) {
+ CondWorklist.append({LHS, RHS});
+ continue;
+ }
+
+ if (!match(Cond, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
+ : FCmpInst::FCMP_OEQ,
+ m_Value(Src), m_AnyZeroFP())))
+ return false;
+ CmpSrcs.push_back(Src);
+ }
+
+ SmallVector<Value *, 4> Roots[2] = {std::move(CmpSrcs), {X, Y}};
SmallPtrSet<Value *, 4> Leaves[2];
for (auto [Worklist, Found] : zip(Roots, Leaves)) {
SmallPtrSet<Value *, 8> Visited;
@@ -1606,23 +1629,6 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
if (!Visited.insert(Cur).second)
continue;
- Value *Src, *LHS, *RHS;
- if (Cur->getType()->isIntegerTy(1)) {
- if (Inverted ? match(Cur, m_And(m_Value(LHS), m_Value(RHS)))
- : match(Cur, m_Or(m_Value(LHS), m_Value(RHS)))) {
- Worklist.append({LHS, RHS});
- continue;
- }
-
- if (!match(Cur, m_SpecificFCmp(Inverted ? FCmpInst::FCMP_UNE
- : FCmpInst::FCMP_OEQ,
- m_Value(Src), m_AnyZeroFP())))
- return false;
-
- Worklist.push_back(Src);
- continue;
- }
-
// Peeling a nnan/ninf op could expose poison that the select hides.
auto *FPOp = dyn_cast<FPMathOperator>(Cur);
if (FPOp && (FPOp->hasNoNaNs() || FPOp->hasNoInfs())) {
@@ -1630,6 +1636,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
continue;
}
+ Value *Src;
// m_UnOp, not m_FNeg: fsub -0.0, X may flush a denormal result to zero.
if (match(Cur, m_CombineOr(m_UnOp(m_Value(Src)), m_FAbs(m_Value(Src))))) {
Worklist.push_back(Src);
@@ -1645,7 +1652,7 @@ bool AMDGPUCodeGenPrepareImpl::foldSelectToFMulLegacy(SelectInst &I) {
IRBuilder<> Builder(&I);
Value *LegacyMul =
- Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {}, {X, Y});
+ Builder.CreateIntrinsic(Intrinsic::amdgcn_fmul_legacy, {X, Y});
LegacyMul->takeName(&I);
I.replaceAllUsesWith(LegacyMul);
DeadVals.push_back(&I);
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
index c0a5d20503f7a1..9d98ad7645bcda 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fmul-legacy.ll
@@ -240,12 +240,7 @@ define float @select_fmul_legacy_inverted_logical_and(float %x, float %y) {
define float @select_fmul_legacy_inverted_fabs(float %x, float %y) {
; CHECK-LABEL: define float @select_fmul_legacy_inverted_fabs(
; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]]) {
-; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT: [[XNZ:%.*]] = fcmp une float [[FABS_X]], 0.000000e+00
-; CHECK-NEXT: [[YNZ:%.*]] = fcmp une float [[Y]], 0.000000e+00
-; CHECK-NEXT: [[C:%.*]] = and i1 [[XNZ]], [[YNZ]]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[X]], [[Y]]
-; CHECK-NEXT: [[R:%.*]] = select i1 [[C]], float [[MUL]], float 0.000000e+00
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[R]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
More information about the llvm-branch-commits
mailing list