[llvm] [AMDGPU] Fix fmul/fma legacy sign-of-zero miscompile (PR #203567)
Wooseok Lee via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 06:52:42 PDT 2026
https://github.com/wooseoklee updated https://github.com/llvm/llvm-project/pull/203567
>From fe607e49173a86e61c36d02670f94b866f2909e5 Mon Sep 17 00:00:00 2001
From: wooseoklee <wolee at amd.com>
Date: Fri, 12 Jun 2026 10:10:13 -0500
Subject: [PATCH] [AMDGPU] Fix fmul/fma legacy sign-of-zero miscompile in
canSimplifyLegacyMulToMul
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
V_MUL_LEGACY_F32 always returns +0.0 when either operand is ±0.0,
while a regular IEEE fmul XORs operand signs. This makes the results
observably different when one operand is a runtime ±0.0:
fmul.legacy(-2.0, +0.0) = +0.0 (legacy zero clause)
fmul (-2.0, +0.0) = -0.0 (IEEE sign XOR: - * + = -)
canSimplifyLegacyMulToMul was incorrectly folding whenever one
operand matched m_FiniteNonZero() or both were known never Inf/NaN,
without accounting for the sign-of-zero divergence on the other operand.
Restructure the guard around the nsz fast-math flag:
- With nsz (sign-of-zero is don't-care): fold if one operand is
finite-nonzero (zero clause can only fire on the other side, but
nsz makes the +0 vs -0 difference irrelevant), or if both operands
are known never Inf/NaN (nsz covers ±0 sign, and 0*Inf/0*NaN = NaN
cases are excluded):
fmul.legacy nsz (-2.0, %x) -> fmul nsz (-2.0, %x)
- Without nsz: only fold if both operands are known non-zero
so the zero clause can never fire:
fmul.legacy(-2.0, 99.5) -> fmul(-2.0, 99.5) ; both nonzero, safe
fmul.legacy(-2.0, %x) -> not folded ; %x may be ±0
The same fix applies to fma.legacy via the shared helper. Update
fmul_legacy.ll and fma_legacy.ll accordingly.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 24 +++---
.../InstCombine/AMDGPU/fma_legacy.ll | 83 +++++++++++++++++--
.../InstCombine/AMDGPU/fmul_legacy.ll | 80 ++++++++++++++++--
3 files changed, 161 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 5788932ae5e2e..32277358f14d5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -404,17 +404,21 @@ bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
// The legacy behaviour is that multiplying +/-0.0 by anything, even NaN or
// infinity, gives +0.0. If we can prove we don't have one of the special
// cases then we can use a normal multiply instead.
- // TODO: Create and use isKnownFiniteNonZero instead of just matching
- // constants here.
- if (match(Op0, PatternMatch::m_FiniteNonZero()) ||
- match(Op1, PatternMatch::m_FiniteNonZero())) {
- // One operand is not zero or infinity or NaN.
- return true;
- }
-
SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(&I);
- if (isKnownNeverInfOrNaN(Op0, SQ) && isKnownNeverInfOrNaN(Op1, SQ)) {
- // Neither operand is infinity or NaN.
+ KnownFPClass Known0 = computeKnownFPClass(Op0, fcZero | fcInf | fcNan, SQ);
+ KnownFPClass Known1 = computeKnownFPClass(Op1, fcZero | fcInf | fcNan, SQ);
+ if (I.hasNoSignedZeros()) {
+ if ((Known0.isKnownNeverZero() && Known0.isKnownNeverInfOrNaN()) ||
+ (Known1.isKnownNeverZero() && Known1.isKnownNeverInfOrNaN())) {
+ // One operand is not zero or infinity or NaN.
+ return true;
+ }
+ if (Known0.isKnownNeverInfOrNaN() && Known1.isKnownNeverInfOrNaN()) {
+ // Neither operand is infinity or NaN.
+ return true;
+ }
+ } else if (Known0.isKnownNeverZero() && Known1.isKnownNeverZero()) {
+ // Without nsz, neither operand is zero.
return true;
}
return false;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
index 3efd68b9c1444..0da4e4731e93b 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
@@ -49,32 +49,44 @@ define float @test_negzero_nsz(float %y, float %z) {
ret float %call
}
-; Combine to fma because the constant is finite and non-zero.
+; Combine to fma with nsz: sign-of-zero is don't-care so the +0 vs +-0
+; difference between legacy and IEEE does not matter.
+define float @test_const_nsz(float %x, float %z) {
+; CHECK-LABEL: @test_const_nsz(
+; CHECK-NEXT: [[CALL:%.*]] = call nsz float @llvm.fma.f32(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call nsz float @llvm.amdgcn.fma.legacy(float %x, float 99.5, float %z)
+ ret float %call
+}
+
+; Do NOT combine: one operand is finite-nonzero but the other may be +-0 at
+; runtime. Legacy mul returns +0; IEEE fmul returns -0. Sign-of-zero mismatch.
define float @test_const(float %x, float %z) {
; CHECK-LABEL: @test_const(
-; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fma.legacy(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%call = call float @llvm.amdgcn.fma.legacy(float %x, float 99.5, float %z)
ret float %call
}
-; Combine to fma because the constant is finite and non-zero, preserving fmf.
+; Do NOT combine: contract does not imply nsz, sign-of-zero still matters.
define float @test_const_fmf(float %x, float %z) {
; CHECK-LABEL: @test_const_fmf(
-; CHECK-NEXT: [[CALL:%.*]] = call contract float @llvm.fma.f32(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call contract float @llvm.amdgcn.fma.legacy(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%call = call contract float @llvm.amdgcn.fma.legacy(float %x, float 99.5, float %z)
ret float %call
}
-; Combine to fma because neither argument can be infinity or NaN.
+; Do NOT combine: sitofp is finite but may be zero, sign-of-zero can diverge.
define float @test_finite(i32 %x, i32 %y, float %z) {
; CHECK-LABEL: @test_finite(
; CHECK-NEXT: [[XF:%.*]] = sitofp i32 [[X:%.*]] to float
; CHECK-NEXT: [[YF:%.*]] = sitofp i32 [[Y:%.*]] to float
-; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float [[XF]], float [[YF]], float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fma.legacy(float [[XF]], float [[YF]], float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%xf = sitofp i32 %x to float
@@ -83,7 +95,30 @@ define float @test_finite(i32 %x, i32 %y, float %z) {
ret float %call
}
-; Combine to fma because neither argument can be infinity or NaN based on assumptions
+; Combine to fma with nsz: neither mul operand can be Inf/NaN (via assumes)
+; and nsz permits ignoring the +-0 sign difference from the legacy zero clause.
+define float @test_finite_assumed_nsz(float %x, float %y, float %z) {
+; CHECK-LABEL: @test_finite_assumed_nsz(
+; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT: [[IS_FINITE_X:%.*]] = fcmp one float [[FABS_X]], +inf
+; CHECK-NEXT: [[FABS_Y:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
+; CHECK-NEXT: [[CALL:%.*]] = call nsz float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %fabs.x = call float @llvm.fabs.f32(float %x)
+ %is.finite.x = fcmp one float %fabs.x, 0x7FF0000000000000
+ %fabs.y = call float @llvm.fabs.f32(float %y)
+ %is.finite.y = fcmp one float %fabs.y, 0x7FF0000000000000
+ call void @llvm.assume(i1 %is.finite.x)
+ call void @llvm.assume(i1 %is.finite.y)
+ %call = call nsz float @llvm.amdgcn.fma.legacy(float %x, float %y, float %z)
+ ret float %call
+}
+
+; Do NOT combine: finite assumptions don't exclude zero, sign-of-zero unsafe.
define float @test_finite_assumed(float %x, float %y, float %z) {
; CHECK-LABEL: @test_finite_assumed(
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
@@ -92,7 +127,7 @@ define float @test_finite_assumed(float %x, float %y, float %z) {
; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
-; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fma.legacy(float [[X]], float [[Y]], float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -105,6 +140,38 @@ define float @test_finite_assumed(float %x, float %y, float %z) {
ret float %call
}
+; Combine without nsz: both mul operands are finite non-zero constants so the
+; legacy zero clause can never fire and sign-of-zero cannot diverge.
+define float @test_both_const_nonzero(float %z) {
+; CHECK-LABEL: @test_both_const_nonzero(
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float -2.000000e+00, float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call float @llvm.amdgcn.fma.legacy(float -2.0, float 99.5, float %z)
+ ret float %call
+}
+
+; Combine without nsz: qNaN and finite non-zero, neither can be zero.
+; Result constant-folds to +qnan.
+define float @test_nan_const_nonzero(float %z) {
+; CHECK-LABEL: @test_nan_const_nonzero(
+; CHECK-NEXT: ret float +qnan
+;
+ %call = call float @llvm.amdgcn.fma.legacy(float 0x7FF8000000000000, float -2.0, float %z)
+ ret float %call
+}
+
+; Combine without nsz: +inf and finite non-zero, neither can be zero.
+; Folds to llvm.fma.f32 (not constant-folded since %z is a runtime value).
+define float @test_inf_const_nonzero(float %z) {
+; CHECK-LABEL: @test_inf_const_nonzero(
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float +inf, float -2.000000e+00, float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call float @llvm.amdgcn.fma.legacy(float 0x7FF0000000000000, float -2.0, float %z)
+ ret float %call
+}
+
define float @test_poison_x_y(float %x, float %y) {
; CHECK-LABEL: @test_poison_x_y(
; CHECK-NEXT: ret float poison
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
index 4600831b60b87..d2e450cc7de7e 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
@@ -19,32 +19,43 @@ define float @test_negzero(float %y) {
ret float %call
}
-; Combine to fmul because the constant is finite and non-zero.
+; Combine to fmul because the constant is finite and non-zero with nsz flag.
+define float @test_const_nsz(float %x) {
+; CHECK-LABEL: @test_const_nsz(
+; CHECK-NEXT: [[CALL:%.*]] = fmul nsz float [[X:%.*]], 9.950000e+01
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call nsz float @llvm.amdgcn.fmul.legacy(float %x, float 99.5)
+ ret float %call
+}
+
+; Do NOT combine: one operand is finite-nonzero but the other may be +-0 at
+; runtime. Legacy returns +0; IEEE fmul returns -0. Sign-of-zero mismatch.
define float @test_const(float %x) {
; CHECK-LABEL: @test_const(
-; CHECK-NEXT: [[CALL:%.*]] = fmul float [[X:%.*]], 9.950000e+01
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X:%.*]], float 9.950000e+01)
; CHECK-NEXT: ret float [[CALL]]
;
%call = call float @llvm.amdgcn.fmul.legacy(float %x, float 99.5)
ret float %call
}
-; Combine to fmul because the constant is finite and non-zero, preserving fmf.
+; Do NOT combine: contract does not imply nsz, sign-of-zero still matters.
define float @test_const_fmf(float %x) {
; CHECK-LABEL: @test_const_fmf(
-; CHECK-NEXT: [[CALL:%.*]] = fmul contract float [[X:%.*]], 9.950000e+01
+; CHECK-NEXT: [[CALL:%.*]] = call contract float @llvm.amdgcn.fmul.legacy(float [[X:%.*]], float 9.950000e+01)
; CHECK-NEXT: ret float [[CALL]]
;
%call = call contract float @llvm.amdgcn.fmul.legacy(float %x, float 99.5)
ret float %call
}
-; Combine to fmul because neither argument can be infinity or NaN.
+; Do NOT combine: sitofp is finite but may be zero, sign-of-zero can diverge.
define float @test_finite(i32 %x, i32 %y) {
; CHECK-LABEL: @test_finite(
; CHECK-NEXT: [[XF:%.*]] = sitofp i32 [[X:%.*]] to float
; CHECK-NEXT: [[YF:%.*]] = sitofp i32 [[Y:%.*]] to float
-; CHECK-NEXT: [[CALL:%.*]] = fmul nnan float [[XF]], [[YF]]
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[XF]], float [[YF]])
; CHECK-NEXT: ret float [[CALL]]
;
%xf = sitofp i32 %x to float
@@ -53,7 +64,29 @@ define float @test_finite(i32 %x, i32 %y) {
ret float %call
}
-; Combine to fmul because neither argument can be infinity or NaN based on assumptions
+; Combine to fmul because neither argument can be infinity or NaN with nsz based on assumptions
+define float @test_finite_assumed_nsz(float %x, float %y) {
+; CHECK-LABEL: @test_finite_assumed_nsz(
+; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT: [[IS_FINITE_X:%.*]] = fcmp one float [[FABS_X]], +inf
+; CHECK-NEXT: [[FABS_Y:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
+; CHECK-NEXT: [[CALL:%.*]] = fmul nnan nsz float [[X]], [[Y]]
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %fabs.x = call float @llvm.fabs.f32(float %x)
+ %is.finite.x = fcmp one float %fabs.x, 0x7FF0000000000000
+ %fabs.y = call float @llvm.fabs.f32(float %y)
+ %is.finite.y = fcmp one float %fabs.y, 0x7FF0000000000000
+ call void @llvm.assume(i1 %is.finite.x)
+ call void @llvm.assume(i1 %is.finite.y)
+ %call = call nsz float @llvm.amdgcn.fmul.legacy(float %x, float %y)
+ ret float %call
+}
+
+; Do NOT combine: finite assumptions don't exclude zero, sign-of-zero unsafe.
define float @test_finite_assumed(float %x, float %y) {
; CHECK-LABEL: @test_finite_assumed(
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
@@ -62,7 +95,7 @@ define float @test_finite_assumed(float %x, float %y) {
; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
-; CHECK-NEXT: [[CALL:%.*]] = fmul nnan float [[X]], [[Y]]
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[CALL]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -75,6 +108,37 @@ define float @test_finite_assumed(float %x, float %y) {
ret float %call
}
+; Combine to fmul without nsz: both operands are finite non-zero constants so
+; the legacy zero clause can never fire (neither operand is zero) and there is
+; no sign-of-zero divergence. The result is further constant-folded to -199.0.
+define float @test_both_const_nonzero() {
+; CHECK-LABEL: @test_both_const_nonzero(
+; CHECK-NEXT: ret float -1.990000e+02
+;
+ %call = call float @llvm.amdgcn.fmul.legacy(float -2.0, float 99.5)
+ ret float %call
+}
+
+; Combine to fmul without nsz: qNaN and finite non-zero, neither can be zero.
+; Result constant-folds to +qnan.
+define float @test_nan_const_nonzero() {
+; CHECK-LABEL: @test_nan_const_nonzero(
+; CHECK-NEXT: ret float +qnan
+;
+ %call = call float @llvm.amdgcn.fmul.legacy(float 0x7FF8000000000000, float -2.0)
+ ret float %call
+}
+
+; Combine to fmul without nsz: +inf and finite non-zero, neither can be zero.
+; Result constant-folds to -inf.
+define float @test_inf_const_nonzero() {
+; CHECK-LABEL: @test_inf_const_nonzero(
+; CHECK-NEXT: ret float -inf
+;
+ %call = call float @llvm.amdgcn.fmul.legacy(float 0x7FF0000000000000, float -2.0)
+ ret float %call
+}
+
define float @test_poison_var(float %x) {
; CHECK-LABEL: @test_poison_var(
; CHECK-NEXT: ret float poison
More information about the llvm-commits
mailing list