[llvm] [AMDGPU] Fix fmul/fma legacy sign-of-zero miscompile (PR #203567)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 08:33:19 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Wooseok Lee (wooseoklee)
<details>
<summary>Changes</summary>
V_MUL_LEGACY_F32 always returns +0.0 when either operand is ±0.0, while a regular IEEE fmul XORs operand signs. This makes the results observably different when one operand is a runtime ±0.0:
fmul.legacy(-2.0, +0.0) = +0.0 (legacy zero clause)
fmul (-2.0, +0.0) = -0.0 (IEEE sign XOR: - * + = -)
canSimplifyLegacyMulToMul was incorrectly folding whenever one operand matched m_FiniteNonZero() or both were known never Inf/NaN, without accounting for the sign-of-zero divergence on the other operand.
Restructure the guard around the nsz fast-math flag:
- With nsz (sign-of-zero is don't-care): fold if one operand is finite-nonzero (zero clause can only fire on the other side, but nsz makes the +0 vs -0 difference irrelevant), or if both operands are known never Inf/NaN (nsz covers ±0 sign, and 0*Inf/0*NaN = NaN cases are excluded):
fmul.legacy nsz (-2.0, %x) -> fmul nsz (-2.0, %x)
- Without nsz: only fold if both operands are known finite and non-zero so the zero clause can never fire:
fmul.legacy(-2.0, 99.5) -> fmul(-2.0, 99.5) ; both nonzero, safe
fmul.legacy(-2.0, %x) -> not folded ; %x may be ±0
The same fix applies to fma.legacy via the shared helper. Update fmul_legacy.ll and fma_legacy.ll accordingly.
---
Full diff: https://github.com/llvm/llvm-project/pull/203567.diff
3 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp (+15-9)
- (modified) llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll (+54-8)
- (modified) llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll (+52-8)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index b17ccef685b21..0cdaf052ba7c3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -405,16 +405,22 @@ bool GCNTTIImpl::canSimplifyLegacyMulToMul(const Instruction &I,
// cases then we can use a normal multiply instead.
// TODO: Create and use isKnownFiniteNonZero instead of just matching
// constants here.
- if (match(Op0, PatternMatch::m_FiniteNonZero()) ||
- match(Op1, PatternMatch::m_FiniteNonZero())) {
- // One operand is not zero or infinity or NaN.
- return true;
- }
-
+ bool Op0FiniteNonZero = match(Op0, PatternMatch::m_FiniteNonZero());
+ bool Op1FiniteNonZero = match(Op1, PatternMatch::m_FiniteNonZero());
SimplifyQuery SQ = IC.getSimplifyQuery().getWithInstruction(&I);
- if (isKnownNeverInfOrNaN(Op0, SQ) && isKnownNeverInfOrNaN(Op1, SQ)) {
- // Neither operand is infinity or NaN.
- return true;
+ if (I.hasNoSignedZeros()) {
+ // With no signed zero (nsz) flag, it's safe to use mul instead of legacy
+ // mul if one of the two operands is not zero or infinity or NaN.
+ if (Op0FiniteNonZero || Op1FiniteNonZero)
+ return true;
+ // Otherwise, we can fold if neither operand is infinity or NaN.
+ if (isKnownNeverInfOrNaN(Op0, SQ) && isKnownNeverInfOrNaN(Op1, SQ))
+ return true;
+ } else {
+ // Without nsz, only fold if both operands are known finite and non-zero so
+ // the legacy zero clause can never fire.
+ if (Op0FiniteNonZero && Op1FiniteNonZero)
+ return true;
}
return false;
}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
index 3efd68b9c1444..a46b5f281618e 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fma_legacy.ll
@@ -49,32 +49,44 @@ define float @test_negzero_nsz(float %y, float %z) {
ret float %call
}
-; Combine to fma because the constant is finite and non-zero.
+; Combine to fma with nsz: sign-of-zero is don't-care so the +0 vs +-0
+; difference between legacy and IEEE does not matter.
+define float @test_const_nsz(float %x, float %z) {
+; CHECK-LABEL: @test_const_nsz(
+; CHECK-NEXT: [[CALL:%.*]] = call nsz float @llvm.fma.f32(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call nsz float @llvm.amdgcn.fma.legacy(float %x, float 99.5, float %z)
+ ret float %call
+}
+
+; Do NOT combine: one operand is finite-nonzero but the other may be +-0 at
+; runtime. Legacy mul returns +0; IEEE fmul returns -0. Sign-of-zero mismatch.
define float @test_const(float %x, float %z) {
; CHECK-LABEL: @test_const(
-; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fma.legacy(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%call = call float @llvm.amdgcn.fma.legacy(float %x, float 99.5, float %z)
ret float %call
}
-; Combine to fma because the constant is finite and non-zero, preserving fmf.
+; Do NOT combine: contract does not imply nsz, sign-of-zero still matters.
define float @test_const_fmf(float %x, float %z) {
; CHECK-LABEL: @test_const_fmf(
-; CHECK-NEXT: [[CALL:%.*]] = call contract float @llvm.fma.f32(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call contract float @llvm.amdgcn.fma.legacy(float [[X:%.*]], float 9.950000e+01, float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%call = call contract float @llvm.amdgcn.fma.legacy(float %x, float 99.5, float %z)
ret float %call
}
-; Combine to fma because neither argument can be infinity or NaN.
+; Do NOT combine: sitofp is finite but may be zero, sign-of-zero can diverge.
define float @test_finite(i32 %x, i32 %y, float %z) {
; CHECK-LABEL: @test_finite(
; CHECK-NEXT: [[XF:%.*]] = sitofp i32 [[X:%.*]] to float
; CHECK-NEXT: [[YF:%.*]] = sitofp i32 [[Y:%.*]] to float
-; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float [[XF]], float [[YF]], float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fma.legacy(float [[XF]], float [[YF]], float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%xf = sitofp i32 %x to float
@@ -83,7 +95,30 @@ define float @test_finite(i32 %x, i32 %y, float %z) {
ret float %call
}
-; Combine to fma because neither argument can be infinity or NaN based on assumptions
+; Combine to fma with nsz: neither mul operand can be Inf/NaN (via assumes)
+; and nsz permits ignoring the +-0 sign difference from the legacy zero clause.
+define float @test_finite_assumed_nsz(float %x, float %y, float %z) {
+; CHECK-LABEL: @test_finite_assumed_nsz(
+; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT: [[IS_FINITE_X:%.*]] = fcmp one float [[FABS_X]], +inf
+; CHECK-NEXT: [[FABS_Y:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
+; CHECK-NEXT: [[CALL:%.*]] = call nsz float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %fabs.x = call float @llvm.fabs.f32(float %x)
+ %is.finite.x = fcmp one float %fabs.x, 0x7FF0000000000000
+ %fabs.y = call float @llvm.fabs.f32(float %y)
+ %is.finite.y = fcmp one float %fabs.y, 0x7FF0000000000000
+ call void @llvm.assume(i1 %is.finite.x)
+ call void @llvm.assume(i1 %is.finite.y)
+ %call = call nsz float @llvm.amdgcn.fma.legacy(float %x, float %y, float %z)
+ ret float %call
+}
+
+; Do NOT combine: finite assumptions don't exclude zero, sign-of-zero unsafe.
define float @test_finite_assumed(float %x, float %y, float %z) {
; CHECK-LABEL: @test_finite_assumed(
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
@@ -92,7 +127,7 @@ define float @test_finite_assumed(float %x, float %y, float %z) {
; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
-; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z:%.*]])
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fma.legacy(float [[X]], float [[Y]], float [[Z:%.*]])
; CHECK-NEXT: ret float [[CALL]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -105,6 +140,17 @@ define float @test_finite_assumed(float %x, float %y, float %z) {
ret float %call
}
+; Combine without nsz: both mul operands are finite non-zero constants so the
+; legacy zero clause can never fire and sign-of-zero cannot diverge.
+define float @test_both_const_nonzero(float %z) {
+; CHECK-LABEL: @test_both_const_nonzero(
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.fma.f32(float -2.000000e+00, float 9.950000e+01, float [[Z:%.*]])
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call float @llvm.amdgcn.fma.legacy(float -2.0, float 99.5, float %z)
+ ret float %call
+}
+
define float @test_poison_x_y(float %x, float %y) {
; CHECK-LABEL: @test_poison_x_y(
; CHECK-NEXT: ret float poison
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
index 4600831b60b87..206193d74e6dd 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fmul_legacy.ll
@@ -19,32 +19,43 @@ define float @test_negzero(float %y) {
ret float %call
}
-; Combine to fmul because the constant is finite and non-zero.
+; Combine to fmul because the constant is finite and non-zero with nsz flag.
+define float @test_const_nsz(float %x) {
+; CHECK-LABEL: @test_const_nsz(
+; CHECK-NEXT: [[CALL:%.*]] = fmul nsz float [[X:%.*]], 9.950000e+01
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %call = call nsz float @llvm.amdgcn.fmul.legacy(float %x, float 99.5)
+ ret float %call
+}
+
+; Do NOT combine: one operand is finite-nonzero but the other may be +-0 at
+; runtime. Legacy returns +0; IEEE fmul returns -0. Sign-of-zero mismatch.
define float @test_const(float %x) {
; CHECK-LABEL: @test_const(
-; CHECK-NEXT: [[CALL:%.*]] = fmul float [[X:%.*]], 9.950000e+01
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X:%.*]], float 9.950000e+01)
; CHECK-NEXT: ret float [[CALL]]
;
%call = call float @llvm.amdgcn.fmul.legacy(float %x, float 99.5)
ret float %call
}
-; Combine to fmul because the constant is finite and non-zero, preserving fmf.
+; Do NOT combine: contract does not imply nsz, sign-of-zero still matters.
define float @test_const_fmf(float %x) {
; CHECK-LABEL: @test_const_fmf(
-; CHECK-NEXT: [[CALL:%.*]] = fmul contract float [[X:%.*]], 9.950000e+01
+; CHECK-NEXT: [[CALL:%.*]] = call contract float @llvm.amdgcn.fmul.legacy(float [[X:%.*]], float 9.950000e+01)
; CHECK-NEXT: ret float [[CALL]]
;
%call = call contract float @llvm.amdgcn.fmul.legacy(float %x, float 99.5)
ret float %call
}
-; Combine to fmul because neither argument can be infinity or NaN.
+; Do NOT combine: sitofp is finite but may be zero, sign-of-zero can diverge.
define float @test_finite(i32 %x, i32 %y) {
; CHECK-LABEL: @test_finite(
; CHECK-NEXT: [[XF:%.*]] = sitofp i32 [[X:%.*]] to float
; CHECK-NEXT: [[YF:%.*]] = sitofp i32 [[Y:%.*]] to float
-; CHECK-NEXT: [[CALL:%.*]] = fmul nnan float [[XF]], [[YF]]
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[XF]], float [[YF]])
; CHECK-NEXT: ret float [[CALL]]
;
%xf = sitofp i32 %x to float
@@ -53,7 +64,29 @@ define float @test_finite(i32 %x, i32 %y) {
ret float %call
}
-; Combine to fmul because neither argument can be infinity or NaN based on assumptions
+; Combine to fmul because neither argument can be infinity or NaN with nsz based on assumptions
+define float @test_finite_assumed_nsz(float %x, float %y) {
+; CHECK-LABEL: @test_finite_assumed_nsz(
+; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
+; CHECK-NEXT: [[IS_FINITE_X:%.*]] = fcmp one float [[FABS_X]], +inf
+; CHECK-NEXT: [[FABS_Y:%.*]] = call float @llvm.fabs.f32(float [[Y:%.*]])
+; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
+; CHECK-NEXT: [[CALL:%.*]] = fmul nnan nsz float [[X]], [[Y]]
+; CHECK-NEXT: ret float [[CALL]]
+;
+ %fabs.x = call float @llvm.fabs.f32(float %x)
+ %is.finite.x = fcmp one float %fabs.x, 0x7FF0000000000000
+ %fabs.y = call float @llvm.fabs.f32(float %y)
+ %is.finite.y = fcmp one float %fabs.y, 0x7FF0000000000000
+ call void @llvm.assume(i1 %is.finite.x)
+ call void @llvm.assume(i1 %is.finite.y)
+ %call = call nsz float @llvm.amdgcn.fmul.legacy(float %x, float %y)
+ ret float %call
+}
+
+; Do NOT combine: finite assumptions don't exclude zero, sign-of-zero unsafe.
define float @test_finite_assumed(float %x, float %y) {
; CHECK-LABEL: @test_finite_assumed(
; CHECK-NEXT: [[FABS_X:%.*]] = call float @llvm.fabs.f32(float [[X:%.*]])
@@ -62,7 +95,7 @@ define float @test_finite_assumed(float %x, float %y) {
; CHECK-NEXT: [[IS_FINITE_Y:%.*]] = fcmp one float [[FABS_Y]], +inf
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_X]])
; CHECK-NEXT: call void @llvm.assume(i1 [[IS_FINITE_Y]])
-; CHECK-NEXT: [[CALL:%.*]] = fmul nnan float [[X]], [[Y]]
+; CHECK-NEXT: [[CALL:%.*]] = call float @llvm.amdgcn.fmul.legacy(float [[X]], float [[Y]])
; CHECK-NEXT: ret float [[CALL]]
;
%fabs.x = call float @llvm.fabs.f32(float %x)
@@ -75,6 +108,17 @@ define float @test_finite_assumed(float %x, float %y) {
ret float %call
}
+; Combine to fmul without nsz: both operands are finite non-zero constants so
+; the legacy zero clause can never fire (neither operand is zero) and there is
+; no sign-of-zero divergence. The result is further constant-folded to -199.0.
+define float @test_both_const_nonzero() {
+; CHECK-LABEL: @test_both_const_nonzero(
+; CHECK-NEXT: ret float -1.990000e+02
+;
+ %call = call float @llvm.amdgcn.fmul.legacy(float -2.0, float 99.5)
+ ret float %call
+}
+
define float @test_poison_var(float %x) {
; CHECK-LABEL: @test_poison_var(
; CHECK-NEXT: ret float poison
``````````
</details>
https://github.com/llvm/llvm-project/pull/203567
More information about the llvm-commits
mailing list