[llvm] `__builtin.canonicalize` in common code (PR #142105)
Dominik Steenken via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 5 02:18:45 PDT 2025
https://github.com/dominik-steenken updated https://github.com/llvm/llvm-project/pull/142105
>From da0f89e80a1a94ef1fe4bf207e13ec63efef5044 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost at de.ibm.com>
Date: Thu, 22 May 2025 12:05:32 +0200
Subject: [PATCH 1/5] Implement generic llvm.canonicalize.f(32|64).
Previously, this intrinsic has been implemented in the X86 backend. As
suggested in the comments for the PR that merges that implementation (#106370),
this commit replicates that implementation (at least for `f32` and `f64`) in
common code. Canonicalization is implemented as multiplication with 1.0.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 17 +++++++++++++++++
llvm/lib/CodeGen/TargetLoweringBase.cpp | 3 +++
2 files changed, 20 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5388183b0dced..d7b5b8890d45d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3356,6 +3356,23 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(Op);
break;
}
+ case ISD::FCANONICALIZE: {
+ // This implements llvm.canonicalize.f* by multiplication with 1.0,
+ // as suggested in https://llvm.org/docs/LangRef.html#id2335.
+ // Get operand x.
+ SDValue Operand = Node->getOperand(0);
+ // Get fp value type used.
+ EVT VT = Operand.getValueType();
+ // Produce appropriately-typed 1.0 constant.
+ SDValue One = DAG.getConstantFP(1.0, dl, VT);
+ // Produce multiplication node x * 1.0.
+ SDValue Chain = DAG.getEntryNode();
+ SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, dl, {VT, MVT::Other},
+ {Chain, Operand, One});
+
+ Results.push_back(Mul);
+ break;
+ }
case ISD::SIGN_EXTEND_INREG: {
EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
EVT VT = Node->getValueType(0);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 935afaf9dd550..ba765afd80bf1 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -870,6 +870,9 @@ void TargetLoweringBase::initActions() {
ISD::FATAN2},
{MVT::f32, MVT::f64, MVT::f128}, Expand);
+ // Insert custom handling default for llvm.canonicalize.*.
+ setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Expand);
+
// FIXME: Query RuntimeLibCalls to make the decision.
setOperationAction({ISD::LRINT, ISD::LLRINT, ISD::LROUND, ISD::LLROUND},
{MVT::f32, MVT::f64, MVT::f128}, LibCall);
>From c1ee0e125a5d8f0f7f75851519e54de54cf8ae06 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost at de.ibm.com>
Date: Wed, 21 May 2025 14:43:51 +0200
Subject: [PATCH 2/5] [SystemZ] Add tests for llvm.canonicalize.f(32|64).
These tests include only the floating point types supported in common code.
Once a SystemZ-specific implementation for `f16` and `f128` is available, these
tests will be expanded to include those types as well.
The tests are heavily based on the equivalent tests for X86.
---
.../test/CodeGen/SystemZ/canonicalize-vars.ll | 131 ++++++++++++++++++
1 file changed, 131 insertions(+)
create mode 100644 llvm/test/CodeGen/SystemZ/canonicalize-vars.ll
diff --git a/llvm/test/CodeGen/SystemZ/canonicalize-vars.ll b/llvm/test/CodeGen/SystemZ/canonicalize-vars.ll
new file mode 100644
index 0000000000000..164f9c2f7008c
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/canonicalize-vars.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --default-march s390x-unknown-linux-gnu --version 5
+; RUN: llc -mtriple=s390x-linux-gnu -mcpu=z16 < %s | FileCheck %s -check-prefixes=Z16
+
+define float @canonicalize_fp32(float %a) {
+; Z16-LABEL: canonicalize_fp32:
+; Z16: # %bb.0:
+; Z16-NEXT: vgmf %v1, 2, 8
+; Z16-NEXT: meebr %f0, %f1
+; Z16-NEXT: br %r14
+ %canonicalized = call float @llvm.canonicalize.f32(float %a)
+ ret float %canonicalized
+}
+
+define double @canonicalize_fp64(double %a) {
+; Z16-LABEL: canonicalize_fp64:
+; Z16: # %bb.0:
+; Z16-NEXT: vgmg %v1, 2, 11
+; Z16-NEXT: mdbr %f0, %f1
+; Z16-NEXT: br %r14
+ %canonicalized = call double @llvm.canonicalize.f64(double %a)
+ ret double %canonicalized
+}
+
+define void @canonicalize_ptr_f32(float * %out) {
+; Z16-LABEL: canonicalize_ptr_f32:
+; Z16: # %bb.0:
+; Z16-NEXT: vgmf %v0, 2, 8
+; Z16-NEXT: meeb %f0, 0(%r2)
+; Z16-NEXT: ste %f0, 0(%r2)
+; Z16-NEXT: br %r14
+ %val = load float, float * %out
+ %canonicalized = call float @llvm.canonicalize.f32(float %val)
+ store float %canonicalized, float * %out
+ ret void
+}
+
+define void @canonicalize_ptr_f64(double * %out) {
+; Z16-LABEL: canonicalize_ptr_f64:
+; Z16: # %bb.0:
+; Z16-NEXT: vgmg %v0, 2, 11
+; Z16-NEXT: mdb %f0, 0(%r2)
+; Z16-NEXT: std %f0, 0(%r2)
+; Z16-NEXT: br %r14
+ %val = load double, double * %out
+ %canonicalized = call double @llvm.canonicalize.f64(double %val)
+ store double %canonicalized, double * %out
+ ret void
+}
+
+define <4 x float> @canonicalize_v4f32(<4 x float> %a) {
+; Z16-LABEL: canonicalize_v4f32:
+; Z16: # %bb.0:
+; Z16-NEXT: vrepf %v0, %v24, 3
+; Z16-NEXT: vgmf %v1, 2, 8
+; Z16-NEXT: vrepf %v2, %v24, 2
+; Z16-NEXT: meebr %f0, %f1
+; Z16-NEXT: meebr %f2, %f1
+; Z16-NEXT: vrepf %v3, %v24, 1
+; Z16-NEXT: vmrhf %v0, %v2, %v0
+; Z16-NEXT: wfmsb %f2, %v24, %f1
+; Z16-NEXT: wfmsb %f1, %f3, %f1
+; Z16-NEXT: vmrhf %v1, %v2, %v1
+; Z16-NEXT: vmrhg %v24, %v1, %v0
+; Z16-NEXT: br %r14
+ %canonicalized = call <4 x float> @llvm.canonicalize.v4f32(<4 x float> %a)
+ ret <4 x float> %canonicalized
+}
+
+define <4 x double> @canonicalize_v4f64(<4 x double> %a) {
+; Z16-LABEL: canonicalize_v4f64:
+; Z16: # %bb.0:
+; Z16-NEXT: vgmg %v0, 2, 11
+; Z16-NEXT: vrepg %v2, %v24, 1
+; Z16-NEXT: wfmdb %f1, %v24, %f0
+; Z16-NEXT: mdbr %f2, %f0
+; Z16-NEXT: vmrhg %v24, %v1, %v2
+; Z16-NEXT: vrepg %v2, %v26, 1
+; Z16-NEXT: wfmdb %f1, %v26, %f0
+; Z16-NEXT: wfmdb %f0, %f2, %f0
+; Z16-NEXT: vmrhg %v26, %v1, %v0
+; Z16-NEXT: br %r14
+ %canonicalized = call <4 x double> @llvm.canonicalize.v4f64(<4 x double> %a)
+ ret <4 x double> %canonicalized
+}
+
+define void @canonicalize_ptr_v4f32(<4 x float> * %out) {
+; Z16-LABEL: canonicalize_ptr_v4f32:
+; Z16: # %bb.0:
+; Z16-NEXT: vl %v0, 0(%r2), 3
+; Z16-NEXT: vrepf %v1, %v0, 3
+; Z16-NEXT: vgmf %v2, 2, 8
+; Z16-NEXT: vrepf %v3, %v0, 2
+; Z16-NEXT: meebr %f1, %f2
+; Z16-NEXT: meebr %f3, %f2
+; Z16-NEXT: vmrhf %v1, %v3, %v1
+; Z16-NEXT: wfmsb %f3, %f0, %f2
+; Z16-NEXT: vrepf %v0, %v0, 1
+; Z16-NEXT: meebr %f0, %f2
+; Z16-NEXT: vmrhf %v0, %v3, %v0
+; Z16-NEXT: vmrhg %v0, %v0, %v1
+; Z16-NEXT: vst %v0, 0(%r2), 3
+; Z16-NEXT: br %r14
+ %val = load <4 x float>, <4 x float> * %out
+ %canonicalized = call <4 x float> @llvm.canonicalize.v4f32(<4 x float> %val)
+ store <4 x float> %canonicalized, <4 x float> * %out
+ ret void
+}
+
+define void @canonicalize_ptr_v4f64(<4 x double> * %out) {
+; Z16-LABEL: canonicalize_ptr_v4f64:
+; Z16: # %bb.0:
+; Z16-NEXT: vl %v1, 16(%r2), 4
+; Z16-NEXT: vgmg %v2, 2, 11
+; Z16-NEXT: wfmdb %f3, %f1, %f2
+; Z16-NEXT: vrepg %v1, %v1, 1
+; Z16-NEXT: mdbr %f1, %f2
+; Z16-NEXT: vl %v0, 0(%r2), 4
+; Z16-NEXT: vmrhg %v1, %v3, %v1
+; Z16-NEXT: wfmdb %f3, %f0, %f2
+; Z16-NEXT: vrepg %v0, %v0, 1
+; Z16-NEXT: mdbr %f0, %f2
+; Z16-NEXT: vmrhg %v0, %v3, %v0
+; Z16-NEXT: vst %v0, 0(%r2), 4
+; Z16-NEXT: vst %v1, 16(%r2), 4
+; Z16-NEXT: br %r14
+ %val = load <4 x double>, <4 x double> * %out
+ %canonicalized = call <4 x double> @llvm.canonicalize.v4f64(<4 x double> %val)
+ store <4 x double> %canonicalized, <4 x double> * %out
+ ret void
+}
+
>From 07f5741b806ed1a899acc9e7a72b6607964ac7a9 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost at de.ibm.com>
Date: Tue, 3 Jun 2025 11:00:58 +0200
Subject: [PATCH 3/5] [AArch64][AMDGPU][Mips][PowerPC][LoongArch] Legalize
fcanonicalize for f32,f64
Since the default OperationAction for `ISD::FCANONICALIZE` is now `Expand`,
these targets need to explicitly set it to `Legal` in their backends in order
to preserve their functionality.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 6 ++++++
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 5 +++++
llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp | 2 ++
llvm/lib/Target/Mips/MipsISelLowering.cpp | 2 ++
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 2 ++
5 files changed, 17 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9f51caef6d228..6ba295aca304f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -767,6 +767,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(Op, MVT::v8bf16, Expand);
}
+ // Legalize fcanonicalize to circumvent default expansion
+ setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
+ if (Subtarget->hasFullFP16()) {
+ setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
+ }
+
// fpextend from f16 or bf16 to f32 is legal
setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Legal);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 5f41bd7d8a617..539a9793500a8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -423,6 +423,11 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, MVT::f16,
Custom);
+ setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
+ if (Subtarget->has16BitInsts()) {
+ setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
+ }
+
// FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
// scalarization code. Can be removed when IS_FPCLASS expand isn't called by
// default unless marked custom/legal.
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index eda71dde620b6..716cafaf35859 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -191,6 +191,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
+ setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
setOperationAction(ISD::IS_FPCLASS, MVT::f32, Legal);
@@ -233,6 +234,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal);
+ setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
setOperationAction(ISD::IS_FPCLASS, MVT::f64, Legal);
setOperationAction(ISD::FSIN, MVT::f64, Expand);
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index e933e97ea3706..4b1c97dd9d34a 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -373,6 +373,8 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
setOperationAction(ISD::IS_FPCLASS, MVT::f32, Legal);
setOperationAction(ISD::IS_FPCLASS, MVT::f64, Legal);
+ setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
+ setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
} else {
setOperationAction(ISD::FCANONICALIZE, MVT::f32, Custom);
setOperationAction(ISD::FCANONICALIZE, MVT::f64, Custom);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 0c2a506005604..8b82ffd82c3a2 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -777,6 +777,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal);
+ setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
+ setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
}
if (Subtarget.hasAltivec()) {
>From cd133887eb953e8b3ece2448d885a34ce07cc3f8 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost at de.ibm.com>
Date: Thu, 5 Jun 2025 11:17:15 +0200
Subject: [PATCH 4/5] Update comment based on review
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index d7b5b8890d45d..508e805596e3d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3358,7 +3358,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
case ISD::FCANONICALIZE: {
// This implements llvm.canonicalize.f* by multiplication with 1.0,
- // as suggested in https://llvm.org/docs/LangRef.html#id2335.
+ // as suggested in https://llvm.org/docs/LangRef.html#id2335. It uses
+ // strict_fp operations even outside a strict_fp context in order to
+ // guarantee that the canonicalization is not optimized away by later passes.
// Get operand x.
SDValue Operand = Node->getOperand(0);
// Get fp value type used.
>From 510da454d40894275ffe72d31d4db9e01a9a8458 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost at de.ibm.com>
Date: Thu, 5 Jun 2025 11:18:23 +0200
Subject: [PATCH 5/5] Straighten out code formatting
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 508e805596e3d..7a110a0062733 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3360,7 +3360,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
// This implements llvm.canonicalize.f* by multiplication with 1.0,
// as suggested in https://llvm.org/docs/LangRef.html#id2335. It uses
// strict_fp operations even outside a strict_fp context in order to
- // guarantee that the canonicalization is not optimized away by later passes.
+ // guarantee that the canonicalization is not optimized away by later
+ // passes.
+
// Get operand x.
SDValue Operand = Node->getOperand(0);
// Get fp value type used.
More information about the llvm-commits
mailing list