[llvm] [AArch64] NFCI: Simplify LowerVectorFP_TO_INT_SAT (part 2) (PR #207199)
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 00:48:11 PDT 2026
https://github.com/sdesmalen-arm updated https://github.com/llvm/llvm-project/pull/207199
>From 10f877ee3aecfc1a027ad8b2afad764cc5b12eb1 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 2 Jul 2026 13:24:27 +0000
Subject: [PATCH 1/4] [AArch64] NFCI: Simplify LowerVectorFP_TO_INT_SAT (part
2)
This simplifies the logic a bit more, such that the flow of the
lowering is as follows:
* Try to promote if necessary (fp16/bf16 types or src < dst)
* Try to handle natively (satwidth == srcwidth == dstwidth)
* Otherwise use min/max + truncate.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 96 +++++++++++--------
1 file changed, 58 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 93b4b33043755..0799f10344006 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5209,52 +5209,72 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
return SDValue();
EVT SrcElementVT = SrcVT.getVectorElementType();
+ if (SrcElementVT != MVT::f64 && SrcElementVT != MVT::f32 &&
+ SrcElementVT != MVT::f16 && SrcElementVT != MVT::bf16)
+ return SDValue();
+
+ // Returns true if the operation can be matched by an isel pattern directly.
+ auto CanHandleNatively = [&DstVT, &SatWidth](EVT SrcVT) -> bool {
+ return SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits() &&
+ SrcVT.getScalarSizeInBits() == SatWidth;
+ };
+
+ // Returns true if the operation is best expanded.
+ auto Expand = [&DstVT, &SatWidth, &CanHandleNatively](EVT SrcVT) -> bool {
+ return !CanHandleNatively(SrcVT) &&
+ (SrcVT.getScalarSizeInBits() < SatWidth ||
+ // NEON has no vector MIN/MAX for i64, so it's simpler to scalarize
+ // (at least until sqxtn is selected).
+ SrcVT.getVectorElementType() == MVT::f64);
+ };
+
+ // Try to promote the operation to a wider type if SrcVT < DstVT,
+ // or if type is bf16 or if the target has no +fullfp16.
+ EVT PromVT = SrcVT;
+ switch (SrcVT.getVectorElementType().getSimpleVT().SimpleTy) {
+ case MVT::f16:
+ case MVT::bf16:
+ if (DstVT.getScalarSizeInBits() == 32 || !Subtarget->hasFullFP16()) {
+ PromVT = MVT::getVectorVT(MVT::f32, SrcVT.getVectorElementCount());
+ break;
+ }
+ [[fallthrough]];
+ case MVT::f32:
+ // Promote to f64
+ if (DstVT.getScalarSizeInBits() == 64) {
+ PromVT = MVT::getVectorVT(MVT::f64, SrcVT.getVectorElementCount());
+ break;
+ }
+ [[fallthrough]];
+ default:
+ break;
+ }
- // In the absence of FP16 support, promote f16 to f32 and saturate the result.
- // Note that SatWidth stays unchanged.
SDLoc DL(Op);
unsigned Opc = Op.getOpcode();
- if ((SrcElementVT == MVT::f16 &&
- (!Subtarget->hasFullFP16() || DstElementWidth > 16)) ||
- SrcElementVT == MVT::bf16) {
- MVT F32VT = MVT::getVectorVT(MVT::f32, SrcVT.getVectorNumElements());
- SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, F32VT, SrcVal);
- // If we are extending to a v8f32, split into two v4f32 to produce legal
- // types.
- if (F32VT == MVT::v8f32) {
- auto [SrcValLo, SrcValHi] = DAG.SplitVector(SrcVal, DL);
- SDValue Lo = DAG.getNode(Opc, DL, MVT::v4i32, SrcValLo, Op.getOperand(1));
- SDValue Hi = DAG.getNode(Opc, DL, MVT::v4i32, SrcValHi, Op.getOperand(1));
- Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Lo);
- Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Hi);
- return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Lo, Hi);
- }
- SrcVT = F32VT;
- SrcElementVT = MVT::f32;
- SrcElementWidth = 32;
- } else if (SrcElementVT != MVT::f64 && SrcElementVT != MVT::f32 &&
- SrcElementVT != MVT::f16 && SrcElementVT != MVT::bf16)
- return SDValue();
-
- // Expand to f64 if we are saturating to i64, to help keep the lanes the same
- // width and produce a fcvtzu. Note that SatWidth stays unchanged.
- if (SatWidth == 64 && SrcElementWidth < 64) {
- MVT F64VT = MVT::getVectorVT(MVT::f64, SrcVT.getVectorNumElements());
- SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, F64VT, SrcVal);
- SrcVT = F64VT;
- SrcElementVT = MVT::f64;
- SrcElementWidth = 64;
+ if (PromVT != SrcVT && !Expand(PromVT)) {
+ // When promoting the input type, SatWidth stays unchanged.
+ SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, PromVT, SrcVal);
+ if (PromVT != MVT::v8f32)
+ return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, Op.getOperand(1));
+
+ // If we are extending to a wider type (e.g. v8f16 -> v8f32) due to lack
+ // of fp16 support, then it's more efficient to split the operation
+ // into two v4f32 to produce legal types.
+ auto [SrcValLo, SrcValHi] = DAG.SplitVector(SrcVal, DL);
+ SDValue Lo = DAG.getNode(Opc, DL, MVT::v4i32, SrcValLo, Op.getOperand(1));
+ SDValue Hi = DAG.getNode(Opc, DL, MVT::v4i32, SrcValHi, Op.getOperand(1));
+ Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Lo);
+ Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Hi);
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Lo, Hi);
}
+
// Cases that we can emit directly.
- if (SrcElementWidth == DstElementWidth && SrcElementWidth == SatWidth)
+ if (CanHandleNatively(SrcVT))
return DAG.getNode(Opc, DL, DstVT, SrcVal,
DAG.getValueType(DstVT.getScalarType()));
- // Otherwise we emit a cvt that saturates to a higher BW, and saturate the
- // result. This is only valid if the legal cvt is larger than the saturate
- // width. For double, as we don't have MIN/MAX, it can be simpler to scalarize
- // (at least until sqxtn is selected).
- if (SrcElementWidth < SatWidth || SrcElementVT == MVT::f64)
+ if (Expand(SrcVT))
return SDValue();
assert((SrcElementWidth > DstElementWidth) ||
>From 6b30064443ace6109c8ba3aa14c0a9c0999122d2 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Thu, 9 Jul 2026 08:01:40 +0000
Subject: [PATCH 2/4] Address comments
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0799f10344006..bbc928807a7f4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5194,6 +5194,7 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
SDValue SrcVal = Op.getOperand(0);
EVT SrcVT = SrcVal.getValueType();
EVT DstVT = Op.getValueType();
+ EVT DstElementVT = DstVT.getVectorElementType();
EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
uint64_t SrcElementWidth = SrcVT.getScalarSizeInBits();
@@ -5220,7 +5221,7 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
};
// Returns true if the operation is best expanded.
- auto Expand = [&DstVT, &SatWidth, &CanHandleNatively](EVT SrcVT) -> bool {
+ auto Expand = [&SatWidth, &CanHandleNatively](EVT SrcVT) -> bool {
return !CanHandleNatively(SrcVT) &&
(SrcVT.getScalarSizeInBits() < SatWidth ||
// NEON has no vector MIN/MAX for i64, so it's simpler to scalarize
@@ -5231,17 +5232,18 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
// Try to promote the operation to a wider type if SrcVT < DstVT,
// or if type is bf16 or if the target has no +fullfp16.
EVT PromVT = SrcVT;
- switch (SrcVT.getVectorElementType().getSimpleVT().SimpleTy) {
+ switch (SrcElementVT.getSimpleVT().SimpleTy) {
case MVT::f16:
case MVT::bf16:
- if (DstVT.getScalarSizeInBits() == 32 || !Subtarget->hasFullFP16()) {
+ if (DstElementVT == MVT::i32 || SrcElementVT == MVT::bf16 ||
+ !Subtarget->hasFullFP16()) {
PromVT = MVT::getVectorVT(MVT::f32, SrcVT.getVectorElementCount());
break;
}
[[fallthrough]];
case MVT::f32:
// Promote to f64
- if (DstVT.getScalarSizeInBits() == 64) {
+ if (DstElementVT == MVT::i64) {
PromVT = MVT::getVectorVT(MVT::f64, SrcVT.getVectorElementCount());
break;
}
@@ -5273,8 +5275,7 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
if (CanHandleNatively(SrcVT))
return DAG.getNode(Opc, DL, DstVT, SrcVal,
DAG.getValueType(DstVT.getScalarType()));
-
- if (Expand(SrcVT))
+ else if (Expand(SrcVT))
return SDValue();
assert((SrcElementWidth > DstElementWidth) ||
>From f5fedb09821e65c13b16efc410dd8fdc33e9efcf Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Mon, 13 Jul 2026 15:28:03 +0000
Subject: [PATCH 3/4] Address more comments (const variables, std::optional
PromVT)
---
.../Target/AArch64/AArch64ISelLowering.cpp | 26 +++++++++----------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index bbc928807a7f4..30ce51e9b8d8d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5192,14 +5192,14 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
// AArch64 FP-to-int conversions saturate to the destination element size, so
// we can lower common saturating conversions to simple instructions.
SDValue SrcVal = Op.getOperand(0);
- EVT SrcVT = SrcVal.getValueType();
- EVT DstVT = Op.getValueType();
- EVT DstElementVT = DstVT.getVectorElementType();
- EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
-
- uint64_t SrcElementWidth = SrcVT.getScalarSizeInBits();
- uint64_t DstElementWidth = DstVT.getScalarSizeInBits();
- uint64_t SatWidth = SatVT.getScalarSizeInBits();
+ const EVT SrcVT = SrcVal.getValueType();
+ const EVT DstVT = Op.getValueType();
+ const EVT DstElementVT = DstVT.getVectorElementType();
+ const EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+
+ const uint64_t SrcElementWidth = SrcVT.getScalarSizeInBits();
+ const uint64_t DstElementWidth = DstVT.getScalarSizeInBits();
+ const uint64_t SatWidth = SatVT.getScalarSizeInBits();
assert(SatWidth <= DstElementWidth &&
"Saturation width cannot exceed result width");
@@ -5209,7 +5209,7 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
if (DstVT.isScalableVector())
return SDValue();
- EVT SrcElementVT = SrcVT.getVectorElementType();
+ const EVT SrcElementVT = SrcVT.getVectorElementType();
if (SrcElementVT != MVT::f64 && SrcElementVT != MVT::f32 &&
SrcElementVT != MVT::f16 && SrcElementVT != MVT::bf16)
return SDValue();
@@ -5231,7 +5231,7 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
// Try to promote the operation to a wider type if SrcVT < DstVT,
// or if type is bf16 or if the target has no +fullfp16.
- EVT PromVT = SrcVT;
+ std::optional<EVT> PromVT;
switch (SrcElementVT.getSimpleVT().SimpleTy) {
case MVT::f16:
case MVT::bf16:
@@ -5254,10 +5254,10 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
SDLoc DL(Op);
unsigned Opc = Op.getOpcode();
- if (PromVT != SrcVT && !Expand(PromVT)) {
+ if (PromVT && !Expand(*PromVT)) {
// When promoting the input type, SatWidth stays unchanged.
- SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, PromVT, SrcVal);
- if (PromVT != MVT::v8f32)
+ SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, *PromVT, SrcVal);
+ if (*PromVT != MVT::v8f32)
return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, Op.getOperand(1));
// If we are extending to a wider type (e.g. v8f16 -> v8f32) due to lack
>From 728982073c978b469616e111004c4e84f0dccf90 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Mon, 13 Jul 2026 15:34:06 +0000
Subject: [PATCH 4/4] Add assert
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 30ce51e9b8d8d..ffd16b16a4e84 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5272,11 +5272,13 @@ AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
}
// Cases that we can emit directly.
- if (CanHandleNatively(SrcVT))
+ if (CanHandleNatively(SrcVT)) {
+ assert(isTypeLegal(SrcVT) && "Expected SrcVT to be a legal type");
return DAG.getNode(Opc, DL, DstVT, SrcVal,
DAG.getValueType(DstVT.getScalarType()));
- else if (Expand(SrcVT))
+ } else if (Expand(SrcVT)) {
return SDValue();
+ }
assert((SrcElementWidth > DstElementWidth) ||
(SrcElementWidth == DstElementWidth && SatWidth < DstElementWidth));
More information about the llvm-commits
mailing list