[llvm] [SelectionDAG] Scalarize one-lane intrinsic results (PR #223622)
Bastian Hagedorn via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 16 22:55:34 PDT 2026
https://github.com/bastianhagedorn updated https://github.com/llvm/llvm-project/pull/223622
>From e2c3a769fff39ebbc53b119f830496640b48bf4b Mon Sep 17 00:00:00 2001
From: Bastian Hagedorn <bhagedorn at nvidia.com>
Date: Tue, 15 Sep 2026 07:16:19 +0000
Subject: [PATCH 1/4] [SelectionDAG] Scalarize one-lane intrinsic results
---
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 29 +++++++++++++++++++
llvm/test/CodeGen/NVPTX/f32-ex2.ll | 20 +++++++++++++
3 files changed, 50 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 6fc6d61c6a38d..ccff9a07a4d29 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -826,6 +826,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
+ SDValue ScalarizeVecRes_INTRINSIC_WO_CHAIN(SDNode *N);
SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
SDValue ScalarizeVecRes_InregOp(SDNode *N);
SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index f6e7feab57ee9..71d1f7c632cee 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -83,6 +83,8 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
R = ScalarizeVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
break;
case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
+ case ISD::INTRINSIC_WO_CHAIN:
+ R = ScalarizeVecRes_INTRINSIC_WO_CHAIN(N); break;
case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
case ISD::VECTOR_DEINTERLEAVE:
case ISD::VECTOR_INTERLEAVE:
@@ -586,6 +588,33 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
return Result;
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_INTRINSIC_WO_CHAIN(SDNode *N) {
+ assert(N->getValueType(0).getVectorNumElements() == 1 &&
+ "Unexpected vector type");
+
+ SDLoc DL(N);
+ SmallVector<SDValue, 4> Ops{N->getOperand(0)};
+ for (unsigned I = 1; I < N->getNumOperands(); ++I) {
+ SDValue Operand = N->getOperand(I);
+ EVT OperandVT = Operand.getValueType();
+ if (OperandVT.isVector()) {
+ assert(OperandVT.getVectorNumElements() == 1 &&
+ "Unexpected vector operand type");
+ if (getTypeAction(OperandVT) == TargetLowering::TypeScalarizeVector) {
+ Operand = GetScalarizedVector(Operand);
+ } else {
+ Operand = DAG.getExtractVectorElt(DL, OperandVT.getVectorElementType(),
+ Operand, 0);
+ }
+ }
+ Ops.push_back(Operand);
+ }
+
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
+ N->getValueType(0).getVectorElementType(), Ops,
+ N->getFlags());
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
// Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
EVT DestVT = N->getValueType(0).getVectorElementType();
diff --git a/llvm/test/CodeGen/NVPTX/f32-ex2.ll b/llvm/test/CodeGen/NVPTX/f32-ex2.ll
index db3dd4a9e6011..523e681c9ea9e 100644
--- a/llvm/test/CodeGen/NVPTX/f32-ex2.ll
+++ b/llvm/test/CodeGen/NVPTX/f32-ex2.ll
@@ -5,6 +5,8 @@ target triple = "nvptx-nvidia-cuda"
declare float @llvm.nvvm.ex2.approx.f32(float)
declare float @llvm.nvvm.ex2.approx.ftz.f32(float)
+declare <1 x float> @llvm.nvvm.ex2.approx.v1f32(<1 x float>)
+declare <1 x float> @llvm.nvvm.ex2.approx.ftz.v1f32(<1 x float>)
; CHECK-LABEL: ex2_float
define float @ex2_float(float %0) {
@@ -35,3 +37,21 @@ define float @ex2_float_ftz(float %0) {
%res = call float @llvm.nvvm.ex2.approx.ftz.f32(float %0)
ret float %res
}
+
+; CHECK-LABEL: ex2_float_v1
+define <1 x float> @ex2_float_v1(<1 x float> %0) {
+; CHECK-LABEL: ex2_float_v1(
+; CHECK: {
+; CHECK: ex2.approx.f32
+ %res = call <1 x float> @llvm.nvvm.ex2.approx.v1f32(<1 x float> %0)
+ ret <1 x float> %res
+}
+
+; CHECK-LABEL: ex2_float_v1_ftz
+define <1 x float> @ex2_float_v1_ftz(<1 x float> %0) {
+; CHECK-LABEL: ex2_float_v1_ftz(
+; CHECK: {
+; CHECK: ex2.approx.ftz.f32
+ %res = call <1 x float> @llvm.nvvm.ex2.approx.ftz.v1f32(<1 x float> %0)
+ ret <1 x float> %res
+}
>From 14fa1393b2e645a6aae1185d43317018a397814c Mon Sep 17 00:00:00 2001
From: Bastian Hagedorn <bhagedorn at nvidia.com>
Date: Tue, 15 Sep 2026 08:33:20 +0000
Subject: [PATCH 2/4] [SelectionDAG] Fix formatting
---
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 71d1f7c632cee..487854fa3f4fd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -84,7 +84,8 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
break;
case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
case ISD::INTRINSIC_WO_CHAIN:
- R = ScalarizeVecRes_INTRINSIC_WO_CHAIN(N); break;
+ R = ScalarizeVecRes_INTRINSIC_WO_CHAIN(N);
+ break;
case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
case ISD::VECTOR_DEINTERLEAVE:
case ISD::VECTOR_INTERLEAVE:
>From d79c34f2d1f171636ea81cf29083f981d7a01981 Mon Sep 17 00:00:00 2001
From: Bastian Hagedorn <bhagedorn at nvidia.com>
Date: Thu, 17 Sep 2026 05:45:42 +0000
Subject: [PATCH 3/4] [NVPTX] Lower v1 EX2 intrinsics
---
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 -
.../SelectionDAG/LegalizeVectorTypes.cpp | 30 -------------------
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 30 ++++++++++++++++++-
3 files changed, 29 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index ccff9a07a4d29..6fc6d61c6a38d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -826,7 +826,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
- SDValue ScalarizeVecRes_INTRINSIC_WO_CHAIN(SDNode *N);
SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
SDValue ScalarizeVecRes_InregOp(SDNode *N);
SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 487854fa3f4fd..f6e7feab57ee9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -83,9 +83,6 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
R = ScalarizeVecRes_ATOMIC_LOAD(cast<AtomicSDNode>(N));
break;
case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
- case ISD::INTRINSIC_WO_CHAIN:
- R = ScalarizeVecRes_INTRINSIC_WO_CHAIN(N);
- break;
case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
case ISD::VECTOR_DEINTERLEAVE:
case ISD::VECTOR_INTERLEAVE:
@@ -589,33 +586,6 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
return Result;
}
-SDValue DAGTypeLegalizer::ScalarizeVecRes_INTRINSIC_WO_CHAIN(SDNode *N) {
- assert(N->getValueType(0).getVectorNumElements() == 1 &&
- "Unexpected vector type");
-
- SDLoc DL(N);
- SmallVector<SDValue, 4> Ops{N->getOperand(0)};
- for (unsigned I = 1; I < N->getNumOperands(); ++I) {
- SDValue Operand = N->getOperand(I);
- EVT OperandVT = Operand.getValueType();
- if (OperandVT.isVector()) {
- assert(OperandVT.getVectorNumElements() == 1 &&
- "Unexpected vector operand type");
- if (getTypeAction(OperandVT) == TargetLowering::TypeScalarizeVector) {
- Operand = GetScalarizedVector(Operand);
- } else {
- Operand = DAG.getExtractVectorElt(DL, OperandVT.getVectorElementType(),
- Operand, 0);
- }
- }
- Ops.push_back(Operand);
- }
-
- return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
- N->getValueType(0).getVectorElementType(), Ops,
- N->getFlags());
-}
-
SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
// Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
EVT DestVT = N->getValueType(0).getVectorElementType();
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index a21af472576cd..da18689dd1cc4 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1166,10 +1166,12 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
// Enable custom lowering for the following:
// * MVT::i128 - clusterlaunchcontrol
// * MVT::i32 - prmt
+ // * MVT::v1f32 - ex2
// * MVT::v4f32 - cvt_rs fp{4/6/8}x4 intrinsics
// * MVT::Other - internal.addrspace.wrap
setOperationAction(ISD::INTRINSIC_WO_CHAIN,
- {MVT::i32, MVT::i128, MVT::v4f32, MVT::Other}, Custom);
+ {MVT::i32, MVT::i128, MVT::v1f32, MVT::v4f32, MVT::Other},
+ Custom);
// Custom lowering for bswap
setOperationAction(ISD::BSWAP, {MVT::i16, MVT::i32, MVT::i64, MVT::v2i16},
@@ -7608,6 +7610,29 @@ static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
}
}
+static void ReplaceINTRINSIC_WO_CHAIN(
+ SDNode *N, SelectionDAG &DAG, SmallVectorImpl<SDValue> &Results) {
+ assert(N->getValueType(0) == MVT::v1f32 &&
+ "Custom handling of non-v1f32 intrinsic?");
+
+ switch (N->getConstantOperandVal(0)) {
+ default:
+ return;
+ case Intrinsic::nvvm_ex2_approx:
+ case Intrinsic::nvvm_ex2_approx_ftz:
+ break;
+ }
+
+ SDLoc DL(N);
+ SDValue ScalarOperand =
+ DAG.getExtractVectorElt(DL, MVT::f32, N->getOperand(1), 0);
+ SDValue ScalarResult = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, MVT::f32,
+ {N->getOperand(0), ScalarOperand}, N->getFlags());
+ Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v1f32,
+ ScalarResult));
+}
+
static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,
SmallVectorImpl<SDValue> &Results) {
// Change the CopyFromReg to output 2 64-bit results instead of a 128-bit
@@ -7706,6 +7731,9 @@ void NVPTXTargetLowering::ReplaceNodeResults(
case ISD::INTRINSIC_W_CHAIN:
ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
return;
+ case ISD::INTRINSIC_WO_CHAIN:
+ ReplaceINTRINSIC_WO_CHAIN(N, DAG, Results);
+ return;
case ISD::CopyFromReg:
ReplaceCopyFromReg_128(N, DAG, Results);
return;
>From f31a30b4e185453407e3a63cdc52a12976b1da31 Mon Sep 17 00:00:00 2001
From: Bastian Hagedorn <bhagedorn at nvidia.com>
Date: Thu, 17 Sep 2026 05:54:53 +0000
Subject: [PATCH 4/4] [NVPTX] Fix EX2 v1 formatting
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index da18689dd1cc4..62945c76ba2d2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7610,8 +7610,8 @@ static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
}
}
-static void ReplaceINTRINSIC_WO_CHAIN(
- SDNode *N, SelectionDAG &DAG, SmallVectorImpl<SDValue> &Results) {
+static void ReplaceINTRINSIC_WO_CHAIN(SDNode *N, SelectionDAG &DAG,
+ SmallVectorImpl<SDValue> &Results) {
assert(N->getValueType(0) == MVT::v1f32 &&
"Custom handling of non-v1f32 intrinsic?");
@@ -7626,11 +7626,11 @@ static void ReplaceINTRINSIC_WO_CHAIN(
SDLoc DL(N);
SDValue ScalarOperand =
DAG.getExtractVectorElt(DL, MVT::f32, N->getOperand(1), 0);
- SDValue ScalarResult = DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, MVT::f32,
- {N->getOperand(0), ScalarOperand}, N->getFlags());
- Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v1f32,
- ScalarResult));
+ SDValue ScalarResult =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::f32,
+ {N->getOperand(0), ScalarOperand}, N->getFlags());
+ Results.push_back(
+ DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v1f32, ScalarResult));
}
static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG,
More information about the llvm-commits
mailing list