[llvm] [SPIR-V] Add PreLegalizer pattern matching for ldexp (PR #227126)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 05:14:13 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/227126
>From 07527ba0d283682c49f11369a1fcdbbc3a566b13 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 28 Sep 2026 23:34:01 +0200
Subject: [PATCH] [SPIR-V] Add PreLegalizer pattern matching for ldexp
HLSL lowers ldexp(X, Exp) to exp2(Exp) * X, so an integer exponent reaches the backend as exp2(sitofp(Exp)) * X
Fold that shape back to Ldexp
Discussed in https://github.com/llvm/llvm-project/pull/225617
---
llvm/lib/Target/SPIRV/SPIRVCombine.td | 12 +-
llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp | 101 ++++++++++++---
llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h | 4 +
.../CodeGen/SPIRV/hlsl-intrinsics/ldexp.ll | 121 ++++++++++++++++++
.../CodeGen/SPIRV/opencl/ldexp-from-exp2.ll | 21 +++
5 files changed, 242 insertions(+), 17 deletions(-)
create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ldexp.ll
create mode 100644 llvm/test/CodeGen/SPIRV/opencl/ldexp-from-exp2.ll
diff --git a/llvm/lib/Target/SPIRV/SPIRVCombine.td b/llvm/lib/Target/SPIRV/SPIRVCombine.td
index 44512229a8ff1..f04f60d0ce01d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCombine.td
+++ b/llvm/lib/Target/SPIRV/SPIRVCombine.td
@@ -29,6 +29,15 @@ def vector_select_to_faceforward_lowering : GICombineRule <
(apply [{ Helper.applySPIRVFaceForward(*${root}); }])
>;
+def ldexp_matchinfo : GIDefMatchData<"std::pair<Register, Register>">;
+
+def fmul_exp2_to_ldexp_lowering : GICombineRule <
+ (defs root:$root, ldexp_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FMUL):$root,
+ [{ return Helper.matchFMulToLdexp(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applySPIRVLdexp(*${root}, ${matchinfo}); }])
+>;
+
def matrix_transpose_lowering
: GICombineRule<
(defs root:$root),
@@ -50,6 +59,7 @@ def SPIRVPreLegalizerCombiner
[vector_length_sub_to_distance_lowering,
vector_fdiv_length_to_normalize_lowering,
vector_select_to_faceforward_lowering,
- matrix_transpose_lowering, matrix_multiply_lowering]> {
+ fmul_exp2_to_ldexp_lowering, matrix_transpose_lowering,
+ matrix_multiply_lowering]> {
let CombineAllMethodName = "tryCombineAllImpl";
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp
index 63fe8688359fd..5fb22d993ae2e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp
@@ -57,6 +57,25 @@ void SPIRVCombinerHelper::applySPIRVDistance(MachineInstr &MI) const {
MI.eraseFromParent();
}
+/// Returns the scalar of an insert-into-lane-0 plus all-zero-shuffle splat.
+static Register getSplatScalar(Register Reg, MachineRegisterInfo &MRI) {
+ MachineInstr *ShuffleInstr = MRI.getVRegDef(Reg);
+ if (ShuffleInstr->getOpcode() != TargetOpcode::G_SHUFFLE_VECTOR)
+ return Register();
+ if (!all_of(cast<GShuffleVector>(ShuffleInstr)->getMask(),
+ [](int M) { return M == 0; }))
+ return Register();
+
+ MachineInstr *InsertInstr =
+ MRI.getVRegDef(ShuffleInstr->getOperand(1).getReg());
+ if (!isSpvIntrinsic(*InsertInstr, Intrinsic::spv_insertelt))
+ return Register();
+ if (!mi_match(InsertInstr->getOperand(4).getReg(), MRI, m_ZeroInt()))
+ return Register();
+
+ return InsertInstr->getOperand(3).getReg();
+}
+
/// This match is part of a combine that
/// rewrites X / length(X) to normalize(X)
/// (vXf32 (g_fdiv
@@ -70,23 +89,11 @@ bool SPIRVCombinerHelper::matchFDivToNormalize(MachineInstr &MI) const {
Register NumeratorReg = MI.getOperand(1).getReg();
Register DivisorReg = MI.getOperand(2).getReg();
- // Match the divisor as a splat of length, inserted into lane 0.
- MachineInstr *ShuffleInstr = MRI.getVRegDef(DivisorReg);
- if (ShuffleInstr->getOpcode() != TargetOpcode::G_SHUFFLE_VECTOR)
+ // Match the divisor as a splat of length.
+ Register LengthReg = getSplatScalar(DivisorReg, MRI);
+ if (!LengthReg.isValid())
return false;
- if (!all_of(cast<GShuffleVector>(ShuffleInstr)->getMask(),
- [](int M) { return M == 0; }))
- return false;
-
- MachineInstr *InsertInstr =
- MRI.getVRegDef(ShuffleInstr->getOperand(1).getReg());
- if (!isSpvIntrinsic(*InsertInstr, Intrinsic::spv_insertelt))
- return false;
- if (!mi_match(InsertInstr->getOperand(4).getReg(), MRI, m_ZeroInt()))
- return false;
-
- MachineInstr *LengthInstr =
- MRI.getVRegDef(InsertInstr->getOperand(3).getReg());
+ MachineInstr *LengthInstr = MRI.getVRegDef(LengthReg);
if (!isSpvIntrinsic(*LengthInstr, Intrinsic::spv_length))
return false;
@@ -233,6 +240,68 @@ void SPIRVCombinerHelper::applySPIRVFaceForward(MachineInstr &MI) const {
MI.eraseFromParent();
}
+/// Ignores spv_assign_type and spv_assign_name, which the pre-legalizer drops.
+static bool hasOneRealUse(Register Reg, MachineRegisterInfo &MRI) {
+ return count_if(MRI.use_nodbg_instructions(Reg), [](MachineInstr &UseMI) {
+ return !isSpvIntrinsic(UseMI, Intrinsic::spv_assign_type) &&
+ !isSpvIntrinsic(UseMI, Intrinsic::spv_assign_name);
+ }) == 1;
+}
+
+/// Rewrites exp2(sitofp(Exp)) * X to ldexp(X, Exp)
+/// (vXf32 (g_fmul
+/// (vXf32 (g_fexp2
+/// (vXf32 splat (f32 (g_sitofp (i32 Exp))))))
+/// (vXf32 X)))
+/// ->
+/// (vXf32 (g_fldexp (vXf32 X) (i32 Exp)))
+///
+/// The exponent is restricted to a scalar i32, what OpenCL.std ldexp takes.
+/// Needs reassoc: the two differ once 2^Exp overflows or underflows.
+bool SPIRVCombinerHelper::matchFMulToLdexp(
+ MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
+ if (!MI.getFlag(MachineInstr::MIFlag::FmReassoc))
+ return false;
+
+ auto TryMatch = [&](Register Exp2Reg, Register XReg) {
+ MachineInstr *Exp2Instr = MRI.getVRegDef(Exp2Reg);
+ if (Exp2Instr->getOpcode() != TargetOpcode::G_FEXP2 ||
+ !Exp2Instr->getFlag(MachineInstr::MIFlag::FmReassoc) ||
+ !hasOneRealUse(Exp2Reg, MRI))
+ return false;
+
+ Register ExpReg = Exp2Instr->getOperand(1).getReg();
+ if (MRI.getType(ExpReg).isVector()) {
+ ExpReg = getSplatScalar(ExpReg, MRI);
+ if (!ExpReg.isValid())
+ return false;
+ }
+
+ MachineInstr *ExpInstr = MRI.getVRegDef(ExpReg);
+ if (ExpInstr->getOpcode() != TargetOpcode::G_SITOFP)
+ return false;
+ Register IntExpReg = ExpInstr->getOperand(1).getReg();
+ if (MRI.getType(IntExpReg) != LLT::scalar(32))
+ return false;
+
+ MatchInfo = {XReg, IntExpReg};
+ return true;
+ };
+
+ Register LHS = MI.getOperand(1).getReg();
+ Register RHS = MI.getOperand(2).getReg();
+ return TryMatch(LHS, RHS) || TryMatch(RHS, LHS);
+}
+
+void SPIRVCombinerHelper::applySPIRVLdexp(
+ MachineInstr &MI, const std::pair<Register, Register> &MatchInfo) const {
+ Builder.setInstrAndDebugLoc(MI);
+ Builder.buildFLdexp(MI.getOperand(0).getReg(), MatchInfo.first,
+ MatchInfo.second, MI.getFlags());
+
+ MI.eraseFromParent();
+}
+
void SPIRVCombinerHelper::applyMatrixTranspose(MachineInstr &MI) const {
Register ResReg = MI.getOperand(0).getReg();
Register InReg = MI.getOperand(2).getReg();
diff --git a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h
index af9bcd7347980..269eb1d4b4011 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h
+++ b/llvm/lib/Target/SPIRV/SPIRVCombinerHelper.h
@@ -35,6 +35,10 @@ class SPIRVCombinerHelper : public CombinerHelper {
void applySPIRVNormalize(MachineInstr &MI) const;
bool matchSelectToFaceForward(MachineInstr &MI) const;
void applySPIRVFaceForward(MachineInstr &MI) const;
+ bool matchFMulToLdexp(MachineInstr &MI,
+ std::pair<Register, Register> &MatchInfo) const;
+ void applySPIRVLdexp(MachineInstr &MI,
+ const std::pair<Register, Register> &MatchInfo) const;
void applyMatrixTranspose(MachineInstr &MI) const;
void applyMatrixMultiply(MachineInstr &MI) const;
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ldexp.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ldexp.ll
new file mode 100644
index 0000000000000..472f1d5f44eb7
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/ldexp.ll
@@ -0,0 +1,121 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
+
+; HLSL lowers ldexp(X, Exp) to exp2(Exp) * X, so an i32 exponent arrives as
+; exp2(sitofp(Exp)) * X.
+
+; CHECK-DAG: %[[#ExtInstSetId:]] = OpExtInstImport "GLSL.std.450"
+; CHECK-DAG: %[[#Float:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#Int32:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#Float4:]] = OpTypeVector %[[#Float]] 4
+; CHECK-DAG: %[[#Int4:]] = OpTypeVector %[[#Int32]] 4
+
+; CHECK-LABEL: Begin function test_exp2_mul{{$}}
+; CHECK-NOT: Exp2
+; CHECK: %[[#]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Ldexp
+define float @test_exp2_mul(float %x, i32 %k) {
+ %e = sitofp i32 %k to float
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %t, %x
+ ret float %r
+}
+
+; CHECK-LABEL: Begin function test_mul_exp2{{$}}
+; CHECK-NOT: Exp2
+; CHECK: %[[#]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Ldexp
+define float @test_mul_exp2(float %x, i32 %k) {
+ %e = sitofp i32 %k to float
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %x, %t
+ ret float %r
+}
+
+; CHECK-LABEL: Begin function test_exp2_mul_v4f32{{$}}
+; CHECK-NOT: Exp2
+; CHECK: %[[#Splat:]] = OpCompositeConstruct %[[#Int4]]
+; CHECK: %[[#]] = OpExtInst %[[#Float4]] %[[#ExtInstSetId]] Ldexp %[[#]] %[[#Splat]]
+define <4 x float> @test_exp2_mul_v4f32(<4 x float> %x, i32 %k) {
+ %e = sitofp i32 %k to float
+ %se = insertelement <4 x float> poison, float %e, i32 0
+ %sp = shufflevector <4 x float> %se, <4 x float> poison, <4 x i32> zeroinitializer
+ %t = call reassoc <4 x float> @llvm.exp2.v4f32(<4 x float> %sp)
+ %r = fmul reassoc <4 x float> %t, %x
+ ret <4 x float> %r
+}
+
+; CHECK-LABEL: Begin function test_no_reassoc{{$}}
+; CHECK-NOT: Ldexp
+; CHECK: %[[#Exp2N:]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Exp2
+; CHECK: %[[#]] = OpFMul %[[#Float]] %[[#Exp2N]] %[[#]]
+define float @test_no_reassoc(float %x, i32 %k) {
+ %e = sitofp i32 %k to float
+ %t = call float @llvm.exp2.f32(float %e)
+ %r = fmul float %t, %x
+ ret float %r
+}
+
+; Ldexp truncates its exponent, so a float one must stay an exp2 and a multiply.
+
+; CHECK-LABEL: Begin function test_float_exponent{{$}}
+; CHECK-NOT: Ldexp
+; CHECK: %[[#Exp2:]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Exp2
+; CHECK: %[[#]] = OpFMul %[[#Float]] %[[#Exp2]] %[[#]]
+define float @test_float_exponent(float %x, float %e) {
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %t, %x
+ ret float %r
+}
+
+; A per-lane exponent does not fit G_FLDEXP, whose exponent is scalar.
+
+; CHECK-LABEL: Begin function test_vector_exponent{{$}}
+; CHECK-NOT: Ldexp
+; CHECK: %[[#Exp2V:]] = OpExtInst %[[#Float4]] %[[#ExtInstSetId]] Exp2
+; CHECK: %[[#]] = OpFMul %[[#Float4]] %[[#Exp2V]] %[[#]]
+define <4 x float> @test_vector_exponent(<4 x float> %x, <4 x i32> %k) {
+ %e = sitofp <4 x i32> %k to <4 x float>
+ %t = call reassoc <4 x float> @llvm.exp2.v4f32(<4 x float> %e)
+ %r = fmul reassoc <4 x float> %t, %x
+ ret <4 x float> %r
+}
+
+; Only an i32 exponent folds: the fold is not environment gated, and OpenCL.std
+; ldexp takes i32. Both of the next two bail out for that reason.
+
+; CHECK-LABEL: Begin function test_i64_exponent{{$}}
+; CHECK-NOT: Ldexp
+; CHECK: %[[#Exp2L:]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Exp2
+; CHECK: %[[#]] = OpFMul %[[#Float]] %[[#Exp2L]] %[[#]]
+define float @test_i64_exponent(float %x, i64 %k) {
+ %e = sitofp i64 %k to float
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %t, %x
+ ret float %r
+}
+
+; CHECK-LABEL: Begin function test_i1_exponent{{$}}
+; CHECK-NOT: Ldexp
+; CHECK: %[[#Exp2B:]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Exp2
+; CHECK: %[[#]] = OpFMul %[[#Float]] %[[#Exp2B]] %[[#]]
+define float @test_i1_exponent(float %x, i1 %k) {
+ %e = sitofp i1 %k to float
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %t, %x
+ ret float %r
+}
+
+; CHECK-LABEL: Begin function test_exp2_multi_use{{$}}
+; CHECK-NOT: Ldexp
+; CHECK: %[[#Exp2M:]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] Exp2
+; CHECK: %[[#Mul:]] = OpFMul %[[#Float]] %[[#Exp2M]] %[[#]]
+; CHECK: %[[#]] = OpFAdd %[[#Float]] %[[#Mul]] %[[#Exp2M]]
+define float @test_exp2_multi_use(float %x, i32 %k) {
+ %e = sitofp i32 %k to float
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %t, %x
+ %s = fadd float %r, %t
+ ret float %s
+}
+
+declare float @llvm.exp2.f32(float)
+declare <4 x float> @llvm.exp2.v4f32(<4 x float>)
diff --git a/llvm/test/CodeGen/SPIRV/opencl/ldexp-from-exp2.ll b/llvm/test/CodeGen/SPIRV/opencl/ldexp-from-exp2.ll
new file mode 100644
index 0000000000000..3eba1675cd89a
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/opencl/ldexp-from-exp2.ll
@@ -0,0 +1,21 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; The fold is not environment gated. The match is covered by the Vulkan test,
+; this checks the OpenCL.std mapping.
+
+; CHECK-DAG: %[[#ExtInstSetId:]] = OpExtInstImport "OpenCL.std"
+; CHECK-DAG: %[[#Float:]] = OpTypeFloat 32
+
+; CHECK-LABEL: Begin function test_i32_exponent{{$}}
+; CHECK-NOT: exp2
+; CHECK: %[[#]] = OpExtInst %[[#Float]] %[[#ExtInstSetId]] ldexp
+define spir_kernel void @test_i32_exponent(float %x, i32 %k, ptr %out) {
+ %e = sitofp i32 %k to float
+ %t = call reassoc float @llvm.exp2.f32(float %e)
+ %r = fmul reassoc float %t, %x
+ store float %r, ptr %out
+ ret void
+}
+
+declare float @llvm.exp2.f32(float)
More information about the llvm-commits
mailing list