[llvm] [DirectX][LongVec] Support large vector splitting for dot product (PR #216383)
Farzon Lotfi via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 11:58:12 PDT 2026
https://github.com/farzonl created https://github.com/llvm/llvm-project/pull/216383
resolves #216179
Added pre-SM 6.9 long-vector fdot lowering into minimal dot2/3/4 chunks, Added tests to confirm we only do the chunking if 6.8 or older and that 6.9 still generates dot2/3/4 but dx.dot for any larger vectors.
>From 177f8fb562c6f47ecde525d1f02f5883010403ee Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi at microsoft.com>
Date: Fri, 14 Aug 2026 14:45:20 -0400
Subject: [PATCH] [DirectX][LongVec] Support large vector splitting for dot
product
resolves #216179
Added pre-SM 6.9 long-vector fdot lowering into minimal dot2/3/4 chunks,
Added tests to confirm we only do the chunking if 6.8 or older and that
6.9 still generates dot2/3/4 but dx.dot for any larger vectors.
---
.../Target/DirectX/DXILIntrinsicExpansion.cpp | 60 ++++++++++++++-----
llvm/test/CodeGen/DirectX/fdot-longvec.ll | 37 ++++++++++++
2 files changed, 81 insertions(+), 16 deletions(-)
create mode 100644 llvm/test/CodeGen/DirectX/fdot-longvec.ll
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index 8ecc7ddc4b64f..de3c8fc40f53f 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -198,6 +198,12 @@ static Value *expand16BitIsNormal(CallInst *Orig) {
return B1;
}
+static bool shouldExpandFloatDotIntrinsic(Function &F) {
+ auto *ParamTy = cast<FixedVectorType>(F.getFunctionType()->getParamType(0));
+ return ParamTy->getNumElements() <= 4 ||
+ F.getParent()->getTargetTriple().getOSVersion() < VersionTuple(6, 9);
+}
+
static bool isIntrinsicExpansion(Function &F) {
switch (F.getIntrinsicID()) {
case Intrinsic::assume:
@@ -219,7 +225,6 @@ static bool isIntrinsicExpansion(Function &F) {
case Intrinsic::dx_isinf:
case Intrinsic::dx_isnan:
case Intrinsic::dx_normalize:
- case Intrinsic::dx_fdot:
case Intrinsic::dx_sdot:
case Intrinsic::dx_udot:
case Intrinsic::dx_sign:
@@ -233,6 +238,8 @@ static bool isIntrinsicExpansion(Function &F) {
case Intrinsic::dx_load_input:
case Intrinsic::dx_store_output:
return true;
+ case Intrinsic::dx_fdot:
+ return shouldExpandFloatDotIntrinsic(F);
case Intrinsic::dx_resource_load_rawbuffer:
return resourceAccessNeeds64BitExpansion(
F.getParent(), F.getReturnType()->getStructElementType(0),
@@ -404,10 +411,8 @@ static Value *expandAbs(CallInst *Orig) {
"dx.max");
}
-// Create appropriate DXIL float dot intrinsic for the given A and B operands
-// The appropriate opcode will be determined by the size of the operands
-// The dot product is placed in the position indicated by Orig
-static Value *expandFloatDotIntrinsic(CallInst *Orig, Value *A, Value *B) {
+// Create a DXIL dot2, dot3, or dot4 for the given operands.
+static Value *expandFloatDotChunk(CallInst *Orig, Value *A, Value *B) {
Type *ATy = A->getType();
[[maybe_unused]] Type *BTy = B->getType();
assert(ATy->isVectorTy() && BTy->isVectorTy());
@@ -418,8 +423,8 @@ static Value *expandFloatDotIntrinsic(CallInst *Orig, Value *A, Value *B) {
assert(ATy->getScalarType()->isFloatingPointTy());
- Intrinsic::ID DotIntrinsic = Intrinsic::dx_dot4;
- int NumElts = AVec->getNumElements();
+ unsigned NumElts = AVec->getNumElements();
+ Intrinsic::ID DotIntrinsic;
switch (NumElts) {
case 2:
DotIntrinsic = Intrinsic::dx_dot2;
@@ -433,24 +438,47 @@ static Value *expandFloatDotIntrinsic(CallInst *Orig, Value *A, Value *B) {
default:
reportFatalUsageError(
"Invalid dot product input vector: length is outside 2-4");
- return nullptr;
}
SmallVector<Value *> Args;
- for (int I = 0; I < NumElts; ++I)
+ for (unsigned I = 0; I < NumElts; ++I)
Args.push_back(Builder.CreateExtractElement(A, Builder.getInt32(I)));
- for (int I = 0; I < NumElts; ++I)
+ for (unsigned I = 0; I < NumElts; ++I)
Args.push_back(Builder.CreateExtractElement(B, Builder.getInt32(I)));
return Builder.CreateIntrinsic(ATy->getScalarType(), DotIntrinsic, Args,
nullptr, "dot");
}
-// Create the appropriate DXIL float dot intrinsic for the operands of Orig
-// The appropriate opcode will be determined by the size of the operands
-// The dot product is placed in the position indicated by Orig
+// Expand an arbitrary-width float dot into the minimum number of legal DXIL
+// dot2, dot3, and dot4 operations.
static Value *expandFloatDotIntrinsic(CallInst *Orig) {
- return expandFloatDotIntrinsic(Orig, Orig->getOperand(0),
- Orig->getOperand(1));
+ Value *A = Orig->getOperand(0);
+ Value *B = Orig->getOperand(1);
+ unsigned NumElts = cast<FixedVectorType>(A->getType())->getNumElements();
+ if (NumElts <= 4)
+ return expandFloatDotChunk(Orig, A, B);
+
+ assert(Orig->getModule()->getTargetTriple().getOSVersion() <
+ VersionTuple(6, 9) &&
+ "long fdot must not be expanded for shader model 6.9 or later");
+
+ IRBuilder<> Builder(Orig);
+ Value *Result = nullptr;
+ for (unsigned Offset = 0; Offset < NumElts;) {
+ unsigned Remaining = NumElts - Offset;
+ // Taking four is optimal unless it would leave an illegal one-element
+ // tail. In that case, take three and finish with dot2.
+ unsigned ChunkSize = Remaining == 5 ? 3 : std::min(Remaining, 4u);
+ SmallVector<int, 4> Mask;
+ for (unsigned I = 0; I < ChunkSize; ++I)
+ Mask.push_back(Offset + I);
+ Value *AChunk = Builder.CreateShuffleVector(A, Mask);
+ Value *BChunk = Builder.CreateShuffleVector(B, Mask);
+ Value *Chunk = expandFloatDotChunk(Orig, AChunk, BChunk);
+ Result = Result ? Builder.CreateFAdd(Result, Chunk, "dot.add") : Chunk;
+ Offset += ChunkSize;
+ }
+ return Result;
}
// Expand integer dot product to multiply and add ops
@@ -643,7 +671,7 @@ static Value *expandNormalizeIntrinsic(CallInst *Orig) {
return Builder.CreateFDiv(X, X);
}
- Value *DotProduct = expandFloatDotIntrinsic(Orig, X, X);
+ Value *DotProduct = expandFloatDotChunk(Orig, X, X);
// verify that the length is non-zero
// (if the dot product is non-zero, then the length is non-zero)
diff --git a/llvm/test/CodeGen/DirectX/fdot-longvec.ll b/llvm/test/CodeGen/DirectX/fdot-longvec.ll
new file mode 100644
index 0000000000000..9d58212c92c83
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/fdot-longvec.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK
+; RUN: opt -S -dxil-intrinsic-expansion -mtriple=dxil-pc-shadermodel6.9-library %s | FileCheck %s --check-prefixes=CHECK,SM69CHECK
+
+; CHECK-LABEL: define noundef float @dot_float5(
+; EXPCHECK: [[A0:%.*]] = shufflevector <5 x float> %a, <5 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; EXPCHECK: [[B0:%.*]] = shufflevector <5 x float> %b, <5 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; EXPCHECK: [[DOT0:%.*]] = call float @llvm.dx.dot3.f32(
+; EXPCHECK: [[A1:%.*]] = shufflevector <5 x float> %a, <5 x float> poison, <2 x i32> <i32 3, i32 4>
+; EXPCHECK: [[B1:%.*]] = shufflevector <5 x float> %b, <5 x float> poison, <2 x i32> <i32 3, i32 4>
+; EXPCHECK: [[DOT1:%.*]] = call float @llvm.dx.dot2.f32(
+; EXPCHECK: [[RESULT:%.*]] = fadd float [[DOT0]], [[DOT1]]
+; EXPCHECK: ret float [[RESULT]]
+; SM69CHECK: [[DOT:%.*]] = call float @llvm.dx.fdot.v5f32(<5 x float> %a, <5 x float> %b)
+; SM69CHECK-NEXT: ret float [[DOT]]
+define noundef float @dot_float5(<5 x float> noundef %a, <5 x float> noundef %b) {
+entry:
+ %dx.dot = call float @llvm.dx.fdot.v5f32(<5 x float> %a, <5 x float> %b)
+ ret float %dx.dot
+}
+
+; CHECK-LABEL: define noundef float @dot_float9(
+; EXPCHECK: [[DOT0:%.*]] = call float @llvm.dx.dot4.f32(
+; EXPCHECK: [[DOT1:%.*]] = call float @llvm.dx.dot3.f32(
+; EXPCHECK: [[SUM:%.*]] = fadd float [[DOT0]], [[DOT1]]
+; EXPCHECK: [[DOT2:%.*]] = call float @llvm.dx.dot2.f32(
+; EXPCHECK: [[RESULT:%.*]] = fadd float [[SUM]], [[DOT2]]
+; EXPCHECK: ret float [[RESULT]]
+; SM69CHECK: [[DOT:%.*]] = call float @llvm.dx.fdot.v9f32(<9 x float> %a, <9 x float> %b)
+; SM69CHECK-NEXT: ret float [[DOT]]
+define noundef float @dot_float9(<9 x float> noundef %a, <9 x float> noundef %b) {
+entry:
+ %dx.dot = call float @llvm.dx.fdot.v9f32(<9 x float> %a, <9 x float> %b)
+ ret float %dx.dot
+}
+
+declare float @llvm.dx.fdot.v5f32(<5 x float>, <5 x float>)
+declare float @llvm.dx.fdot.v9f32(<9 x float>, <9 x float>)
More information about the llvm-commits
mailing list