[llvm] [LowerMatrixIntrinsics] Preserve volatile and alignment when flattening loads (PR #205759)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 3 01:48:53 PDT 2026
https://github.com/adalal-2441 updated https://github.com/llvm/llvm-project/pull/205759
>From 671669e62095e6633e7bc9698364e32927e6defa Mon Sep 17 00:00:00 2001
From: Akshat <adalal at amd.com>
Date: Thu, 25 Jun 2026 14:49:26 +0530
Subject: [PATCH] [LowerMatrixIntrinsics] Preserve volatile and alignment when
flattening loads
The dot-product flatten path rebuilt a matrix.column.major.load with
Builder.CreateLoad, dropping the volatile flag and the load's specified
alignment. Dropping volatile is a miscompile, and falling back to the type's
default ABI alignment can over-claim and fault on alignment-sensitive targets.
We should instead use CreateAlignedLoad with the intrinsic's volatile operand
and getAlignForIndex, matching the regular LowerColumnMajorLoad path.
---
.../Scalar/LowerMatrixIntrinsics.cpp | 10 +++++++-
.../dot-product-float.ll | 4 ++--
.../LowerMatrixIntrinsics/dot-product-int.ll | 2 +-
.../dot-product-volatile-align.ll | 24 +++++++++++++++++++
4 files changed, 36 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index c031574260c3c..2871431b12b31 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1721,7 +1721,15 @@ class LowerMatrixIntrinsics {
Value *Arg;
if (match(Op, m_Intrinsic<Intrinsic::matrix_column_major_load>(
m_Value(Arg)))) {
- auto *NewLoad = Builder.CreateLoad(Op->getType(), Arg);
+ auto *MatLoad = cast<IntrinsicInst>(Op);
+ bool IsVolatile = cast<ConstantInt>(MatLoad->getArgOperand(2))->isOne();
+ // Preserve the volatile flag and alignment of the original load.
+ Align Alignment = getAlignForIndex(
+ 0, MatLoad->getArgOperand(1),
+ cast<FixedVectorType>(Op->getType())->getElementType(),
+ MatLoad->getParamAlign(0));
+ auto *NewLoad = Builder.CreateAlignedLoad(Op->getType(), Arg, Alignment,
+ IsVolatile);
Op->replaceAllUsesWith(NewLoad);
eraseFromParentAndRemoveFromShapeMap(cast<Instruction>(Op));
return;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
index 97faa23cb516d..6643c6fbd4e6e 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
@@ -51,7 +51,7 @@ define <1 x float> @intrinsic_column_major_load_dot_product_float_v6(ptr %lhs_ad
; CHECK-LABEL: @intrinsic_column_major_load_dot_product_float_v6(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <6 x float>, ptr [[RHS_ADDRESS:%.*]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[LHS_ADDRESS:%.*]], align 32
+; CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[LHS_ADDRESS:%.*]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = fmul <6 x float> [[TMP0]], [[COL_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v6f32(float 0.000000e+00, <6 x float> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x float> poison, float [[TMP2]], i64 0
@@ -104,7 +104,7 @@ define <1 x double> @intrinsic_column_major_load_dot_product_double_v6(ptr %lhs_
; CHECK-LABEL: @intrinsic_column_major_load_dot_product_double_v6(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <6 x double>, ptr [[RHS_ADDRESS:%.*]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <6 x double>, ptr [[LHS_ADDRESS:%.*]], align 64
+; CHECK-NEXT: [[TMP0:%.*]] = load <6 x double>, ptr [[LHS_ADDRESS:%.*]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = fmul <6 x double> [[TMP0]], [[COL_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fadd.v6f64(double 0.000000e+00, <6 x double> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x double> poison, double [[TMP2]], i64 0
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
index de9911013479b..b4711ce047aff 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
@@ -230,7 +230,7 @@ define <1 x i32> @intrinsic_column_major_load_dot_product_i32_v8(ptr %lhs_addres
; CHECK-LABEL: @intrinsic_column_major_load_dot_product_i32_v8(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <8 x i32>, ptr [[RHS_ADDRESS:%.*]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[LHS_ADDRESS:%.*]], align 32
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[LHS_ADDRESS:%.*]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = mul <8 x i32> [[TMP0]], [[COL_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x i32> poison, i32 [[TMP2]], i64 0
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll
new file mode 100644
index 0000000000000..c48ff6f7e3ecb
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+; The dot-product / flatten lowering rewrites a matrix.column.major.load feeding
+; a 1xK * Kx1 matmul into a single vector load. The volatile flag and alignment
+; of the original intrinsic load must be carried over to the rewritten load; the
+; non-volatile operand must stay non-volatile.
+
+define <1 x float> @dot_product_flatten_volatile_align(ptr %a, ptr %b) {
+; CHECK-LABEL: @dot_product_flatten_volatile_align(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x float>, ptr [[B:%.*]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load volatile <4 x float>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[TMP0]], [[COL_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x float> poison, float [[TMP2]], i64 0
+; CHECK-NEXT: ret <1 x float> [[TMP3]]
+;
+entry:
+ %lhs = call <4 x float> @llvm.matrix.column.major.load.v4f32.i64(ptr align 8 %a, i64 1, i1 true, i32 1, i32 4)
+ %rhs = call <4 x float> @llvm.matrix.column.major.load.v4f32.i64(ptr %b, i64 4, i1 false, i32 4, i32 1)
+ %res = call reassoc <1 x float> @llvm.matrix.multiply.v1f32.v4f32.v4f32(<4 x float> %lhs, <4 x float> %rhs, i32 1, i32 4, i32 1)
+ ret <1 x float> %res
+}
More information about the llvm-commits
mailing list