[llvm] 3e1d5e3 - [LowerMatrixIntrinsics] Preserve volatile and alignment when flattening loads (#205759)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 12:51:18 PDT 2026
Author: Akshat Dalal
Date: 2026-07-13T21:51:13+02:00
New Revision: 3e1d5e38bd9383001a6a2abff2e0fdcdf1e8313f
URL: https://github.com/llvm/llvm-project/commit/3e1d5e38bd9383001a6a2abff2e0fdcdf1e8313f
DIFF: https://github.com/llvm/llvm-project/commit/3e1d5e38bd9383001a6a2abff2e0fdcdf1e8313f.diff
LOG: [LowerMatrixIntrinsics] Preserve volatile and alignment when flattening loads (#205759)
The dot-product flatten path rebuilt a matrix.column.major.load with
Builder.CreateLoad, dropping the volatile flag and the load's specified
alignment. Dropping volatile is a miscompile, and falling back to the
type's default ABI alignment can over-claim and fault on
alignment-sensitive targets.
We should instead use CreateAlignedLoad with the intrinsic's volatile
operand and getAlignForIndex, matching the non-fused load lowering in
loadMatrix() (the standard LowerColumnMajorLoad path), which already
preserves both.
Added:
llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll
Modified:
llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index c031574260c3c..2871431b12b31 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1721,7 +1721,15 @@ class LowerMatrixIntrinsics {
Value *Arg;
if (match(Op, m_Intrinsic<Intrinsic::matrix_column_major_load>(
m_Value(Arg)))) {
- auto *NewLoad = Builder.CreateLoad(Op->getType(), Arg);
+ auto *MatLoad = cast<IntrinsicInst>(Op);
+ bool IsVolatile = cast<ConstantInt>(MatLoad->getArgOperand(2))->isOne();
+ // Preserve the volatile flag and alignment of the original load.
+ Align Alignment = getAlignForIndex(
+ 0, MatLoad->getArgOperand(1),
+ cast<FixedVectorType>(Op->getType())->getElementType(),
+ MatLoad->getParamAlign(0));
+ auto *NewLoad = Builder.CreateAlignedLoad(Op->getType(), Arg, Alignment,
+ IsVolatile);
Op->replaceAllUsesWith(NewLoad);
eraseFromParentAndRemoveFromShapeMap(cast<Instruction>(Op));
return;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
index 97faa23cb516d..6643c6fbd4e6e 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-float.ll
@@ -51,7 +51,7 @@ define <1 x float> @intrinsic_column_major_load_dot_product_float_v6(ptr %lhs_ad
; CHECK-LABEL: @intrinsic_column_major_load_dot_product_float_v6(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <6 x float>, ptr [[RHS_ADDRESS:%.*]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[LHS_ADDRESS:%.*]], align 32
+; CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[LHS_ADDRESS:%.*]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = fmul <6 x float> [[TMP0]], [[COL_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v6f32(float 0.000000e+00, <6 x float> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x float> poison, float [[TMP2]], i64 0
@@ -104,7 +104,7 @@ define <1 x double> @intrinsic_column_major_load_dot_product_double_v6(ptr %lhs_
; CHECK-LABEL: @intrinsic_column_major_load_dot_product_double_v6(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <6 x double>, ptr [[RHS_ADDRESS:%.*]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <6 x double>, ptr [[LHS_ADDRESS:%.*]], align 64
+; CHECK-NEXT: [[TMP0:%.*]] = load <6 x double>, ptr [[LHS_ADDRESS:%.*]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = fmul <6 x double> [[TMP0]], [[COL_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = call fast double @llvm.vector.reduce.fadd.v6f64(double 0.000000e+00, <6 x double> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x double> poison, double [[TMP2]], i64 0
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
index de9911013479b..b4711ce047aff 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int.ll
@@ -230,7 +230,7 @@ define <1 x i32> @intrinsic_column_major_load_dot_product_i32_v8(ptr %lhs_addres
; CHECK-LABEL: @intrinsic_column_major_load_dot_product_i32_v8(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <8 x i32>, ptr [[RHS_ADDRESS:%.*]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[LHS_ADDRESS:%.*]], align 32
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[LHS_ADDRESS:%.*]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = mul <8 x i32> [[TMP0]], [[COL_LOAD]]
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP1]])
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x i32> poison, i32 [[TMP2]], i64 0
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll
new file mode 100644
index 0000000000000..c48ff6f7e3ecb
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-volatile-align.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+; The dot-product / flatten lowering rewrites a matrix.column.major.load feeding
+; a 1xK * Kx1 matmul into a single vector load. The volatile flag and alignment
+; of the original intrinsic load must be carried over to the rewritten load; the
+; non-volatile operand must stay non-volatile.
+
+define <1 x float> @dot_product_flatten_volatile_align(ptr %a, ptr %b) {
+; CHECK-LABEL: @dot_product_flatten_volatile_align(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x float>, ptr [[B:%.*]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load volatile <4 x float>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[TMP0]], [[COL_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <1 x float> poison, float [[TMP2]], i64 0
+; CHECK-NEXT: ret <1 x float> [[TMP3]]
+;
+entry:
+ %lhs = call <4 x float> @llvm.matrix.column.major.load.v4f32.i64(ptr align 8 %a, i64 1, i1 true, i32 1, i32 4)
+ %rhs = call <4 x float> @llvm.matrix.column.major.load.v4f32.i64(ptr %b, i64 4, i1 false, i32 4, i32 1)
+ %res = call reassoc <1 x float> @llvm.matrix.multiply.v1f32.v4f32.v4f32(<4 x float> %lhs, <4 x float> %rhs, i32 1, i32 4, i32 1)
+ ret <1 x float> %res
+}
More information about the llvm-commits
mailing list