[llvm-branch-commits] [llvm] [Matrix] Add/remove transposes to row vectors to see them as column vectors (PR #211443)
Jon Roelofs via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Jul 22 18:54:08 PDT 2026
https://github.com/jroelofs created https://github.com/llvm/llvm-project/pull/211443
In column-major, add or remove transposes to row vectors in order to see
them as column vector during the rest of the lowering.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From e278418a70c481adf47a9a94ee2b5e4c4d86bbcf Mon Sep 17 00:00:00 2001
From: Francis Visoiu Mistrih <francisvm at apple.com>
Date: Thu, 2 Jul 2026 17:01:00 -0700
Subject: [PATCH] [Matrix] Add/remove transposes to row vectors to see them as
column vectors
In column-major, add or remove transposes to row vectors in order to see
them as column vector during the rest of the lowering.
---
.../Scalar/LowerMatrixIntrinsics.cpp | 140 +++++++++++++++++
.../dot-product-int-also-fusable-multiply.ll | 31 +---
.../matrix-row-vector-transpose.ll | 143 ++++++++++++++++++
.../LowerMatrixIntrinsics/multiply-double.ll | 11 +-
.../LowerMatrixIntrinsics/multiply-float.ll | 11 +-
.../LowerMatrixIntrinsics/multiply-i32.ll | 11 +-
.../transpose-opts-lifting.ll | 103 +++++++------
7 files changed, 360 insertions(+), 90 deletions(-)
create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/matrix-row-vector-transpose.ll
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 99116ebb4db3e..b8eb5d3f94984 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1107,6 +1107,144 @@ class LowerMatrixIntrinsics {
return false;
}
+ /// Transpose the \p OpNIdx operand of the \p II call from a 1xN to a (Nx1)^T.
+ /// Note that this doesn't update the ShapeInfo and relies on it being called
+ /// before shape propagation.
+ Instruction *transposeOperand(IntrinsicInst &II, unsigned OpNIdx,
+ unsigned NumRows, unsigned NumColumns) {
+ Value *OpN = II.getArgOperand(OpNIdx);
+ // Don't add a transpose if the operand is already transposed.
+ if (match(OpN, m_Intrinsic<Intrinsic::matrix_transpose>()))
+ return nullptr;
+ // Check if the operand is a row vector: 1xN (RxM shape).
+ if (NumRows != 1)
+ return nullptr;
+ IRBuilder<> IB(&II);
+ MatrixBuilder MBuilder(IB);
+ // Transpose the operand so that the lowering pass sees it as a transposed
+ // column vector.
+ Instruction *T = MBuilder.CreateMatrixTranspose(OpN, NumColumns, NumRows);
+ return T;
+ }
+
+ /// If an operand is transposed from a Nx1 to a 1xN, drop the
+ /// transpose and update the shape: (1xN)^T -> Nx1.
+ /// Note that this doesn't update the ShapeInfo and relies on it being called
+ /// before shape propagation.
+ Instruction *untransposeOperand(IntrinsicInst &II, unsigned OpNIdx,
+ unsigned NumRows, unsigned NumColumns) {
+ Value *OpN = II.getArgOperand(OpNIdx);
+ Instruction *N = nullptr;
+ // Look for the transpose.
+ if (!match(OpN, m_Intrinsic<Intrinsic::matrix_transpose>(
+ m_Instruction(N), m_Value(), m_Value())))
+ return nullptr;
+ // If it's not a row vector, leave it as is.
+ if (NumColumns != 1)
+ return nullptr;
+
+ return N;
+ }
+
+ /// When lowering an NN matrix multiply A * B, where A is a row vector matrix
+ /// in a column-major setting (1xM * MxC), we currently operate on the columns
+ /// of the matrix which results in scalar code instead of vectorized dot
+ /// products.
+ ///
+ /// In order to vectorize this properly, we want to treat A as a column
+ /// vector, which has the same memory layout as row vector. To do
+ /// that, we use a combine to insert a transpose on the operand that is a
+ /// row vector, which ends up being a free operation for a 1xM matrix given
+ /// the memory layout. Example:
+ ///
+ /// 1x3 * 3x4 -> (3x1)^T * 3x4
+ ///
+ /// (RxM * MxC -> (MxR)^T * MxC)
+ ///
+ /// This has the effect of triggering the TN (or NT) optimized lowering in
+ /// the matrix lowering pass which avoids scalarizing the entire operation.
+ ///
+ /// The goal is to have as few row vectors as possible, so if an operand is
+ /// already transposed to become a row vector, remove the transpose to make it
+ /// a column vector, which we can easily vectorize.
+ ///
+ /// Return true if II was changed.
+ bool transposeRowVectorOperands(IntrinsicInst &II, unsigned R, unsigned M,
+ unsigned C) {
+ bool A_t =
+ match(II.getArgOperand(0), m_Intrinsic<Intrinsic::matrix_transpose>());
+ bool B_t =
+ match(II.getArgOperand(1), m_Intrinsic<Intrinsic::matrix_transpose>());
+ if (!A_t && !B_t && R == 1 && C == 1)
+ return false;
+ assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&
+ "Optimization only supports column-major layout!");
+ // 1xM * MxC -> (Mx1)^T * MxC
+ // TN lowering -> vector loads + inner product
+ if (Instruction *T = transposeOperand(II, 0, R, M)) {
+ II.setOperand(0, T);
+ return true;
+ }
+
+ // (1xM)^T * 1xC -> Mx1 * 1xC
+ if (Instruction *N = untransposeOperand(II, 0, R, M)) {
+ auto *OldOp = cast<Instruction>(II.getOperand(0));
+ II.setOperand(0, N);
+ if (OldOp->use_empty())
+ OldOp->eraseFromParent();
+ return true;
+ }
+
+ // Rx1 * 1xC -> Rx1 * (Cx1)^T
+ // NT lowering -> vector loads + outer product
+ if (Instruction *T = transposeOperand(II, 1, M, C)) {
+ II.setOperand(1, T);
+ return true;
+ }
+
+ // RxM * (1xM)^T -> RxM * Mx1
+ if (Instruction *N = untransposeOperand(II, 1, M, C)) {
+ auto *OldOp = cast<Instruction>(II.getOperand(1));
+ II.setOperand(1, N);
+ if (OldOp->use_empty())
+ OldOp->eraseFromParent();
+ return true;
+ }
+ return false;
+ }
+
+ /// Perform inst-combine-like transformations on matrix operations.
+ /// Note that this doesn't update the ShapeInfo and relies on it being called
+ /// before shape propagation.
+ void combineMatrixOps() {
+ // Only do this for column major.
+ if (MatrixLayout != MatrixLayoutTy::ColumnMajor)
+ return;
+ // Try to transpose all row vectors to column vectors if they are matrix
+ // multiply operands.
+ for (BasicBlock &BB : Func) {
+ BasicBlock::iterator Next;
+ for (auto It = BB.begin(); It != BB.end(); It = Next) {
+ Instruction &I = *It;
+ Next = std::next(It);
+
+ Value *A, *B;
+ ConstantInt *R, *M, *C;
+ if (!match(&I, m_Intrinsic<Intrinsic::matrix_multiply>(
+ m_Value(A), m_Value(B), m_ConstantInt(R),
+ m_ConstantInt(M), m_ConstantInt(C))))
+ continue;
+ auto &II = cast<IntrinsicInst>(I);
+ if (transposeRowVectorOperands(II, R->getZExtValue(), M->getZExtValue(),
+ C->getZExtValue())) {
+ // Re-process the changed instruction.
+ Next = It;
+ continue;
+ }
+ }
+ }
+ }
+
/// Try moving transposes in order to fold them away or into multiplies.
bool optimizeTransposes() {
bool Changed = false;
@@ -1191,6 +1329,8 @@ class LowerMatrixIntrinsics {
}
}
+ combineMatrixOps();
+
// Propagate shapes until nothing changes any longer.
while (!WorkList.empty()) {
WorkList = propagateShapeForward(WorkList);
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll
index b78d56646d9e4..7767542668ba5 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/dot-product-int-also-fusable-multiply.ll
@@ -5,34 +5,13 @@ define void @test(ptr %p, <8 x i32> %x) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[P:%.*]], <8 x i32> [[X:%.*]]) {
; CHECK-NEXT: [[L:%.*]] = load <8 x i32>, ptr [[P]], align 4
-; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 3>
-; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 4>
-; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 5>
-; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 6>
-; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <1 x i32> <i32 7>
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <1 x i32> [[SPLIT]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[TMP5]], i64 2
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <1 x i32> [[SPLIT3]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP7]], i64 3
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <1 x i32> [[SPLIT4]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[TMP9]], i64 4
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <1 x i32> [[SPLIT5]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP11]], i64 5
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <1 x i32> [[SPLIT6]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[TMP13]], i64 6
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <1 x i32> [[SPLIT7]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[TMP15]], i64 7
-; CHECK-NEXT: [[TMP17:%.*]] = mul <8 x i32> [[L]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = mul <8 x i32> [[X]], [[L]]
; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP17]])
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <1 x i32> poison, i32 [[TMP18]], i64 0
-; CHECK-NEXT: [[E:%.*]] = extractelement <1 x i32> [[TMP19]], i64 0
+; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <1 x i32> [[TMP19]], <1 x i32> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <1 x i32> [[SPLIT]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <1 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: [[E:%.*]] = extractelement <1 x i32> [[TMP5]], i64 0
; CHECK-NEXT: store i32 [[E]], ptr [[P]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/matrix-row-vector-transpose.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/matrix-row-vector-transpose.ll
new file mode 100644
index 0000000000000..772cfc1f1563e
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/matrix-row-vector-transpose.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -enable-matrix=true -passes=lower-matrix-intrinsics -S %s | FileCheck %s
+;
+; Verify that 1xM row-vector multiply operands get a transpose inserted before
+; lowering so the TN/NT paths produce efficient vector code instead of scalars.
+
+define void @multiply_1x4x4(ptr %A, ptr %B, ptr %C) {
+; CHECK-LABEL: define void @multiply_1x4x4(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]]) {
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, ptr [[A]], align 32
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <4 x double>, ptr [[B]], align 128
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr inbounds double, ptr [[B]], i64 4
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <4 x double>, ptr [[VEC_GEP]], align 32
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr inbounds double, ptr [[B]], i64 8
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, ptr [[VEC_GEP3]], align 64
+; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr inbounds double, ptr [[B]], i64 12
+; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <4 x double>, ptr [[VEC_GEP5]], align 32
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <1 x double> poison, double [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <1 x double> poison, double [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 2
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <1 x double> poison, double [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 3
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <1 x double> poison, double [[TMP7]], i64 0
+; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <1 x double> [[TMP2]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP9]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT: [[BLOCK7:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT8:%.*]] = insertelement <1 x double> poison, double [[TMP11]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT9:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT8]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = fmul <1 x double> [[BLOCK7]], [[SPLAT_SPLAT9]]
+; CHECK-NEXT: [[TMP14:%.*]] = fadd <1 x double> [[TMP10]], [[TMP12]]
+; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <1 x double> [[TMP6]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> poison, double [[TMP13]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = fmul <1 x double> [[BLOCK10]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT: [[TMP22:%.*]] = fadd <1 x double> [[TMP14]], [[TMP20]]
+; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <1 x double> [[TMP8]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 3
+; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> poison, double [[TMP15]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <1 x double> [[BLOCK13]], [[SPLAT_SPLAT15]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <1 x double> [[TMP16]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <1 x double> poison, <1 x double> [[TMP17]], <1 x i32> <i32 1>
+; CHECK-NEXT: [[BLOCK16:%.*]] = shufflevector <1 x double> [[TMP2]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> poison, double [[TMP19]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP30:%.*]] = fmul <1 x double> [[BLOCK16]], [[SPLAT_SPLAT18]]
+; CHECK-NEXT: [[BLOCK19:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> poison, double [[TMP21]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = fmul <1 x double> [[BLOCK19]], [[SPLAT_SPLAT21]]
+; CHECK-NEXT: [[TMP34:%.*]] = fadd <1 x double> [[TMP30]], [[TMP32]]
+; CHECK-NEXT: [[BLOCK22:%.*]] = shufflevector <1 x double> [[TMP6]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> poison, double [[TMP23]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP40:%.*]] = fmul <1 x double> [[BLOCK22]], [[SPLAT_SPLAT24]]
+; CHECK-NEXT: [[TMP50:%.*]] = fadd <1 x double> [[TMP34]], [[TMP40]]
+; CHECK-NEXT: [[BLOCK25:%.*]] = shufflevector <1 x double> [[TMP8]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 3
+; CHECK-NEXT: [[SPLAT_SPLATINSERT26:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT27:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT26]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP53:%.*]] = fmul <1 x double> [[BLOCK25]], [[SPLAT_SPLAT27]]
+; CHECK-NEXT: [[TMP26:%.*]] = fadd <1 x double> [[TMP50]], [[TMP53]]
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <1 x double> [[TMP26]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <1 x double> poison, <1 x double> [[TMP27]], <1 x i32> <i32 1>
+; CHECK-NEXT: [[BLOCK28:%.*]] = shufflevector <1 x double> [[TMP2]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = insertelement <1 x double> poison, double [[TMP29]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT29]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP56:%.*]] = fmul <1 x double> [[BLOCK28]], [[SPLAT_SPLAT30]]
+; CHECK-NEXT: [[BLOCK31:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = insertelement <1 x double> poison, double [[TMP31]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP58:%.*]] = fmul <1 x double> [[BLOCK31]], [[SPLAT_SPLAT33]]
+; CHECK-NEXT: [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
+; CHECK-NEXT: [[BLOCK34:%.*]] = shufflevector <1 x double> [[TMP6]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <4 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = insertelement <1 x double> poison, double [[TMP33]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT36:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT35]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP60:%.*]] = fmul <1 x double> [[BLOCK34]], [[SPLAT_SPLAT36]]
+; CHECK-NEXT: [[TMP42:%.*]] = fadd <1 x double> [[TMP59]], [[TMP60]]
+; CHECK-NEXT: [[BLOCK37:%.*]] = shufflevector <1 x double> [[TMP8]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x double> [[COL_LOAD4]], i64 3
+; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT39:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP44:%.*]] = fmul <1 x double> [[BLOCK37]], [[SPLAT_SPLAT39]]
+; CHECK-NEXT: [[TMP36:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
+; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <1 x double> [[TMP36]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <1 x double> poison, <1 x double> [[TMP37]], <1 x i32> <i32 1>
+; CHECK-NEXT: [[BLOCK40:%.*]] = shufflevector <1 x double> [[TMP2]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP39:%.*]] = extractelement <4 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT41:%.*]] = insertelement <1 x double> poison, double [[TMP39]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT42:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT41]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK40]], [[SPLAT_SPLAT42]]
+; CHECK-NEXT: [[BLOCK43:%.*]] = shufflevector <1 x double> [[TMP4]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT44:%.*]] = insertelement <1 x double> poison, double [[TMP41]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT45:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT44]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK43]], [[SPLAT_SPLAT45]]
+; CHECK-NEXT: [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[BLOCK46:%.*]] = shufflevector <1 x double> [[TMP6]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <4 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT47:%.*]] = insertelement <1 x double> poison, double [[TMP43]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT48:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT47]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP54:%.*]] = fmul <1 x double> [[BLOCK46]], [[SPLAT_SPLAT48]]
+; CHECK-NEXT: [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
+; CHECK-NEXT: [[BLOCK49:%.*]] = shufflevector <1 x double> [[TMP8]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x double> [[COL_LOAD6]], i64 3
+; CHECK-NEXT: [[SPLAT_SPLATINSERT50:%.*]] = insertelement <1 x double> poison, double [[TMP45]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLAT51:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT50]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP57:%.*]] = fmul <1 x double> [[BLOCK49]], [[SPLAT_SPLAT51]]
+; CHECK-NEXT: [[TMP46:%.*]] = fadd <1 x double> [[TMP55]], [[TMP57]]
+; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <1 x double> poison, <1 x double> [[TMP47]], <1 x i32> <i32 1>
+; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[C]], align 32
+; CHECK-NEXT: [[VEC_GEP52:%.*]] = getelementptr inbounds double, ptr [[C]], i64 1
+; CHECK-NEXT: store <1 x double> [[TMP28]], ptr [[VEC_GEP52]], align 8
+; CHECK-NEXT: [[VEC_GEP53:%.*]] = getelementptr inbounds double, ptr [[C]], i64 2
+; CHECK-NEXT: store <1 x double> [[TMP38]], ptr [[VEC_GEP53]], align 16
+; CHECK-NEXT: [[VEC_GEP54:%.*]] = getelementptr inbounds double, ptr [[C]], i64 3
+; CHECK-NEXT: store <1 x double> [[TMP48]], ptr [[VEC_GEP54]], align 8
+; CHECK-NEXT: ret void
+;
+ %a = load <4 x double>, ptr %A
+ %b = load <16 x double>, ptr %B
+ %c = call <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v16f64(
+ <4 x double> %a, <16 x double> %b, i32 1, i32 4, i32 4)
+ store <4 x double> %c, ptr %C
+ ret void
+}
+
+declare <4 x double> @llvm.matrix.multiply.v4f64.v4f64.v16f64(<4 x double>, <16 x double>, i32 immarg, i32 immarg, i32 immarg)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
index fa07c51630305..8e337169859e0 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-double.ll
@@ -75,31 +75,30 @@ define <4 x double> @multiply_1x2(<2 x double> %a, <2 x double> %b) {
; CHECK-LABEL: @multiply_1x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <2 x double> [[B:%.*]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP0]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> poison, <2 x double> [[TMP2]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK3:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <1 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x double> poison, double [[TMP4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT4]], <1 x double> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = fmul <1 x double> [[BLOCK3]], [[SPLAT_SPLAT5]]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <1 x double> [[TMP5]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <1 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 1
; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> poison, double [[TMP8]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <1 x double> [[TMP9]], <1 x double> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> poison, <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <1 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT1]], i64 1
; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP12]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
index 5f22cd9e4679b..3ca74a307d276 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-float.ll
@@ -75,31 +75,30 @@ define <4 x float> @multiply_1x2(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: @multiply_1x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <2 x float> [[A:%.*]], <2 x float> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <2 x float> [[B:%.*]], <2 x float> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <2 x float> [[B]], <2 x float> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <2 x float> [[B:%.*]], <2 x float> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <1 x float> [[SPLIT1]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x float> [[SPLIT1]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x float> poison, float [[TMP0]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT]], <1 x float> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = fmul <1 x float> [[BLOCK]], [[SPLAT_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> poison, <2 x float> [[TMP2]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK3:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <1 x float> [[SPLIT1]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[SPLIT1]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x float> poison, float [[TMP4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT4]], <1 x float> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = fmul <1 x float> [[BLOCK3]], [[SPLAT_SPLAT5]]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <1 x float> [[TMP5]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP6]], <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <1 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x float> [[SPLIT1]], i64 1
; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x float> poison, float [[TMP8]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT7]], <1 x float> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = fmul <1 x float> [[BLOCK6]], [[SPLAT_SPLAT8]]
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <1 x float> [[TMP9]], <1 x float> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> poison, <2 x float> [[TMP10]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x float> [[SPLIT]], <2 x float> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <1 x float> [[SPLIT2]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x float> [[SPLIT1]], i64 1
; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x float> poison, float [[TMP12]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x float> [[SPLAT_SPLATINSERT10]], <1 x float> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = fmul <1 x float> [[BLOCK9]], [[SPLAT_SPLAT11]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
index 243cf84b64647..f15e90cdfc246 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-i32.ll
@@ -75,31 +75,30 @@ define <4 x i32> @multiply_1x2(<2 x i32> %a, <2 x i32> %b) {
; CHECK-LABEL: @multiply_1x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <2 x i32> [[A:%.*]], <2 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <2 x i32> [[B:%.*]], <2 x i32> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <2 x i32> [[B]], <2 x i32> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <2 x i32> [[B:%.*]], <2 x i32> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x i32> poison, i32 [[TMP0]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT]], <1 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = mul <1 x i32> [[BLOCK]], [[SPLAT_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> poison, <2 x i32> [[TMP2]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK3:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <1 x i32> [[SPLIT1]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT4:%.*]] = insertelement <1 x i32> poison, i32 [[TMP4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT5:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT4]], <1 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = mul <1 x i32> [[BLOCK3]], [[SPLAT_SPLAT5]]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <1 x i32> [[TMP5]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP6]], <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 1
; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x i32> poison, i32 [[TMP8]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT7]], <1 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP9:%.*]] = mul <1 x i32> [[BLOCK6]], [[SPLAT_SPLAT8]]
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <1 x i32> [[TMP9]], <1 x i32> poison, <2 x i32> <i32 0, i32 poison>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> poison, <2 x i32> [[TMP10]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x i32> [[SPLIT]], <2 x i32> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <1 x i32> [[SPLIT2]], i64 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[SPLIT1]], i64 1
; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x i32> poison, i32 [[TMP12]], i64 0
; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x i32> [[SPLAT_SPLATINSERT10]], <1 x i32> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = mul <1 x i32> [[BLOCK9]], [[SPLAT_SPLAT11]]
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll
index 1b3b41d8cfe1f..0d153d441de36 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts-lifting.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p lower-matrix-intrinsics -matrix-print-after-transpose-opt -disable-output -S %s 2>&1 | FileCheck %s
; REQUIRES: asserts
@@ -6,10 +7,11 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
; FIXME: Lifted transpose dimensions are incorrect.
define <6 x double> @lift_through_add_matching_transpose_dimensions(<6 x double> %a, <6 x double> %b) {
-; CHECK-LABEL: define <6 x double> @lift_through_add_matching_transpose_dimensions(<6 x double> %a, <6 x double> %b) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.+]] = fadd <6 x double> %a, %b
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[A]], i32 3, i32 2)
+; CHECK-LABEL: define <6 x double> @lift_through_add_matching_transpose_dimensions(
+; CHECK-SAME: <6 x double> [[A:%.*]], <6 x double> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MFADD:%.*]] = fadd <6 x double> [[A]], [[B]]
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MFADD]], i32 3, i32 2)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -20,12 +22,13 @@ entry:
}
define <6 x double> @lift_through_add_matching_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr) {
-; CHECK-LABEL: define <6 x double> @lift_through_add_matching_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr)
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.+]] = load <6 x double>, ptr %a.ptr
-; CHECK-NEXT: [[B:%.+]] = load <6 x double>, ptr %b.ptr
-; CHECK-NEXT: [[ADD:%.+]] = fadd <6 x double> [[A]], [[B]]
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 3, i32 2)
+; CHECK-LABEL: define <6 x double> @lift_through_add_matching_transpose_dimensions_ops_also_have_shape_info(
+; CHECK-SAME: ptr [[A_PTR:%.*]], ptr [[B_PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A:%.*]] = load <6 x double>, ptr [[A_PTR]], align 64
+; CHECK-NEXT: [[B:%.*]] = load <6 x double>, ptr [[B_PTR]], align 64
+; CHECK-NEXT: [[ADD:%.*]] = fadd <6 x double> [[A]], [[B]]
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 3, i32 2)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -38,10 +41,11 @@ entry:
}
define <6 x double> @lift_through_add_mismatching_dimensions_1(<6 x double> %a, <6 x double> %b) {
-; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_1(<6 x double> %a, <6 x double> %b) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.+]] = fadd <6 x double> %a, %b
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[A]], i32 1, i32 6)
+; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_1(
+; CHECK-SAME: <6 x double> [[A:%.*]], <6 x double> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MFADD:%.*]] = fadd <6 x double> [[A]], [[B]]
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MFADD]], i32 1, i32 6)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -52,12 +56,13 @@ entry:
}
define <6 x double> @lift_through_add_mismatching_dimensions_1_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr) {
-; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_1_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr)
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.+]] = load <6 x double>, ptr %a.ptr
-; CHECK-NEXT: [[B:%.+]] = load <6 x double>, ptr %b.ptr
-; CHECK-NEXT: [[ADD:%.+]] = fadd <6 x double> [[A]], [[B]]
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 1, i32 6)
+; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_1_transpose_dimensions_ops_also_have_shape_info(
+; CHECK-SAME: ptr [[A_PTR:%.*]], ptr [[B_PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A:%.*]] = load <6 x double>, ptr [[A_PTR]], align 64
+; CHECK-NEXT: [[B:%.*]] = load <6 x double>, ptr [[B_PTR]], align 64
+; CHECK-NEXT: [[ADD:%.*]] = fadd <6 x double> [[A]], [[B]]
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 1, i32 6)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -70,10 +75,11 @@ entry:
}
define <6 x double> @lift_through_add_mismatching_dimensions_2(<6 x double> %a, <6 x double> %b) {
-; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_2(<6 x double> %a, <6 x double> %b) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.+]] = fadd <6 x double> %a, %b
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[A]], i32 3, i32 2)
+; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_2(
+; CHECK-SAME: <6 x double> [[A:%.*]], <6 x double> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MFADD:%.*]] = fadd <6 x double> [[A]], [[B]]
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MFADD]], i32 3, i32 2)
; CHECK-NEXT: ret <6 x double> [[T]]
;
@@ -85,12 +91,13 @@ entry:
}
define <6 x double> @lift_through_add_mismatching_dimensions_2_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr) {
-; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_2_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr)
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.+]] = load <6 x double>, ptr %a.ptr
-; CHECK-NEXT: [[B:%.+]] = load <6 x double>, ptr %b.ptr
-; CHECK-NEXT: [[ADD:%.+]] = fadd <6 x double> [[A]], [[B]]
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 3, i32 2)
+; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_2_transpose_dimensions_ops_also_have_shape_info(
+; CHECK-SAME: ptr [[A_PTR:%.*]], ptr [[B_PTR:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A:%.*]] = load <6 x double>, ptr [[A_PTR]], align 64
+; CHECK-NEXT: [[B:%.*]] = load <6 x double>, ptr [[B_PTR]], align 64
+; CHECK-NEXT: [[ADD:%.*]] = fadd <6 x double> [[A]], [[B]]
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 3, i32 2)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -103,11 +110,12 @@ entry:
}
define <9 x double> @lift_through_multiply(<6 x double> %a, <6 x double> %b) {
-; CHECK-LABEL: define <9 x double> @lift_through_multiply(<6 x double> %a, <6 x double> %b) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MUL:%.+]] = call <9 x double> @llvm.matrix.multiply.v9f64.v6f64.v6f64(<6 x double> %b, <6 x double> %a, i32 3, i32 2, i32 3)
-; CHECK-NEXT: [[T:%.+]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[MUL]], i32 3, i32 3)
-; CHECK-NEXT: ret <9 x double> [[T]]
+; CHECK-LABEL: define <9 x double> @lift_through_multiply(
+; CHECK-SAME: <6 x double> [[A:%.*]], <6 x double> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v6f64.v6f64(<6 x double> [[B]], <6 x double> [[A]], i32 3, i32 2, i32 3)
+; CHECK-NEXT: [[T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[MUL]], i32 3, i32 3)
+; CHECK-NEXT: ret <9 x double> [[T]]
;
entry:
%a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)
@@ -117,10 +125,11 @@ entry:
}
define <6 x double> @lift_through_multiply_2(<6 x double> %a, <4 x double> %b) {
-; CHECK-LABEL: define <6 x double> @lift_through_multiply_2(<6 x double> %a, <4 x double> %b) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MUL:%.+]] = call <6 x double> @llvm.matrix.multiply.v6f64.v4f64.v6f64(<4 x double> %b, <6 x double> %a, i32 2, i32 2, i32 3)
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MUL]], i32 2, i32 3)
+; CHECK-LABEL: define <6 x double> @lift_through_multiply_2(
+; CHECK-SAME: <6 x double> [[A:%.*]], <4 x double> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL:%.*]] = call <6 x double> @llvm.matrix.multiply.v6f64.v4f64.v6f64(<4 x double> [[B]], <6 x double> [[A]], i32 2, i32 2, i32 3)
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MUL]], i32 2, i32 3)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -131,10 +140,11 @@ entry:
}
define <6 x double> @lift_through_multiply_3(<4 x double> %a, <6 x double> %b) {
-; CHECK-LABEL: define <6 x double> @lift_through_multiply_3(<4 x double> %a, <6 x double> %b) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MUL:%.+]] = call <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v4f64(<6 x double> %b, <4 x double> %a, i32 3, i32 2, i32 2)
-; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MUL]], i32 3, i32 2)
+; CHECK-LABEL: define <6 x double> @lift_through_multiply_3(
+; CHECK-SAME: <4 x double> [[A:%.*]], <6 x double> [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL:%.*]] = call <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v4f64(<6 x double> [[B]], <4 x double> [[A]], i32 3, i32 2, i32 2)
+; CHECK-NEXT: [[T:%.*]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MUL]], i32 3, i32 2)
; CHECK-NEXT: ret <6 x double> [[T]]
;
entry:
@@ -148,9 +158,10 @@ define void @test_remove_entries_from_shape_map(<3 x float> %a, <2 x float> %b,
; CHECK-LABEL: define void @test_remove_entries_from_shape_map(
; CHECK-SAME: <3 x float> [[A:%.*]], <2 x float> [[B:%.*]], <6 x float> [[C:%.*]], ptr [[DST:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = call <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float> [[A]], <2 x float> [[B]], i32 3, i32 1, i32 2)
-; CHECK-NEXT: [[MFADD:%.*]] = fadd <6 x float> [[C]], [[TMP0]]
-; CHECK-NEXT: [[MFADD_T:%.*]] = call <6 x float> @llvm.matrix.transpose.v6f32(<6 x float> [[MFADD]], i32 3, i32 2)
+; CHECK-NEXT: [[C_T:%.*]] = call <6 x float> @llvm.matrix.transpose.v6f32(<6 x float> [[C]], i32 3, i32 2)
+; CHECK-NEXT: [[A_T:%.*]] = call <3 x float> @llvm.matrix.transpose.v3f32(<3 x float> [[A]], i32 3, i32 1)
+; CHECK-NEXT: [[MMUL:%.*]] = call <6 x float> @llvm.matrix.multiply.v6f32.v2f32.v3f32(<2 x float> [[B]], <3 x float> [[A_T]], i32 2, i32 1, i32 3)
+; CHECK-NEXT: [[MFADD_T:%.*]] = fadd <6 x float> [[C_T]], [[MMUL]]
; CHECK-NEXT: store <6 x float> [[MFADD_T]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
More information about the llvm-branch-commits
mailing list