[llvm] [Matrix] De-duplicate reshaped matrixes used as incoming values for phi. (PR #211210)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 22 11:57:29 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/211210
>From a2360f9cd5a83c2ea99079ff6c71cf43f6697e3d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 21 Jul 2026 20:41:50 +0100
Subject: [PATCH] [Matrix] De-duplicate reshaped matrixes used as incoming
values for phi.
Phis can have multiple incoming entries for the same block. In that
case, all incoming values for the block must be the same.
Update visitPHI to avoid expanding the incoming matrix multiple times
for the some incoming block.
Fixes a verifier error for the newly added test case.
---
.../Scalar/LowerMatrixIntrinsics.cpp | 8 +-
.../Transforms/LowerMatrixIntrinsics/phi.ll | 86 +++++++++++++------
2 files changed, 67 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index fc5313b2d868f..1e363437348f9 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -2397,6 +2397,9 @@ class LowerMatrixIntrinsics {
Builder.SetInsertPoint(BlockIP);
MatrixTy PhiM = getMatrix(Inst, SI, Builder);
+ // Cache the reshaped columns per incoming block, so that a block listed
+ // more than once contributes identical incoming values to the new PHIs.
+ SmallDenseMap<BasicBlock *, MatrixTy> ReshapedIncoming;
for (auto [IncomingV, IncomingB] :
llvm::zip_equal(Inst->incoming_values(), Inst->blocks())) {
// getMatrix() may insert some instructions to help with reshaping. The
@@ -2407,7 +2410,10 @@ class LowerMatrixIntrinsics {
if (auto MaybeIP = IncomingInst->getInsertionPointAfterDef())
Builder.SetInsertPoint(*MaybeIP);
- MatrixTy OpM = getMatrix(IncomingV, SI, Builder);
+ auto [It, Inserted] = ReshapedIncoming.try_emplace(IncomingB);
+ if (Inserted)
+ It->second = getMatrix(IncomingV, SI, Builder);
+ const MatrixTy &OpM = It->second;
for (unsigned VI = 0, VE = PhiM.getNumVectors(); VI != VE; ++VI) {
PHINode *NewPHI = cast<PHINode>(PhiM.getVector(VI));
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll
index a8e71666421b4..e5b1e80508b94 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll
@@ -335,19 +335,19 @@ define void @matrix_phi_loop_delay_reshape(ptr %in1, ptr %in2, ptr %in3, i32 %co
; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: [[VEC_GEP1:%.*]] = getelementptr inbounds double, ptr [[IN3]], i64 4
; CHECK-NEXT: [[COL_LOAD12:%.*]] = load <2 x double>, ptr [[VEC_GEP1]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> [[COL_LOAD8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> [[TMP1]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[TMP2]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT: [[COL_LOAD10:%.*]] = shufflevector <6 x double> [[TMP2]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> [[COL_LOAD8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP10]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT: [[SPLIT16:%.*]] = shufflevector <6 x double> [[TMP11]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[SPLIT17:%.*]] = shufflevector <6 x double> [[TMP11]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <6 x double>, ptr [[IN2:%.*]], align 8
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN1:%.*]], align 8
; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr inbounds double, ptr [[IN1]], i64 3
; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
-; CHECK-NEXT: [[PHI2:%.*]] = phi <3 x double> [ [[SPLIT]], [[ENTRY:%.*]] ], [ [[PHI1:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[PHI39:%.*]] = phi <3 x double> [ [[COL_LOAD10]], [[ENTRY]] ], [ [[PHI4:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[PHI2:%.*]] = phi <3 x double> [ [[SPLIT16]], [[ENTRY:%.*]] ], [ [[PHI1:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[PHI39:%.*]] = phi <3 x double> [ [[SPLIT17]], [[ENTRY]] ], [ [[PHI4:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[PHI25:%.*]] = phi <6 x double> [ [[COL_LOAD11]], [[ENTRY]] ], [ [[PHI25]], [[LOOP]] ]
; CHECK-NEXT: [[PHI1]] = phi <3 x double> [ [[COL_LOAD]], [[ENTRY]] ], [ [[PHI2]], [[LOOP]] ]
; CHECK-NEXT: [[PHI4]] = phi <3 x double> [ [[COL_LOAD14]], [[ENTRY]] ], [ [[PHI39]], [[LOOP]] ]
@@ -463,16 +463,16 @@ define void @matrix_phi_two_preds_shape_mismatch1(i1 %cond1, ptr %a, ptr %b, ptr
; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr inbounds double, ptr [[A]], i64 6
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[VEC_GEP3]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x double> [[COL_LOAD1]], <3 x double> [[COL_LOAD2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD4]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <6 x double> [[TMP0]], <6 x double> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[TMP2]], <9 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <3 x double> [[COL_LOAD1]], <3 x double> [[COL_LOAD2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <3 x double> [[COL_LOAD4]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> [[TMP4]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: bbb:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x double>, ptr [[B:%.*]], align 8
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
-; CHECK-NEXT: [[PHI5:%.*]] = phi <9 x double> [ [[SPLIT]], [[BBA]] ], [ [[COL_LOAD]], [[BBB]] ]
+; CHECK-NEXT: [[PHI5:%.*]] = phi <9 x double> [ [[SPLIT6]], [[BBA]] ], [ [[COL_LOAD]], [[BBB]] ]
; CHECK-NEXT: store <9 x double> [[PHI5]], ptr [[OUT:%.*]], align 128
; CHECK-NEXT: ret void
;
@@ -499,9 +499,9 @@ define void @matrix_phi_two_preds_shape_mismatch2(i1 %cond1, ptr %a, ptr %b, ptr
; CHECK-NEXT: br i1 [[COND1:%.*]], label [[BBA:%.*]], label [[BBB:%.*]]
; CHECK: bba:
; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <9 x double>, ptr [[A:%.*]], align 8
-; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT: [[SPLIT8:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[SPLIT9:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT: [[SPLIT10:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[SPLIT11:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT: [[SPLIT12:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: bbb:
; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[B:%.*]], align 8
@@ -511,9 +511,9 @@ define void @matrix_phi_two_preds_shape_mismatch2(i1 %cond1, ptr %a, ptr %b, ptr
; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
-; CHECK-NEXT: [[PHI5:%.*]] = phi <3 x double> [ [[SPLIT]], [[BBA]] ], [ [[COL_LOAD]], [[BBB]] ]
-; CHECK-NEXT: [[PHI6:%.*]] = phi <3 x double> [ [[SPLIT8]], [[BBA]] ], [ [[COL_LOAD1]], [[BBB]] ]
-; CHECK-NEXT: [[PHI7:%.*]] = phi <3 x double> [ [[SPLIT9]], [[BBA]] ], [ [[COL_LOAD3]], [[BBB]] ]
+; CHECK-NEXT: [[PHI5:%.*]] = phi <3 x double> [ [[SPLIT10]], [[BBA]] ], [ [[COL_LOAD]], [[BBB]] ]
+; CHECK-NEXT: [[PHI6:%.*]] = phi <3 x double> [ [[SPLIT11]], [[BBA]] ], [ [[COL_LOAD1]], [[BBB]] ]
+; CHECK-NEXT: [[PHI7:%.*]] = phi <3 x double> [ [[SPLIT12]], [[BBA]] ], [ [[COL_LOAD3]], [[BBB]] ]
; CHECK-NEXT: store <3 x double> [[PHI5]], ptr [[OUT:%.*]], align 128
; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 3
; CHECK-NEXT: store <3 x double> [[PHI6]], ptr [[VEC_GEP10]], align 8
@@ -791,14 +791,14 @@ if.end: ; preds = %if.then, %if.else
define <4 x float> @matrix_phi_argument_incoming(<4 x float> %arg, i1 %cond) {
; CHECK-LABEL: @matrix_phi_argument_incoming(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <4 x float> [[ARG:%.*]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <4 x float> [[ARG]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <4 x float> [[ARG:%.*]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <4 x float> [[ARG]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br i1 [[COND:%.*]], label [[EXIT:%.*]], label [[BB:%.*]]
; CHECK: bb:
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
-; CHECK-NEXT: [[PHI1:%.*]] = phi <2 x float> [ zeroinitializer, [[BB]] ], [ [[SPLIT]], [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[PHI2:%.*]] = phi <2 x float> [ zeroinitializer, [[BB]] ], [ [[SPLIT3]], [[ENTRY]] ]
+; CHECK-NEXT: [[PHI1:%.*]] = phi <2 x float> [ zeroinitializer, [[BB]] ], [ [[SPLIT4]], [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[PHI2:%.*]] = phi <2 x float> [ zeroinitializer, [[BB]] ], [ [[SPLIT5]], [[ENTRY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x float> [[PHI1]], <2 x float> [[PHI2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <4 x float> [[TMP0]]
;
@@ -825,8 +825,8 @@ define <4 x float> @matrix_phi_invoke_incoming(i1 %cond) personality ptr @__gxx_
; CHECK-NEXT: [[INV:%.*]] = invoke <4 x float> @get_matrix()
; CHECK-NEXT: to label [[CONT:%.*]] unwind label [[LPAD:%.*]]
; CHECK: cont:
-; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <4 x float> [[INV]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <4 x float> [[INV]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <4 x float> [[INV]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <4 x float> [[INV]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br label [[EXIT:%.*]]
; CHECK: lpad:
; CHECK-NEXT: [[L:%.*]] = landingpad { ptr, i32 }
@@ -835,8 +835,8 @@ define <4 x float> @matrix_phi_invoke_incoming(i1 %cond) personality ptr @__gxx_
; CHECK: bb:
; CHECK-NEXT: br label [[EXIT]]
; CHECK: exit:
-; CHECK-NEXT: [[PHI1:%.*]] = phi <2 x float> [ [[SPLIT]], [[CONT]] ], [ zeroinitializer, [[BB]] ]
-; CHECK-NEXT: [[PHI2:%.*]] = phi <2 x float> [ [[SPLIT3]], [[CONT]] ], [ zeroinitializer, [[BB]] ]
+; CHECK-NEXT: [[PHI1:%.*]] = phi <2 x float> [ [[SPLIT4]], [[CONT]] ], [ zeroinitializer, [[BB]] ]
+; CHECK-NEXT: [[PHI2:%.*]] = phi <2 x float> [ [[SPLIT5]], [[CONT]] ], [ zeroinitializer, [[BB]] ]
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x float> [[PHI1]], <2 x float> [[PHI2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: ret <4 x float> [[TMP0]]
;
@@ -861,3 +861,37 @@ exit:
%phi = phi <4 x float> [ %inv, %cont ], [ %t, %bb ]
ret <4 x float> %phi
}
+
+define <4 x float> @matrix_phi_duplicate_predecessor(ptr %arg, i32 %sw) {
+; CHECK-LABEL: @matrix_phi_duplicate_predecessor(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SPLIT:%.*]] = load <2 x float>, ptr [[ARG:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr inbounds float, ptr [[ARG]], i64 2
+; CHECK-NEXT: [[SPLIT3:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: switch i32 [[SW:%.*]], label [[BB:%.*]] [
+; CHECK-NEXT: i32 0, label [[EXIT:%.*]]
+; CHECK-NEXT: i32 1, label [[EXIT]]
+; CHECK-NEXT: ]
+; CHECK: bb:
+; CHECK-NEXT: br label [[EXIT]]
+; CHECK: exit:
+; CHECK-NEXT: [[PHI1:%.*]] = phi <2 x float> [ [[SPLIT]], [[ENTRY:%.*]] ], [ [[SPLIT]], [[ENTRY]] ], [ zeroinitializer, [[BB]] ]
+; CHECK-NEXT: [[PHI2:%.*]] = phi <2 x float> [ [[SPLIT3]], [[ENTRY]] ], [ [[SPLIT3]], [[ENTRY]] ], [ zeroinitializer, [[BB]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x float> [[PHI1]], <2 x float> [[PHI2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <4 x float> [[TMP0]]
+;
+entry:
+ %m = load <4 x float>, ptr %arg
+ switch i32 %sw, label %bb [
+ i32 0, label %exit
+ i32 1, label %exit
+ ]
+
+bb:
+ %t = call <4 x float> @llvm.matrix.transpose.v4f32(<4 x float> zeroinitializer, i32 2, i32 2)
+ br label %exit
+
+exit:
+ %phi = phi <4 x float> [ %m, %entry ], [ %m, %entry ], [ %t, %bb ]
+ ret <4 x float> %phi
+}
More information about the llvm-commits
mailing list