[llvm] 256d5ad - LowerMatrixIntrinsics: Convert tests to opaque pointers
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Sun Nov 27 18:56:32 PST 2022
Author: Matt Arsenault
Date: 2022-11-27T21:42:25-05:00
New Revision: 256d5ad3e8db42e125e4954fadeba2b37421d91f
URL: https://github.com/llvm/llvm-project/commit/256d5ad3e8db42e125e4954fadeba2b37421d91f
DIFF: https://github.com/llvm/llvm-project/commit/256d5ad3e8db42e125e4954fadeba2b37421d91f.diff
LOG: LowerMatrixIntrinsics: Convert tests to opaque pointers
store-align-volatile.ll needed manually updated check lines for a
-NEXT check after a deleted bitcast.
Also avoided breaking the example C++ comment in remarks-inlining.ll
Added:
Modified:
llvm/test/Transforms/LowerMatrixIntrinsics/after-transpose-opts.ll
llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll
llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll
llvm/test/Transforms/LowerMatrixIntrinsics/store-align-volatile.ll
llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
Removed:
################################################################################
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/after-transpose-opts.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/after-transpose-opts.ll
index 4f0b119b3145d..9f1670360191e 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/after-transpose-opts.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/after-transpose-opts.ll
@@ -7,156 +7,156 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "aarch64-apple-ios"
; k * A^T
-define void @kat(<9 x double>* %Aptr, double %k, <9 x double>* %C) {
+define void @kat(ptr %Aptr, double %k, ptr %C) {
; CHECK-LABEL: @kat(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer
; CHECK-NEXT: [[AT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)
; CHECK-NEXT: [[MUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[SPLAT]], <9 x double> [[AT]], i32 3, i32 3, i32 3)
-; CHECK-NEXT: store <9 x double> [[MUL]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[MUL]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
+ %a = load <9 x double>, ptr %Aptr
%veck = insertelement <9 x double> poison, double %k, i64 0
%splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%mul = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %at, i32 3, i32 3, i32 3)
- store <9 x double> %mul, <9 x double>* %C
+ store <9 x double> %mul, ptr %C
ret void
}
; (k * A)^T -> A^T * k
-define void @ka_t(<9 x double>* %Aptr, double %k, <9 x double>* %C) {
+define void @ka_t(ptr %Aptr, double %k, ptr %C) {
; CHECK-LABEL: @ka_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer
; CHECK-NEXT: [[A_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)
; CHECK-NEXT: [[MMUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[A_T]], <9 x double> [[SPLAT]], i32 3, i32 3, i32 3)
-; CHECK-NEXT: store <9 x double> [[MMUL]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[MMUL]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
+ %a = load <9 x double>, ptr %Aptr
%veck = insertelement <9 x double> poison, double %k, i64 0
%splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer
%mul = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %a, i32 3, i32 3, i32 3)
%t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %mul, i32 3, i32 3)
- store <9 x double> %t, <9 x double>* %C
+ store <9 x double> %t, ptr %C
ret void
}
; (k * A)^T -> A^T * k with fmul
-define void @ka_t_fmul(<9 x double>* %Aptr, double %k, <9 x double>* %C) {
+define void @ka_t_fmul(ptr %Aptr, double %k, ptr %C) {
; CHECK-LABEL: @ka_t_fmul(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer
; CHECK-NEXT: [[A_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)
; CHECK-NEXT: [[MMUL:%.*]] = fmul <9 x double> [[SPLAT]], [[A_T]]
-; CHECK-NEXT: store <9 x double> [[MMUL]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[MMUL]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
+ %a = load <9 x double>, ptr %Aptr
%veck = insertelement <9 x double> poison, double %k, i64 0
%splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer
%mul = fmul <9 x double> %splat, %a
%t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %mul, i32 3, i32 3)
- store <9 x double> %t, <9 x double>* %C
+ store <9 x double> %t, ptr %C
ret void
}
; (k * A)^T -> A^T * k with mul (non-fp types)
-define void @ka_t_mul(<9 x i32>* %Aptr, i32 %k, <9 x i32>* %C) {
+define void @ka_t_mul(ptr %Aptr, i32 %k, ptr %C) {
; CHECK-LABEL: @ka_t_mul(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x i32>, <9 x i32>* [[APTR:%.*]], align 64
+; CHECK-NEXT: [[A:%.*]] = load <9 x i32>, ptr [[APTR:%.*]], align 64
; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x i32> poison, i32 [[K:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x i32> [[VECK]], <9 x i32> poison, <9 x i32> zeroinitializer
; CHECK-NEXT: [[A_T:%.*]] = call <9 x i32> @llvm.matrix.transpose.v9i32(<9 x i32> [[A]], i32 3, i32 3)
; CHECK-NEXT: [[MMUL:%.*]] = mul <9 x i32> [[SPLAT]], [[A_T]]
-; CHECK-NEXT: store <9 x i32> [[MMUL]], <9 x i32>* [[C:%.*]], align 64
+; CHECK-NEXT: store <9 x i32> [[MMUL]], ptr [[C:%.*]], align 64
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x i32>, <9 x i32>* %Aptr
+ %a = load <9 x i32>, ptr %Aptr
%veck = insertelement <9 x i32> poison, i32 %k, i64 0
%splat = shufflevector <9 x i32> %veck, <9 x i32> poison, <9 x i32> zeroinitializer
%mul = mul <9 x i32> %splat, %a
%t = call <9 x i32> @llvm.matrix.transpose.v9i32.v9i32(<9 x i32> %mul, i32 3, i32 3)
- store <9 x i32> %t, <9 x i32>* %C
+ store <9 x i32> %t, ptr %C
ret void
}
; A^T + B^T -> (A + B)^T
-define void @at_plus_bt(<9 x double>* %Aptr, <9 x double>* %Bptr, <9 x double>* %C) {
+define void @at_plus_bt(ptr %Aptr, ptr %Bptr, ptr %C) {
; CHECK-LABEL: @at_plus_bt(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
-; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[BPTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
+; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128
; CHECK-NEXT: [[AT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)
; CHECK-NEXT: [[BT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[B]], i32 3, i32 3)
; CHECK-NEXT: [[FADD:%.*]] = fadd <9 x double> [[AT]], [[BT]]
-; CHECK-NEXT: store <9 x double> [[FADD]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[FADD]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
- %b = load <9 x double>, <9 x double>* %Bptr
+ %a = load <9 x double>, ptr %Aptr
+ %b = load <9 x double>, ptr %Bptr
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
%fadd = fadd <9 x double> %at, %bt
- store <9 x double> %fadd, <9 x double>* %C
+ store <9 x double> %fadd, ptr %C
ret void
}
; (A + B)^T -> A^T + B^T -> (A + B)^T
-define void @a_plus_b_t(<9 x double>* %Aptr, <9 x double>* %Bptr, <9 x double>* %C) {
+define void @a_plus_b_t(ptr %Aptr, ptr %Bptr, ptr %C) {
; CHECK-LABEL: @a_plus_b_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
-; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[BPTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
+; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128
; CHECK-NEXT: [[FADD:%.*]] = fadd <9 x double> [[A]], [[B]]
; CHECK-NEXT: [[T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[FADD]], i32 3, i32 3)
-; CHECK-NEXT: store <9 x double> [[T]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[T]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
- %b = load <9 x double>, <9 x double>* %Bptr
+ %a = load <9 x double>, ptr %Aptr
+ %b = load <9 x double>, ptr %Bptr
%fadd = fadd <9 x double> %a, %b
%t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %fadd, i32 3, i32 3)
- store <9 x double> %t, <9 x double>* %C
+ store <9 x double> %t, ptr %C
ret void
}
; A^T * B^T + C^T * D^T -> (B * A + D * C)^T
-define void @atbt_plus_ctdt(<9 x double>* %Aptr, <9 x double>* %Bptr, <9 x double>* %Cptr, <9 x double>* %Dptr, <9 x double>* %E) {
+define void @atbt_plus_ctdt(ptr %Aptr, ptr %Bptr, ptr %Cptr, ptr %Dptr, ptr %E) {
; CHECK-LABEL: @atbt_plus_ctdt(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
-; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[BPTR:%.*]], align 128
-; CHECK-NEXT: [[C:%.*]] = load <9 x double>, <9 x double>* [[CPTR:%.*]], align 128
-; CHECK-NEXT: [[D:%.*]] = load <9 x double>, <9 x double>* [[DPTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
+; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128
+; CHECK-NEXT: [[C:%.*]] = load <9 x double>, ptr [[CPTR:%.*]], align 128
+; CHECK-NEXT: [[D:%.*]] = load <9 x double>, ptr [[DPTR:%.*]], align 128
; CHECK-NEXT: [[TMP0:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[B]], <9 x double> [[A]], i32 3, i32 3, i32 3)
; CHECK-NEXT: [[TMP1:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[TMP0]], i32 3, i32 3)
; CHECK-NEXT: [[TMP2:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[D]], <9 x double> [[C]], i32 3, i32 3, i32 3)
; CHECK-NEXT: [[TMP3:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[TMP2]], i32 3, i32 3)
; CHECK-NEXT: [[FADD:%.*]] = fadd <9 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: store <9 x double> [[FADD]], <9 x double>* [[E:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[FADD]], ptr [[E:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
- %b = load <9 x double>, <9 x double>* %Bptr
- %c = load <9 x double>, <9 x double>* %Cptr
- %d = load <9 x double>, <9 x double>* %Dptr
+ %a = load <9 x double>, ptr %Aptr
+ %b = load <9 x double>, ptr %Bptr
+ %c = load <9 x double>, ptr %Cptr
+ %d = load <9 x double>, ptr %Dptr
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
%ct = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %c, i32 3, i32 3)
@@ -164,42 +164,42 @@ entry:
%atbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %at, <9 x double> %bt, i32 3, i32 3, i32 3)
%ctdt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %ct, <9 x double> %dt, i32 3, i32 3, i32 3)
%fadd = fadd <9 x double> %atbt, %ctdt
- store <9 x double> %fadd, <9 x double>* %E
+ store <9 x double> %fadd, ptr %E
ret void
}
; -(A^T) + B^T
-define void @negat_plus_bt(<9 x double>* %Aptr, <9 x double>* %Bptr, <9 x double>* %C) {
+define void @negat_plus_bt(ptr %Aptr, ptr %Bptr, ptr %C) {
; CHECK-LABEL: @negat_plus_bt(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
-; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[BPTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
+; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128
; CHECK-NEXT: [[AT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)
; CHECK-NEXT: [[NEGAT:%.*]] = fneg <9 x double> [[AT]]
; CHECK-NEXT: [[BT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[B]], i32 3, i32 3)
; CHECK-NEXT: [[FADD:%.*]] = fadd <9 x double> [[NEGAT]], [[BT]]
-; CHECK-NEXT: store <9 x double> [[FADD]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[FADD]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
- %b = load <9 x double>, <9 x double>* %Bptr
+ %a = load <9 x double>, ptr %Aptr
+ %b = load <9 x double>, ptr %Bptr
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%negat = fneg <9 x double> %at
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
%fadd = fadd <9 x double> %negat, %bt
- store <9 x double> %fadd, <9 x double>* %C
+ store <9 x double> %fadd, ptr %C
ret void
}
; (A^T * B^T + k * C^T * D^T)^T -> (B * A) + (D * C * k)
-define void @atbt_plus_kctdt_t(<9 x double>* %Aptr, <9 x double>* %Bptr, <9 x double>* %Cptr, <9 x double>* %Dptr, double %k, <9 x double>* %E) {
+define void @atbt_plus_kctdt_t(ptr %Aptr, ptr %Bptr, ptr %Cptr, ptr %Dptr, double %k, ptr %E) {
; CHECK-LABEL: @atbt_plus_kctdt_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
-; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[BPTR:%.*]], align 128
-; CHECK-NEXT: [[C:%.*]] = load <9 x double>, <9 x double>* [[CPTR:%.*]], align 128
-; CHECK-NEXT: [[D:%.*]] = load <9 x double>, <9 x double>* [[DPTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
+; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128
+; CHECK-NEXT: [[C:%.*]] = load <9 x double>, ptr [[CPTR:%.*]], align 128
+; CHECK-NEXT: [[D:%.*]] = load <9 x double>, ptr [[DPTR:%.*]], align 128
; CHECK-NEXT: [[CT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[C]], i32 3, i32 3)
; CHECK-NEXT: [[DT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[D]], i32 3, i32 3)
; CHECK-NEXT: [[TMP0:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[B]], <9 x double> [[A]], i32 3, i32 3, i32 3)
@@ -210,14 +210,14 @@ define void @atbt_plus_kctdt_t(<9 x double>* %Aptr, <9 x double>* %Bptr, <9 x do
; CHECK-NEXT: [[KCTDT:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[KCT]], <9 x double> [[DT]], i32 3, i32 3, i32 3)
; CHECK-NEXT: [[FADD:%.*]] = fadd <9 x double> [[TMP1]], [[KCTDT]]
; CHECK-NEXT: [[T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[FADD]], i32 3, i32 3)
-; CHECK-NEXT: store <9 x double> [[T]], <9 x double>* [[E:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[T]], ptr [[E:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
- %b = load <9 x double>, <9 x double>* %Bptr
- %c = load <9 x double>, <9 x double>* %Cptr
- %d = load <9 x double>, <9 x double>* %Dptr
+ %a = load <9 x double>, ptr %Aptr
+ %b = load <9 x double>, ptr %Bptr
+ %c = load <9 x double>, ptr %Cptr
+ %d = load <9 x double>, ptr %Dptr
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
%ct = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %c, i32 3, i32 3)
@@ -229,26 +229,26 @@ entry:
%kctdt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %kct, <9 x double> %dt, i32 3, i32 3, i32 3)
%fadd = fadd <9 x double> %atbt, %kctdt
%t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %fadd, i32 3, i32 3)
- store <9 x double> %t, <9 x double>* %E
+ store <9 x double> %t, ptr %E
ret void
}
; (A^T * (k * B^T))^T => (B * k) * A
-define void @atkbt_t(<9 x double>* %Aptr, <9 x double>* %Bptr, double %k, <9 x double>* %C) {
+define void @atkbt_t(ptr %Aptr, ptr %Bptr, double %k, ptr %C) {
; CHECK-LABEL: @atkbt_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = load <9 x double>, <9 x double>* [[APTR:%.*]], align 128
-; CHECK-NEXT: [[B:%.*]] = load <9 x double>, <9 x double>* [[BPTR:%.*]], align 128
+; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128
+; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128
; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer
; CHECK-NEXT: [[MMUL1:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[B]], <9 x double> [[SPLAT]], i32 3, i32 3, i32 3)
; CHECK-NEXT: [[MMUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[MMUL1]], <9 x double> [[A]], i32 3, i32 3, i32 3)
-; CHECK-NEXT: store <9 x double> [[MMUL]], <9 x double>* [[C:%.*]], align 128
+; CHECK-NEXT: store <9 x double> [[MMUL]], ptr [[C:%.*]], align 128
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %Aptr
- %b = load <9 x double>, <9 x double>* %Bptr
+ %a = load <9 x double>, ptr %Aptr
+ %b = load <9 x double>, ptr %Bptr
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
%veck = insertelement <9 x double> poison, double %k, i64 0
@@ -256,7 +256,7 @@ entry:
%kbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %bt, i32 3, i32 3, i32 3)
%atkbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %at, <9 x double> %kbt, i32 3, i32 3, i32 3)
%t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %atkbt, i32 3, i32 3)
- store <9 x double> %t, <9 x double>* %C
+ store <9 x double> %t, ptr %C
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
index cf0731edc157a..66e3f5b34705b 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll
@@ -1,225 +1,213 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define void @transpose_multiply(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
+define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
; CHECK-LABEL: @transpose_multiply(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST6]], align 8
-; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i64 3
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST9]], align 8
-; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i64 6
-; CHECK-NEXT: [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST12]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> poison, double [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> [[TMP5]], double [[TMP6]], i64 2
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> poison, double [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> [[TMP11]], double [[TMP12]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> poison, double [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 2
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <3 x double> [[TMP17]], double [[TMP18]], i64 2
-; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 3
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[B_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, ptr [[B_PTR]], i64 3
+; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> poison, double [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> poison, double [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP18]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[BLOCK14:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x double> poison, double [[TMP22]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
-; CHECK-NEXT: [[TMP24:%.*]] = fadd <1 x double> [[TMP21]], [[TMP23]]
-; CHECK-NEXT: [[BLOCK17:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP26:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
-; CHECK-NEXT: [[TMP27:%.*]] = fadd <1 x double> [[TMP24]], [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <1 x double> [[TMP27]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP28]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT: [[BLOCK20:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
-; CHECK-NEXT: [[BLOCK23:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x double> poison, double [[TMP32]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP33:%.*]] = fmul <1 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
-; CHECK-NEXT: [[TMP34:%.*]] = fadd <1 x double> [[TMP31]], [[TMP33]]
-; CHECK-NEXT: [[BLOCK26:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT27]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP36:%.*]] = fmul <1 x double> [[BLOCK26]], [[SPLAT_SPLAT28]]
-; CHECK-NEXT: [[TMP37:%.*]] = fadd <1 x double> [[TMP34]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <1 x double> [[TMP37]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> [[TMP38]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT: [[BLOCK29:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
-; CHECK-NEXT: [[BLOCK32:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT33]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP43:%.*]] = fmul <1 x double> [[BLOCK32]], [[SPLAT_SPLAT34]]
-; CHECK-NEXT: [[TMP44:%.*]] = fadd <1 x double> [[TMP41]], [[TMP43]]
-; CHECK-NEXT: [[BLOCK35:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x double> poison, double [[TMP45]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP46:%.*]] = fmul <1 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
-; CHECK-NEXT: [[TMP47:%.*]] = fadd <1 x double> [[TMP44]], [[TMP46]]
-; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <1 x double> [[TMP47]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP49:%.*]] = shufflevector <3 x double> [[TMP39]], <3 x double> [[TMP48]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[BLOCK38:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT39]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK38]], [[SPLAT_SPLAT40]]
-; CHECK-NEXT: [[BLOCK41:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP52:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x double> poison, double [[TMP52]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP53:%.*]] = fmul <1 x double> [[BLOCK41]], [[SPLAT_SPLAT43]]
-; CHECK-NEXT: [[TMP54:%.*]] = fadd <1 x double> [[TMP51]], [[TMP53]]
-; CHECK-NEXT: [[BLOCK44:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP55:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x double> poison, double [[TMP55]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP56:%.*]] = fmul <1 x double> [[BLOCK44]], [[SPLAT_SPLAT46]]
-; CHECK-NEXT: [[TMP57:%.*]] = fadd <1 x double> [[TMP54]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <1 x double> [[TMP57]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP58]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT: [[BLOCK47:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP61:%.*]] = fmul <1 x double> [[BLOCK47]], [[SPLAT_SPLAT49]]
-; CHECK-NEXT: [[BLOCK50:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP62:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x double> poison, double [[TMP62]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT51]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP63:%.*]] = fmul <1 x double> [[BLOCK50]], [[SPLAT_SPLAT52]]
-; CHECK-NEXT: [[TMP64:%.*]] = fadd <1 x double> [[TMP61]], [[TMP63]]
-; CHECK-NEXT: [[BLOCK53:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP65:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x double> poison, double [[TMP65]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP66:%.*]] = fmul <1 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
-; CHECK-NEXT: [[TMP67:%.*]] = fadd <1 x double> [[TMP64]], [[TMP66]]
-; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <1 x double> [[TMP67]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP69:%.*]] = shufflevector <3 x double> [[TMP59]], <3 x double> [[TMP68]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT: [[BLOCK56:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT57:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT58:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT57]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP71:%.*]] = fmul <1 x double> [[BLOCK56]], [[SPLAT_SPLAT58]]
-; CHECK-NEXT: [[BLOCK59:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP72:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT60:%.*]] = insertelement <1 x double> poison, double [[TMP72]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT61:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT60]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP73:%.*]] = fmul <1 x double> [[BLOCK59]], [[SPLAT_SPLAT61]]
-; CHECK-NEXT: [[TMP74:%.*]] = fadd <1 x double> [[TMP71]], [[TMP73]]
-; CHECK-NEXT: [[BLOCK62:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP75:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT63:%.*]] = insertelement <1 x double> poison, double [[TMP75]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT64:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT63]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP76:%.*]] = fmul <1 x double> [[BLOCK62]], [[SPLAT_SPLAT64]]
-; CHECK-NEXT: [[TMP77:%.*]] = fadd <1 x double> [[TMP74]], [[TMP76]]
-; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <1 x double> [[TMP77]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP79:%.*]] = shufflevector <3 x double> [[TMP69]], <3 x double> [[TMP78]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[BLOCK65:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT66:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT67:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT66]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP81:%.*]] = fmul <1 x double> [[BLOCK65]], [[SPLAT_SPLAT67]]
-; CHECK-NEXT: [[BLOCK68:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT69:%.*]] = insertelement <1 x double> poison, double [[TMP82]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT70:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT69]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP83:%.*]] = fmul <1 x double> [[BLOCK68]], [[SPLAT_SPLAT70]]
-; CHECK-NEXT: [[TMP84:%.*]] = fadd <1 x double> [[TMP81]], [[TMP83]]
-; CHECK-NEXT: [[BLOCK71:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP85:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT72:%.*]] = insertelement <1 x double> poison, double [[TMP85]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT73:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT72]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP86:%.*]] = fmul <1 x double> [[BLOCK71]], [[SPLAT_SPLAT73]]
-; CHECK-NEXT: [[TMP87:%.*]] = fadd <1 x double> [[TMP84]], [[TMP86]]
-; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <1 x double> [[TMP87]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP88]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT: [[BLOCK74:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT75:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT76:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT75]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP91:%.*]] = fmul <1 x double> [[BLOCK74]], [[SPLAT_SPLAT76]]
-; CHECK-NEXT: [[BLOCK77:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP92:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT78:%.*]] = insertelement <1 x double> poison, double [[TMP92]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT79:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT78]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP93:%.*]] = fmul <1 x double> [[BLOCK77]], [[SPLAT_SPLAT79]]
-; CHECK-NEXT: [[TMP94:%.*]] = fadd <1 x double> [[TMP91]], [[TMP93]]
-; CHECK-NEXT: [[BLOCK80:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP95:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT81:%.*]] = insertelement <1 x double> poison, double [[TMP95]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT82:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT81]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP96:%.*]] = fmul <1 x double> [[BLOCK80]], [[SPLAT_SPLAT82]]
-; CHECK-NEXT: [[TMP97:%.*]] = fadd <1 x double> [[TMP94]], [[TMP96]]
-; CHECK-NEXT: [[TMP98:%.*]] = shufflevector <1 x double> [[TMP97]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP99:%.*]] = shufflevector <3 x double> [[TMP89]], <3 x double> [[TMP98]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT: [[BLOCK83:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT84:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT85:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT84]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP101:%.*]] = fmul <1 x double> [[BLOCK83]], [[SPLAT_SPLAT85]]
-; CHECK-NEXT: [[BLOCK86:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT87:%.*]] = insertelement <1 x double> poison, double [[TMP102]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT88:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT87]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP103:%.*]] = fmul <1 x double> [[BLOCK86]], [[SPLAT_SPLAT88]]
-; CHECK-NEXT: [[TMP104:%.*]] = fadd <1 x double> [[TMP101]], [[TMP103]]
-; CHECK-NEXT: [[BLOCK89:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP105:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT90:%.*]] = insertelement <1 x double> poison, double [[TMP105]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT91:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT90]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP106:%.*]] = fmul <1 x double> [[BLOCK89]], [[SPLAT_SPLAT91]]
-; CHECK-NEXT: [[TMP107:%.*]] = fadd <1 x double> [[TMP104]], [[TMP106]]
-; CHECK-NEXT: [[TMP108:%.*]] = shufflevector <1 x double> [[TMP107]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP109:%.*]] = shufflevector <3 x double> [[TMP99]], <3 x double> [[TMP108]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[TMP110:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST92:%.*]] = bitcast double* [[TMP110]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP49]], <3 x double>* [[VEC_CAST92]], align 8
-; CHECK-NEXT: [[VEC_GEP93:%.*]] = getelementptr double, double* [[TMP110]], i64 3
-; CHECK-NEXT: [[VEC_CAST94:%.*]] = bitcast double* [[VEC_GEP93]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP79]], <3 x double>* [[VEC_CAST94]], align 8
-; CHECK-NEXT: [[VEC_GEP95:%.*]] = getelementptr double, double* [[TMP110]], i64 6
-; CHECK-NEXT: [[VEC_CAST96:%.*]] = bitcast double* [[VEC_GEP95]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP109]], <3 x double>* [[VEC_CAST96]], align 8
+; CHECK-NEXT: [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT: [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
+; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP23]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT: [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP28]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT: [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
+; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP33]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT: [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP38]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT: [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
+; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP43]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
+; CHECK-NEXT: [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP48]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
+; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
+; CHECK-NEXT: [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP53:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> poison, double [[TMP53]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP54:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
+; CHECK-NEXT: [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
+; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT: [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> poison, double [[TMP58]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP59:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
+; CHECK-NEXT: [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP61:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT: [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]]
+; CHECK-NEXT: [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP63:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> poison, double [[TMP63]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP64:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT: [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
+; CHECK-NEXT: [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> poison, double [[TMP68]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP69:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
+; CHECK-NEXT: [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP71:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT: [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]]
+; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP73:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> poison, double [[TMP73]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP74:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
+; CHECK-NEXT: [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
+; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT: [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> poison, double [[TMP78]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP79:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT: [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP81:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
+; CHECK-NEXT: [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]]
+; CHECK-NEXT: [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP83:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> poison, double [[TMP83]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP84:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
+; CHECK-NEXT: [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
+; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT: [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> poison, double [[TMP88]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP89:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
+; CHECK-NEXT: [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP91:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
+; CHECK-NEXT: [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]]
+; CHECK-NEXT: [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP93:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> poison, double [[TMP93]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP94:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
+; CHECK-NEXT: [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
+; CHECK-NEXT: [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> poison, double [[TMP98]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP99:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
+; CHECK-NEXT: [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP101:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT: [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]]
+; CHECK-NEXT: [[BLOCK84:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP103:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> poison, double [[TMP103]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP104:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
+; CHECK-NEXT: [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
+; CHECK-NEXT: [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT: store <3 x double> [[TMP47]], ptr [[C_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP87:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
+; CHECK-NEXT: store <3 x double> [[TMP77]], ptr [[VEC_GEP87]], align 8
+; CHECK-NEXT: [[VEC_GEP88:%.*]] = getelementptr double, ptr [[C_PTR]], i64 6
+; CHECK-NEXT: store <3 x double> [[TMP107]], ptr [[VEC_GEP88]], align 8
; CHECK-NEXT: ret void
;
@@ -236,248 +224,232 @@ define void @transpose_multiply(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x
entry:
- %a = load <9 x double>, <9 x double>* %A.Ptr, align 8
- %b = load <9 x double>, <9 x double>* %B.Ptr, align 8
+ %a = load <9 x double>, ptr %A.Ptr, align 8
+ %b = load <9 x double>, ptr %B.Ptr, align 8
%a.trans = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
%c = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
- store <9 x double> %c, <9 x double>* %C.Ptr, align 8
+ store <9 x double> %c, ptr %C.Ptr, align 8
ret void
}
declare <9 x double> @llvm.matrix.transpose(<9 x double>, i32, i32)
declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32, i32, i32)
-define void @transpose_multiply_add(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr, <9 x double>* %C.Ptr) {
+define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
; CHECK-LABEL: @transpose_multiply_add(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <9 x double>* [[B_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST6]], align 8
-; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i64 3
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST9]], align 8
-; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i64 6
-; CHECK-NEXT: [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST12]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> poison, double [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> [[TMP5]], double [[TMP6]], i64 2
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> poison, double [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> [[TMP11]], double [[TMP12]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> poison, double [[TMP14]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD2]], i64 2
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD5]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <3 x double> [[TMP17]], double [[TMP18]], i64 2
-; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 3
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[B_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, ptr [[B_PTR]], i64 3
+; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> poison, double [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> poison, double [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 2
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP18]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[BLOCK14:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT15:%.*]] = insertelement <1 x double> poison, double [[TMP22]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT15]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = fmul <1 x double> [[BLOCK14]], [[SPLAT_SPLAT16]]
-; CHECK-NEXT: [[TMP24:%.*]] = fadd <1 x double> [[TMP21]], [[TMP23]]
-; CHECK-NEXT: [[BLOCK17:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT18:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT18]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP26:%.*]] = fmul <1 x double> [[BLOCK17]], [[SPLAT_SPLAT19]]
-; CHECK-NEXT: [[TMP27:%.*]] = fadd <1 x double> [[TMP24]], [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <1 x double> [[TMP27]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP28]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT: [[BLOCK20:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT21:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT21]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[BLOCK20]], [[SPLAT_SPLAT22]]
-; CHECK-NEXT: [[BLOCK23:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT24:%.*]] = insertelement <1 x double> poison, double [[TMP32]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT25:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT24]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP33:%.*]] = fmul <1 x double> [[BLOCK23]], [[SPLAT_SPLAT25]]
-; CHECK-NEXT: [[TMP34:%.*]] = fadd <1 x double> [[TMP31]], [[TMP33]]
-; CHECK-NEXT: [[BLOCK26:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT27:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT28:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT27]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP36:%.*]] = fmul <1 x double> [[BLOCK26]], [[SPLAT_SPLAT28]]
-; CHECK-NEXT: [[TMP37:%.*]] = fadd <1 x double> [[TMP34]], [[TMP36]]
-; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <1 x double> [[TMP37]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <3 x double> [[TMP29]], <3 x double> [[TMP38]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT: [[BLOCK29:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT30:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT31:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT30]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[BLOCK29]], [[SPLAT_SPLAT31]]
-; CHECK-NEXT: [[BLOCK32:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT33:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT34:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT33]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP43:%.*]] = fmul <1 x double> [[BLOCK32]], [[SPLAT_SPLAT34]]
-; CHECK-NEXT: [[TMP44:%.*]] = fadd <1 x double> [[TMP41]], [[TMP43]]
-; CHECK-NEXT: [[BLOCK35:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <3 x double> [[COL_LOAD7]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT36:%.*]] = insertelement <1 x double> poison, double [[TMP45]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT37:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT36]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP46:%.*]] = fmul <1 x double> [[BLOCK35]], [[SPLAT_SPLAT37]]
-; CHECK-NEXT: [[TMP47:%.*]] = fadd <1 x double> [[TMP44]], [[TMP46]]
-; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <1 x double> [[TMP47]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP49:%.*]] = shufflevector <3 x double> [[TMP39]], <3 x double> [[TMP48]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[BLOCK38:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT39:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT39]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK38]], [[SPLAT_SPLAT40]]
-; CHECK-NEXT: [[BLOCK41:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP52:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT42:%.*]] = insertelement <1 x double> poison, double [[TMP52]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT43:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT42]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP53:%.*]] = fmul <1 x double> [[BLOCK41]], [[SPLAT_SPLAT43]]
-; CHECK-NEXT: [[TMP54:%.*]] = fadd <1 x double> [[TMP51]], [[TMP53]]
-; CHECK-NEXT: [[BLOCK44:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP55:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT45:%.*]] = insertelement <1 x double> poison, double [[TMP55]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT46:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP56:%.*]] = fmul <1 x double> [[BLOCK44]], [[SPLAT_SPLAT46]]
-; CHECK-NEXT: [[TMP57:%.*]] = fadd <1 x double> [[TMP54]], [[TMP56]]
-; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <1 x double> [[TMP57]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP58]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT: [[BLOCK47:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT48:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT49:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT48]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP61:%.*]] = fmul <1 x double> [[BLOCK47]], [[SPLAT_SPLAT49]]
-; CHECK-NEXT: [[BLOCK50:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP62:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT51:%.*]] = insertelement <1 x double> poison, double [[TMP62]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT52:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT51]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP63:%.*]] = fmul <1 x double> [[BLOCK50]], [[SPLAT_SPLAT52]]
-; CHECK-NEXT: [[TMP64:%.*]] = fadd <1 x double> [[TMP61]], [[TMP63]]
-; CHECK-NEXT: [[BLOCK53:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP65:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT54:%.*]] = insertelement <1 x double> poison, double [[TMP65]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT55:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT54]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP66:%.*]] = fmul <1 x double> [[BLOCK53]], [[SPLAT_SPLAT55]]
-; CHECK-NEXT: [[TMP67:%.*]] = fadd <1 x double> [[TMP64]], [[TMP66]]
-; CHECK-NEXT: [[TMP68:%.*]] = shufflevector <1 x double> [[TMP67]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP69:%.*]] = shufflevector <3 x double> [[TMP59]], <3 x double> [[TMP68]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT: [[BLOCK56:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT57:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT58:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT57]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP71:%.*]] = fmul <1 x double> [[BLOCK56]], [[SPLAT_SPLAT58]]
-; CHECK-NEXT: [[BLOCK59:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP72:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT60:%.*]] = insertelement <1 x double> poison, double [[TMP72]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT61:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT60]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP73:%.*]] = fmul <1 x double> [[BLOCK59]], [[SPLAT_SPLAT61]]
-; CHECK-NEXT: [[TMP74:%.*]] = fadd <1 x double> [[TMP71]], [[TMP73]]
-; CHECK-NEXT: [[BLOCK62:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP75:%.*]] = extractelement <3 x double> [[COL_LOAD10]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT63:%.*]] = insertelement <1 x double> poison, double [[TMP75]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT64:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT63]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP76:%.*]] = fmul <1 x double> [[BLOCK62]], [[SPLAT_SPLAT64]]
-; CHECK-NEXT: [[TMP77:%.*]] = fadd <1 x double> [[TMP74]], [[TMP76]]
-; CHECK-NEXT: [[TMP78:%.*]] = shufflevector <1 x double> [[TMP77]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP79:%.*]] = shufflevector <3 x double> [[TMP69]], <3 x double> [[TMP78]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[BLOCK65:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT66:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT67:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT66]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP81:%.*]] = fmul <1 x double> [[BLOCK65]], [[SPLAT_SPLAT67]]
-; CHECK-NEXT: [[BLOCK68:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT69:%.*]] = insertelement <1 x double> poison, double [[TMP82]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT70:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT69]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP83:%.*]] = fmul <1 x double> [[BLOCK68]], [[SPLAT_SPLAT70]]
-; CHECK-NEXT: [[TMP84:%.*]] = fadd <1 x double> [[TMP81]], [[TMP83]]
-; CHECK-NEXT: [[BLOCK71:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP85:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT72:%.*]] = insertelement <1 x double> poison, double [[TMP85]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT73:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT72]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP86:%.*]] = fmul <1 x double> [[BLOCK71]], [[SPLAT_SPLAT73]]
-; CHECK-NEXT: [[TMP87:%.*]] = fadd <1 x double> [[TMP84]], [[TMP86]]
-; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <1 x double> [[TMP87]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP88]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT: [[BLOCK74:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT75:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT76:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT75]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP91:%.*]] = fmul <1 x double> [[BLOCK74]], [[SPLAT_SPLAT76]]
-; CHECK-NEXT: [[BLOCK77:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP92:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT78:%.*]] = insertelement <1 x double> poison, double [[TMP92]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT79:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT78]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP93:%.*]] = fmul <1 x double> [[BLOCK77]], [[SPLAT_SPLAT79]]
-; CHECK-NEXT: [[TMP94:%.*]] = fadd <1 x double> [[TMP91]], [[TMP93]]
-; CHECK-NEXT: [[BLOCK80:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP95:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT81:%.*]] = insertelement <1 x double> poison, double [[TMP95]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT82:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT81]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP96:%.*]] = fmul <1 x double> [[BLOCK80]], [[SPLAT_SPLAT82]]
-; CHECK-NEXT: [[TMP97:%.*]] = fadd <1 x double> [[TMP94]], [[TMP96]]
-; CHECK-NEXT: [[TMP98:%.*]] = shufflevector <1 x double> [[TMP97]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP99:%.*]] = shufflevector <3 x double> [[TMP89]], <3 x double> [[TMP98]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT: [[BLOCK83:%.*]] = shufflevector <3 x double> [[TMP7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT84:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT85:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT84]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP101:%.*]] = fmul <1 x double> [[BLOCK83]], [[SPLAT_SPLAT85]]
-; CHECK-NEXT: [[BLOCK86:%.*]] = shufflevector <3 x double> [[TMP13]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT87:%.*]] = insertelement <1 x double> poison, double [[TMP102]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT88:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT87]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP103:%.*]] = fmul <1 x double> [[BLOCK86]], [[SPLAT_SPLAT88]]
-; CHECK-NEXT: [[TMP104:%.*]] = fadd <1 x double> [[TMP101]], [[TMP103]]
-; CHECK-NEXT: [[BLOCK89:%.*]] = shufflevector <3 x double> [[TMP19]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT: [[TMP105:%.*]] = extractelement <3 x double> [[COL_LOAD13]], i64 2
-; CHECK-NEXT: [[SPLAT_SPLATINSERT90:%.*]] = insertelement <1 x double> poison, double [[TMP105]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT91:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT90]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP106:%.*]] = fmul <1 x double> [[BLOCK89]], [[SPLAT_SPLAT91]]
-; CHECK-NEXT: [[TMP107:%.*]] = fadd <1 x double> [[TMP104]], [[TMP106]]
-; CHECK-NEXT: [[TMP108:%.*]] = shufflevector <1 x double> [[TMP107]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
-; CHECK-NEXT: [[TMP109:%.*]] = shufflevector <3 x double> [[TMP99]], <3 x double> [[TMP108]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[TMP110:%.*]] = bitcast <9 x double>* [[C_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST92:%.*]] = bitcast double* [[TMP110]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD93:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST92]], align 8
-; CHECK-NEXT: [[VEC_GEP94:%.*]] = getelementptr double, double* [[TMP110]], i64 3
-; CHECK-NEXT: [[VEC_CAST95:%.*]] = bitcast double* [[VEC_GEP94]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD96:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST95]], align 8
-; CHECK-NEXT: [[VEC_GEP97:%.*]] = getelementptr double, double* [[TMP110]], i64 6
-; CHECK-NEXT: [[VEC_CAST98:%.*]] = bitcast double* [[VEC_GEP97]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD99:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST98]], align 8
-; CHECK-NEXT: [[TMP111:%.*]] = fadd <3 x double> [[COL_LOAD93]], [[TMP49]]
-; CHECK-NEXT: [[TMP112:%.*]] = fadd <3 x double> [[COL_LOAD96]], [[TMP79]]
-; CHECK-NEXT: [[TMP113:%.*]] = fadd <3 x double> [[COL_LOAD99]], [[TMP109]]
-; CHECK-NEXT: [[TMP114:%.*]] = bitcast <9 x double>* [[C_PTR]] to double*
-; CHECK-NEXT: [[VEC_CAST100:%.*]] = bitcast double* [[TMP114]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP111]], <3 x double>* [[VEC_CAST100]], align 8
-; CHECK-NEXT: [[VEC_GEP101:%.*]] = getelementptr double, double* [[TMP114]], i64 3
-; CHECK-NEXT: [[VEC_CAST102:%.*]] = bitcast double* [[VEC_GEP101]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP112]], <3 x double>* [[VEC_CAST102]], align 8
-; CHECK-NEXT: [[VEC_GEP103:%.*]] = getelementptr double, double* [[TMP114]], i64 6
-; CHECK-NEXT: [[VEC_CAST104:%.*]] = bitcast double* [[VEC_GEP103]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP113]], <3 x double>* [[VEC_CAST104]], align 8
+; CHECK-NEXT: [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT: [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
+; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP23]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT: [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP28]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT: [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
+; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP33]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT: [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP38]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT: [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
+; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP43]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
+; CHECK-NEXT: [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP48]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
+; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP51:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
+; CHECK-NEXT: [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
+; CHECK-NEXT: [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP53:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> poison, double [[TMP53]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP54:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
+; CHECK-NEXT: [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
+; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT: [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> poison, double [[TMP58]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP59:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
+; CHECK-NEXT: [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP61:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT: [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]]
+; CHECK-NEXT: [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP63:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> poison, double [[TMP63]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP64:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT: [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
+; CHECK-NEXT: [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> poison, double [[TMP68]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP69:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
+; CHECK-NEXT: [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP71:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT: [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]]
+; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP73:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> poison, double [[TMP73]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP74:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
+; CHECK-NEXT: [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
+; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT: [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> poison, double [[TMP78]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP79:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT: [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP81:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
+; CHECK-NEXT: [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]]
+; CHECK-NEXT: [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP83:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> poison, double [[TMP83]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP84:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
+; CHECK-NEXT: [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
+; CHECK-NEXT: [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <3 x double> undef, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT: [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> poison, double [[TMP88]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP89:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
+; CHECK-NEXT: [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP91:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
+; CHECK-NEXT: [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]]
+; CHECK-NEXT: [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP93:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> poison, double [[TMP93]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP94:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
+; CHECK-NEXT: [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
+; CHECK-NEXT: [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> poison, double [[TMP98]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP99:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
+; CHECK-NEXT: [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP101:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT: [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]]
+; CHECK-NEXT: [[BLOCK84:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT: [[TMP103:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
+; CHECK-NEXT: [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> poison, double [[TMP103]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP104:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
+; CHECK-NEXT: [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
+; CHECK-NEXT: [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT: [[COL_LOAD87:%.*]] = load <3 x double>, ptr [[C_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP88:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
+; CHECK-NEXT: [[COL_LOAD89:%.*]] = load <3 x double>, ptr [[VEC_GEP88]], align 8
+; CHECK-NEXT: [[VEC_GEP90:%.*]] = getelementptr double, ptr [[C_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD91:%.*]] = load <3 x double>, ptr [[VEC_GEP90]], align 8
+; CHECK-NEXT: [[TMP108:%.*]] = fadd <3 x double> [[COL_LOAD87]], [[TMP47]]
+; CHECK-NEXT: [[TMP109:%.*]] = fadd <3 x double> [[COL_LOAD89]], [[TMP77]]
+; CHECK-NEXT: [[TMP110:%.*]] = fadd <3 x double> [[COL_LOAD91]], [[TMP107]]
+; CHECK-NEXT: store <3 x double> [[TMP108]], ptr [[C_PTR]], align 8
+; CHECK-NEXT: [[VEC_GEP92:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
+; CHECK-NEXT: store <3 x double> [[TMP109]], ptr [[VEC_GEP92]], align 8
+; CHECK-NEXT: [[VEC_GEP93:%.*]] = getelementptr double, ptr [[C_PTR]], i64 6
+; CHECK-NEXT: store <3 x double> [[TMP110]], ptr [[VEC_GEP93]], align 8
; CHECK-NEXT: ret void
;
@@ -501,13 +473,13 @@ define void @transpose_multiply_add(<9 x double>* %A.Ptr, <9 x double>* %B.Ptr,
entry:
- %a = load <9 x double>, <9 x double>* %A.Ptr, align 8
- %b = load <9 x double>, <9 x double>* %B.Ptr, align 8
+ %a = load <9 x double>, ptr %A.Ptr, align 8
+ %b = load <9 x double>, ptr %B.Ptr, align 8
%a.trans = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
%mult = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
- %c = load <9 x double>, <9 x double>* %C.Ptr, align 8
+ %c = load <9 x double>, ptr %C.Ptr, align 8
%res = fadd <9 x double> %c, %mult
- store <9 x double> %res, <9 x double>* %C.Ptr, align 8
+ store <9 x double> %res, ptr %C.Ptr, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
index bd2f414ab4ea8..6671fa64ef9fa 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/const-gep.ll
@@ -11,10 +11,10 @@ define void @test(i32 %r, i32 %c) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[R_ADDR:%.*]] = alloca i32, align 4
; CHECK-NEXT: [[C_ADDR:%.*]] = alloca i32, align 4
-; CHECK-NEXT: store i32 [[R:%.*]], i32* [[R_ADDR]], align 4
-; CHECK-NEXT: store i32 [[C:%.*]], i32* [[C_ADDR]], align 4
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* bitcast ([5 x <4 x double>]* @foo to <2 x double>*), align 8
-; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, <2 x double>* bitcast (double* getelementptr ([5 x <4 x double>], [5 x <4 x double>]* @foo, i32 0, i32 0, i64 2) to <2 x double>*), align 8
+; CHECK-NEXT: store i32 [[R:%.*]], ptr [[R_ADDR]], align 4
+; CHECK-NEXT: store i32 [[C:%.*]], ptr [[C_ADDR]], align 4
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr @foo, align 8
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr getelementptr (double, ptr @foo, i64 2), align 8
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <1 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x double> [[COL_LOAD]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP0]], i32 0
@@ -67,18 +67,18 @@ define void @test(i32 %r, i32 %c) {
; CHECK-NEXT: [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP26]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: store <2 x double> [[COL_LOAD]], <2 x double>* bitcast (double* getelementptr inbounds ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 2, i32 0) to <2 x double>*), align 8
-; CHECK-NEXT: store <2 x double> [[COL_LOAD1]], <2 x double>* bitcast (double* getelementptr ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 2, i64 2) to <2 x double>*), align 8
+; CHECK-NEXT: store <2 x double> [[COL_LOAD]], ptr getelementptr inbounds ([5 x <4 x double>], ptr @foo, i64 0, i64 2), align 8
+; CHECK-NEXT: store <2 x double> [[COL_LOAD1]], ptr getelementptr (double, ptr getelementptr inbounds ([5 x <4 x double>], ptr @foo, i64 0, i64 2), i64 2), align 8
; CHECK-NEXT: ret void
;
entry:
%r.addr = alloca i32, align 4
%c.addr = alloca i32, align 4
- store i32 %r, i32* %r.addr, align 4
- store i32 %c, i32* %c.addr, align 4
- %0 = load <4 x double>, <4 x double>* getelementptr inbounds ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 0), align 8
+ store i32 %r, ptr %r.addr, align 4
+ store i32 %c, ptr %c.addr, align 4
+ %0 = load <4 x double>, ptr @foo, align 8
%mul = call <4 x double> @llvm.matrix.multiply(<4 x double> %0, <4 x double> %0, i32 2, i32 2, i32 2)
- store <4 x double> %0, <4 x double>* getelementptr inbounds ([5 x <4 x double>], [5 x <4 x double>]* @foo, i64 0, i64 2), align 8
+ store <4 x double> %0, ptr getelementptr inbounds ([5 x <4 x double>], ptr @foo, i64 0, i64 2), align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
index 25741f43055d8..f027cf397c6b1 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/load-align-volatile.ll
@@ -1,40 +1,34 @@
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define <9 x double> @strided_load_3x3_volatile(double* %in, i64 %stride) {
+define <9 x double> @strided_load_3x3_volatile(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3_volatile(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START]]
+; CHECK-NEXT: load volatile <3 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
-; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST3]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START1]]
+; CHECK-NEXT: load volatile <3 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
-; CHECK-NEXT: load volatile <3 x double>, <3 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START5]]
+; CHECK-NEXT: load volatile <3 x double>, ptr [[VEC_GEP6]], align 8
; CHECK-NOT: = load
;
entry:
- %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(double* %in, i64 %stride, i1 true, i32 3, i32 3)
+ %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(ptr %in, i64 %stride, i1 true, i32 3, i32 3)
ret <9 x double> %load
}
-declare <9 x double> @llvm.matrix.column.major.load.v9f64(double*, i64, i1, i32, i32)
+declare <9 x double> @llvm.matrix.column.major.load.v9f64(ptr, i64, i1, i32, i32)
-define <4 x double> @load_volatile_multiply(<4 x double>* %in) {
+define <4 x double> @load_volatile_multiply(ptr %in) {
; CHECK-LABEL: @load_volatile_multiply(
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x double>* [[IN:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
-; CHECK-NEXT: load volatile <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: load volatile <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT: load volatile <2 x double>, ptr [[IN:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2
+; CHECK-NEXT: load volatile <2 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NOT: = load
;
- %in.m = load volatile <4 x double>, <4 x double>* %in, align 8
+ %in.m = load volatile <4 x double>, ptr %in, align 8
%res = call <4 x double> @llvm.matrix.multiply(<4 x double> %in.m, <4 x double> %in.m, i32 2, i32 2, i32 2)
ret <4 x double> %res
}
@@ -42,85 +36,73 @@ define <4 x double> @load_volatile_multiply(<4 x double>* %in) {
declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)
-define <9 x double> @strided_load_3x3_align32(double* %in, i64 %stride) {
+define <9 x double> @strided_load_3x3_align32(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3_align32(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST]], align 32
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START]]
+; CHECK-NEXT: load <3 x double>, ptr [[VEC_GEP]], align 32
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
-; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST3]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START1]]
+; CHECK-NEXT: load <3 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
-; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START5]]
+; CHECK-NEXT: load <3 x double>, ptr [[VEC_GEP6]], align 8
; CHECK-NOT: = load
;
entry:
- %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(double* align 32 %in, i64 %stride, i1 false, i32 3, i32 3)
+ %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(ptr align 32 %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x double> %load
}
-define <9 x double> @strided_load_3x3_align2(double* %in, i64 %stride) {
+define <9 x double> @strided_load_3x3_align2(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3_align2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* %in, i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST]], align 2
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START]]
+; CHECK-NEXT: load <3 x double>, ptr [[VEC_GEP]], align 2
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* %in, i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
-; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST3]], align 2
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START1]]
+; CHECK-NEXT: load <3 x double>, ptr [[VEC_GEP2]], align 2
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* %in, i64 [[VEC_START5]]
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
-; CHECK-NEXT: load <3 x double>, <3 x double>* [[VEC_CAST7]], align 2
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, ptr %in, i64 [[VEC_START5]]
+; CHECK-NEXT: load <3 x double>, ptr [[VEC_GEP6]], align 2
; CHECK-NOT: = load
;
entry:
- %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(double* align 2 %in, i64 %stride, i1 false, i32 3, i32 3)
+ %load = call <9 x double> @llvm.matrix.column.major.load.v9f64(ptr align 2 %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x double> %load
}
-define <4 x double> @load_align2_multiply(<4 x double>* %in) {
+define <4 x double> @load_align2_multiply(ptr %in) {
; CHECK-LABEL: @load_align2_multiply(
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x double>* [[IN:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
-; CHECK-NEXT: load <2 x double>, <2 x double>* [[VEC_CAST]], align 2
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: load <2 x double>, <2 x double>* [[VEC_CAST1]], align 2
+; CHECK-NEXT: load <2 x double>, ptr [[IN:%.*]], align 2
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2
+; CHECK-NEXT: load <2 x double>, ptr [[VEC_GEP]], align 2
; CHECK-NOT: = load
;
- %in.m = load <4 x double>, <4 x double>* %in, align 2
+ %in.m = load <4 x double>, ptr %in, align 2
%res = call <4 x double> @llvm.matrix.multiply(<4 x double> %in.m, <4 x double> %in.m, i32 2, i32 2, i32 2)
ret <4 x double> %res
}
-define <6 x float> @strided_load_2x3_align16_stride2(float* %in) {
+define <6 x float> @strided_load_2x3_align16_stride2(ptr %in) {
; CHECK-LABEL: @strided_load_2x3_align16_stride2(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* %in to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* %in, i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast float* [[VEC_GEP]] to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, float* %in, i64 4
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast float* [[VEC_GEP3]] to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST4]], align 16
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr %in, align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr %in, i64 2
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, ptr %in, i64 4
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 16
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[COL_LOAD]], <2 x float> [[COL_LOAD2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[COL_LOAD5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: ret <6 x float> [[TMP3]]
;
entry:
- %load = call <6 x float> @llvm.matrix.column.major.load.v6f32(float* align 16 %in, i64 2, i1 false, i32 2, i32 3)
+ %load = call <6 x float> @llvm.matrix.column.major.load.v6f32(ptr align 16 %in, i64 2, i1 false, i32 2, i32 3)
ret <6 x float> %load
}
-declare <6 x float> @llvm.matrix.column.major.load.v6f32(float*, i64, i1, i32, i32)
+declare <6 x float> @llvm.matrix.column.major.load.v6f32(ptr, i64, i1, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
index 4343de2c9021c..caacff2e84895 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-add-sub-double-row-major.ll
@@ -5,147 +5,127 @@
; Check row-major code generation for loads, stores, binary operators (fadd/fsub) and multiply.
; %a.ptr is a pointer to a 2x3 matrix, %b.ptr to a 3x2 matrix and %c.ptr to a 2x2 matrix.
; Load, store and binary operators on %a should operate on 3 element vectors and on 2 element vectors for %b.
-define void @multiply_sub_add_2x3_3x2(<6 x double>* %a.ptr, <6 x double>* %b.ptr, <4 x double>* %c.ptr) {
+define void @multiply_sub_add_2x3_3x2(ptr %a.ptr, ptr %b.ptr, ptr %c.ptr) {
; RM-LABEL: @multiply_sub_add_2x3_3x2(
; RM-NEXT: entry:
-; RM-NEXT: [[TMP0:%.*]] = bitcast <6 x double>* [[A_PTR:%.*]] to double*
-; RM-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
-; RM-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
-; RM-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
-; RM-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; RM-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
-; RM-NEXT: [[TMP1:%.*]] = bitcast <6 x double>* [[B_PTR:%.*]] to double*
-; RM-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
-; RM-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST3]], align 8
-; RM-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, double* [[TMP1]], i64 2
-; RM-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[VEC_GEP5]] to <2 x double>*
-; RM-NEXT: [[COL_LOAD7:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST6]], align 8
-; RM-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i64 4
-; RM-NEXT: [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <2 x double>*
-; RM-NEXT: [[COL_LOAD10:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST9]], align 8
-; RM-NEXT: [[TMP2:%.*]] = fadd <3 x double> [[COL_LOAD]], [[COL_LOAD]]
-; RM-NEXT: [[TMP3:%.*]] = fadd <3 x double> [[COL_LOAD2]], [[COL_LOAD2]]
-; RM-NEXT: [[TMP4:%.*]] = bitcast <6 x double>* [[A_PTR]] to double*
-; RM-NEXT: [[VEC_CAST11:%.*]] = bitcast double* [[TMP4]] to <3 x double>*
-; RM-NEXT: store <3 x double> [[TMP2]], <3 x double>* [[VEC_CAST11]], align 8
-; RM-NEXT: [[VEC_GEP12:%.*]] = getelementptr double, double* [[TMP4]], i64 3
-; RM-NEXT: [[VEC_CAST13:%.*]] = bitcast double* [[VEC_GEP12]] to <3 x double>*
-; RM-NEXT: store <3 x double> [[TMP3]], <3 x double>* [[VEC_CAST13]], align 8
-; RM-NEXT: [[TMP5:%.*]] = fsub <2 x double> [[COL_LOAD4]], <double 1.000000e+00, double 1.000000e+00>
-; RM-NEXT: [[TMP6:%.*]] = fsub <2 x double> [[COL_LOAD7]], <double 1.000000e+00, double 1.000000e+00>
-; RM-NEXT: [[TMP7:%.*]] = fsub <2 x double> [[COL_LOAD10]], <double 1.000000e+00, double 1.000000e+00>
-; RM-NEXT: [[TMP8:%.*]] = bitcast <6 x double>* [[B_PTR]] to double*
-; RM-NEXT: [[VEC_CAST14:%.*]] = bitcast double* [[TMP8]] to <2 x double>*
-; RM-NEXT: store <2 x double> [[TMP5]], <2 x double>* [[VEC_CAST14]], align 8
-; RM-NEXT: [[VEC_GEP15:%.*]] = getelementptr double, double* [[TMP8]], i64 2
-; RM-NEXT: [[VEC_CAST16:%.*]] = bitcast double* [[VEC_GEP15]] to <2 x double>*
-; RM-NEXT: store <2 x double> [[TMP6]], <2 x double>* [[VEC_CAST16]], align 8
-; RM-NEXT: [[VEC_GEP17:%.*]] = getelementptr double, double* [[TMP8]], i64 4
-; RM-NEXT: [[VEC_CAST18:%.*]] = bitcast double* [[VEC_GEP17]] to <2 x double>*
-; RM-NEXT: store <2 x double> [[TMP7]], <2 x double>* [[VEC_CAST18]], align 8
-; RM-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP9:%.*]] = extractelement <3 x double> [[TMP2]], i64 0
-; RM-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP9]], i32 0
+; RM-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A_PTR:%.*]], align 8
+; RM-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 3
+; RM-NEXT: [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; RM-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B_PTR:%.*]], align 8
+; RM-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B_PTR]], i64 2
+; RM-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; RM-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, ptr [[B_PTR]], i64 4
+; RM-NEXT: [[COL_LOAD6:%.*]] = load <2 x double>, ptr [[VEC_GEP5]], align 8
+; RM-NEXT: [[TMP0:%.*]] = fadd <3 x double> [[COL_LOAD]], [[COL_LOAD]]
+; RM-NEXT: [[TMP1:%.*]] = fadd <3 x double> [[COL_LOAD1]], [[COL_LOAD1]]
+; RM-NEXT: store <3 x double> [[TMP0]], ptr [[A_PTR]], align 8
+; RM-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[A_PTR]], i64 3
+; RM-NEXT: store <3 x double> [[TMP1]], ptr [[VEC_GEP7]], align 8
+; RM-NEXT: [[TMP2:%.*]] = fsub <2 x double> [[COL_LOAD2]], <double 1.000000e+00, double 1.000000e+00>
+; RM-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[COL_LOAD4]], <double 1.000000e+00, double 1.000000e+00>
+; RM-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[COL_LOAD6]], <double 1.000000e+00, double 1.000000e+00>
+; RM-NEXT: store <2 x double> [[TMP2]], ptr [[B_PTR]], align 8
+; RM-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, ptr [[B_PTR]], i64 2
+; RM-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP8]], align 8
+; RM-NEXT: [[VEC_GEP9:%.*]] = getelementptr double, ptr [[B_PTR]], i64 4
+; RM-NEXT: store <2 x double> [[TMP4]], ptr [[VEC_GEP9]], align 8
+; RM-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP5:%.*]] = extractelement <3 x double> [[TMP0]], i64 0
+; RM-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP5]], i32 0
; RM-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP10:%.*]] = fmul <1 x double> [[SPLAT_SPLAT]], [[BLOCK]]
-; RM-NEXT: [[BLOCK19:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP11:%.*]] = extractelement <3 x double> [[TMP2]], i64 1
-; RM-NEXT: [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> poison, double [[TMP11]], i32 0
+; RM-NEXT: [[TMP6:%.*]] = fmul <1 x double> [[SPLAT_SPLAT]], [[BLOCK]]
+; RM-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP7:%.*]] = extractelement <3 x double> [[TMP0]], i64 1
+; RM-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <1 x double> poison, double [[TMP7]], i32 0
+; RM-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT11]], <1 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP8:%.*]] = fmul <1 x double> [[SPLAT_SPLAT12]], [[BLOCK10]]
+; RM-NEXT: [[TMP9:%.*]] = fadd <1 x double> [[TMP6]], [[TMP8]]
+; RM-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[TMP0]], i64 2
+; RM-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <1 x double> poison, double [[TMP10]], i32 0
+; RM-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT14]], <1 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP11:%.*]] = fmul <1 x double> [[SPLAT_SPLAT15]], [[BLOCK13]]
+; RM-NEXT: [[TMP12:%.*]] = fadd <1 x double> [[TMP9]], [[TMP11]]
+; RM-NEXT: [[TMP13:%.*]] = shufflevector <1 x double> [[TMP12]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP13]], <2 x i32> <i32 2, i32 1>
+; RM-NEXT: [[BLOCK16:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> <i32 1>
+; RM-NEXT: [[TMP15:%.*]] = extractelement <3 x double> [[TMP0]], i64 0
+; RM-NEXT: [[SPLAT_SPLATINSERT17:%.*]] = insertelement <1 x double> poison, double [[TMP15]], i32 0
+; RM-NEXT: [[SPLAT_SPLAT18:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT17]], <1 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP16:%.*]] = fmul <1 x double> [[SPLAT_SPLAT18]], [[BLOCK16]]
+; RM-NEXT: [[BLOCK19:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <1 x i32> <i32 1>
+; RM-NEXT: [[TMP17:%.*]] = extractelement <3 x double> [[TMP0]], i64 1
+; RM-NEXT: [[SPLAT_SPLATINSERT20:%.*]] = insertelement <1 x double> poison, double [[TMP17]], i32 0
; RM-NEXT: [[SPLAT_SPLAT21:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT20]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP12:%.*]] = fmul <1 x double> [[SPLAT_SPLAT21]], [[BLOCK19]]
-; RM-NEXT: [[TMP13:%.*]] = fadd <1 x double> [[TMP10]], [[TMP12]]
-; RM-NEXT: [[BLOCK22:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[TMP2]], i64 2
-; RM-NEXT: [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> poison, double [[TMP14]], i32 0
+; RM-NEXT: [[TMP18:%.*]] = fmul <1 x double> [[SPLAT_SPLAT21]], [[BLOCK19]]
+; RM-NEXT: [[TMP19:%.*]] = fadd <1 x double> [[TMP16]], [[TMP18]]
+; RM-NEXT: [[BLOCK22:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <1 x i32> <i32 1>
+; RM-NEXT: [[TMP20:%.*]] = extractelement <3 x double> [[TMP0]], i64 2
+; RM-NEXT: [[SPLAT_SPLATINSERT23:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i32 0
; RM-NEXT: [[SPLAT_SPLAT24:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT23]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP15:%.*]] = fmul <1 x double> [[SPLAT_SPLAT24]], [[BLOCK22]]
-; RM-NEXT: [[TMP16:%.*]] = fadd <1 x double> [[TMP13]], [[TMP15]]
-; RM-NEXT: [[TMP17:%.*]] = shufflevector <1 x double> [[TMP16]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; RM-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP17]], <2 x i32> <i32 2, i32 1>
-; RM-NEXT: [[BLOCK25:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <1 x i32> <i32 1>
-; RM-NEXT: [[TMP19:%.*]] = extractelement <3 x double> [[TMP2]], i64 0
-; RM-NEXT: [[SPLAT_SPLATINSERT26:%.*]] = insertelement <1 x double> poison, double [[TMP19]], i32 0
+; RM-NEXT: [[TMP21:%.*]] = fmul <1 x double> [[SPLAT_SPLAT24]], [[BLOCK22]]
+; RM-NEXT: [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
+; RM-NEXT: [[TMP23:%.*]] = shufflevector <1 x double> [[TMP22]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT: [[TMP24:%.*]] = shufflevector <2 x double> [[TMP14]], <2 x double> [[TMP23]], <2 x i32> <i32 0, i32 2>
+; RM-NEXT: [[BLOCK25:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP25:%.*]] = extractelement <3 x double> [[TMP1]], i64 0
+; RM-NEXT: [[SPLAT_SPLATINSERT26:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i32 0
; RM-NEXT: [[SPLAT_SPLAT27:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT26]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP20:%.*]] = fmul <1 x double> [[SPLAT_SPLAT27]], [[BLOCK25]]
-; RM-NEXT: [[BLOCK28:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <1 x i32> <i32 1>
-; RM-NEXT: [[TMP21:%.*]] = extractelement <3 x double> [[TMP2]], i64 1
-; RM-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = insertelement <1 x double> poison, double [[TMP21]], i32 0
+; RM-NEXT: [[TMP26:%.*]] = fmul <1 x double> [[SPLAT_SPLAT27]], [[BLOCK25]]
+; RM-NEXT: [[BLOCK28:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP27:%.*]] = extractelement <3 x double> [[TMP1]], i64 1
+; RM-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = insertelement <1 x double> poison, double [[TMP27]], i32 0
; RM-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT29]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP22:%.*]] = fmul <1 x double> [[SPLAT_SPLAT30]], [[BLOCK28]]
-; RM-NEXT: [[TMP23:%.*]] = fadd <1 x double> [[TMP20]], [[TMP22]]
-; RM-NEXT: [[BLOCK31:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <1 x i32> <i32 1>
-; RM-NEXT: [[TMP24:%.*]] = extractelement <3 x double> [[TMP2]], i64 2
-; RM-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = insertelement <1 x double> poison, double [[TMP24]], i32 0
+; RM-NEXT: [[TMP28:%.*]] = fmul <1 x double> [[SPLAT_SPLAT30]], [[BLOCK28]]
+; RM-NEXT: [[TMP29:%.*]] = fadd <1 x double> [[TMP26]], [[TMP28]]
+; RM-NEXT: [[BLOCK31:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <1 x i32> zeroinitializer
+; RM-NEXT: [[TMP30:%.*]] = extractelement <3 x double> [[TMP1]], i64 2
+; RM-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i32 0
; RM-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP25:%.*]] = fmul <1 x double> [[SPLAT_SPLAT33]], [[BLOCK31]]
-; RM-NEXT: [[TMP26:%.*]] = fadd <1 x double> [[TMP23]], [[TMP25]]
-; RM-NEXT: [[TMP27:%.*]] = shufflevector <1 x double> [[TMP26]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; RM-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP18]], <2 x double> [[TMP27]], <2 x i32> <i32 0, i32 2>
-; RM-NEXT: [[BLOCK34:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP29:%.*]] = extractelement <3 x double> [[TMP3]], i64 0
-; RM-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = insertelement <1 x double> poison, double [[TMP29]], i32 0
+; RM-NEXT: [[TMP31:%.*]] = fmul <1 x double> [[SPLAT_SPLAT33]], [[BLOCK31]]
+; RM-NEXT: [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
+; RM-NEXT: [[TMP33:%.*]] = shufflevector <1 x double> [[TMP32]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT: [[TMP34:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP33]], <2 x i32> <i32 2, i32 1>
+; RM-NEXT: [[BLOCK34:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <1 x i32> <i32 1>
+; RM-NEXT: [[TMP35:%.*]] = extractelement <3 x double> [[TMP1]], i64 0
+; RM-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i32 0
; RM-NEXT: [[SPLAT_SPLAT36:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT35]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP30:%.*]] = fmul <1 x double> [[SPLAT_SPLAT36]], [[BLOCK34]]
-; RM-NEXT: [[BLOCK37:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP31:%.*]] = extractelement <3 x double> [[TMP3]], i64 1
-; RM-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = insertelement <1 x double> poison, double [[TMP31]], i32 0
+; RM-NEXT: [[TMP36:%.*]] = fmul <1 x double> [[SPLAT_SPLAT36]], [[BLOCK34]]
+; RM-NEXT: [[BLOCK37:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <1 x i32> <i32 1>
+; RM-NEXT: [[TMP37:%.*]] = extractelement <3 x double> [[TMP1]], i64 1
+; RM-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = insertelement <1 x double> poison, double [[TMP37]], i32 0
; RM-NEXT: [[SPLAT_SPLAT39:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP32:%.*]] = fmul <1 x double> [[SPLAT_SPLAT39]], [[BLOCK37]]
-; RM-NEXT: [[TMP33:%.*]] = fadd <1 x double> [[TMP30]], [[TMP32]]
-; RM-NEXT: [[BLOCK40:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP34:%.*]] = extractelement <3 x double> [[TMP3]], i64 2
-; RM-NEXT: [[SPLAT_SPLATINSERT41:%.*]] = insertelement <1 x double> poison, double [[TMP34]], i32 0
+; RM-NEXT: [[TMP38:%.*]] = fmul <1 x double> [[SPLAT_SPLAT39]], [[BLOCK37]]
+; RM-NEXT: [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
+; RM-NEXT: [[BLOCK40:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <1 x i32> <i32 1>
+; RM-NEXT: [[TMP40:%.*]] = extractelement <3 x double> [[TMP1]], i64 2
+; RM-NEXT: [[SPLAT_SPLATINSERT41:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i32 0
; RM-NEXT: [[SPLAT_SPLAT42:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT41]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP35:%.*]] = fmul <1 x double> [[SPLAT_SPLAT42]], [[BLOCK40]]
-; RM-NEXT: [[TMP36:%.*]] = fadd <1 x double> [[TMP33]], [[TMP35]]
-; RM-NEXT: [[TMP37:%.*]] = shufflevector <1 x double> [[TMP36]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; RM-NEXT: [[TMP38:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP37]], <2 x i32> <i32 2, i32 1>
-; RM-NEXT: [[BLOCK43:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <1 x i32> <i32 1>
-; RM-NEXT: [[TMP39:%.*]] = extractelement <3 x double> [[TMP3]], i64 0
-; RM-NEXT: [[SPLAT_SPLATINSERT44:%.*]] = insertelement <1 x double> poison, double [[TMP39]], i32 0
-; RM-NEXT: [[SPLAT_SPLAT45:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT44]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP40:%.*]] = fmul <1 x double> [[SPLAT_SPLAT45]], [[BLOCK43]]
-; RM-NEXT: [[BLOCK46:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <1 x i32> <i32 1>
-; RM-NEXT: [[TMP41:%.*]] = extractelement <3 x double> [[TMP3]], i64 1
-; RM-NEXT: [[SPLAT_SPLATINSERT47:%.*]] = insertelement <1 x double> poison, double [[TMP41]], i32 0
-; RM-NEXT: [[SPLAT_SPLAT48:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT47]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP42:%.*]] = fmul <1 x double> [[SPLAT_SPLAT48]], [[BLOCK46]]
-; RM-NEXT: [[TMP43:%.*]] = fadd <1 x double> [[TMP40]], [[TMP42]]
-; RM-NEXT: [[BLOCK49:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <1 x i32> <i32 1>
-; RM-NEXT: [[TMP44:%.*]] = extractelement <3 x double> [[TMP3]], i64 2
-; RM-NEXT: [[SPLAT_SPLATINSERT50:%.*]] = insertelement <1 x double> poison, double [[TMP44]], i32 0
-; RM-NEXT: [[SPLAT_SPLAT51:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT50]], <1 x double> poison, <1 x i32> zeroinitializer
-; RM-NEXT: [[TMP45:%.*]] = fmul <1 x double> [[SPLAT_SPLAT51]], [[BLOCK49]]
-; RM-NEXT: [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
-; RM-NEXT: [[TMP47:%.*]] = shufflevector <1 x double> [[TMP46]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; RM-NEXT: [[TMP48:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> [[TMP47]], <2 x i32> <i32 0, i32 2>
-; RM-NEXT: [[TMP49:%.*]] = bitcast <4 x double>* [[C_PTR:%.*]] to double*
-; RM-NEXT: [[VEC_CAST52:%.*]] = bitcast double* [[TMP49]] to <2 x double>*
-; RM-NEXT: [[COL_LOAD53:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST52]], align 8
-; RM-NEXT: [[VEC_GEP54:%.*]] = getelementptr double, double* [[TMP49]], i64 2
-; RM-NEXT: [[VEC_CAST55:%.*]] = bitcast double* [[VEC_GEP54]] to <2 x double>*
-; RM-NEXT: [[COL_LOAD56:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST55]], align 8
-; RM-NEXT: [[TMP50:%.*]] = fsub <2 x double> [[COL_LOAD53]], [[TMP28]]
-; RM-NEXT: [[TMP51:%.*]] = fsub <2 x double> [[COL_LOAD56]], [[TMP48]]
-; RM-NEXT: [[TMP52:%.*]] = bitcast <4 x double>* [[C_PTR]] to double*
-; RM-NEXT: [[VEC_CAST57:%.*]] = bitcast double* [[TMP52]] to <2 x double>*
-; RM-NEXT: store <2 x double> [[TMP50]], <2 x double>* [[VEC_CAST57]], align 8
-; RM-NEXT: [[VEC_GEP58:%.*]] = getelementptr double, double* [[TMP52]], i64 2
-; RM-NEXT: [[VEC_CAST59:%.*]] = bitcast double* [[VEC_GEP58]] to <2 x double>*
-; RM-NEXT: store <2 x double> [[TMP51]], <2 x double>* [[VEC_CAST59]], align 8
+; RM-NEXT: [[TMP41:%.*]] = fmul <1 x double> [[SPLAT_SPLAT42]], [[BLOCK40]]
+; RM-NEXT: [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
+; RM-NEXT: [[TMP43:%.*]] = shufflevector <1 x double> [[TMP42]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; RM-NEXT: [[TMP44:%.*]] = shufflevector <2 x double> [[TMP34]], <2 x double> [[TMP43]], <2 x i32> <i32 0, i32 2>
+; RM-NEXT: [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[C_PTR:%.*]], align 8
+; RM-NEXT: [[VEC_GEP44:%.*]] = getelementptr double, ptr [[C_PTR]], i64 2
+; RM-NEXT: [[COL_LOAD45:%.*]] = load <2 x double>, ptr [[VEC_GEP44]], align 8
+; RM-NEXT: [[TMP45:%.*]] = fsub <2 x double> [[COL_LOAD43]], [[TMP24]]
+; RM-NEXT: [[TMP46:%.*]] = fsub <2 x double> [[COL_LOAD45]], [[TMP44]]
+; RM-NEXT: store <2 x double> [[TMP45]], ptr [[C_PTR]], align 8
+; RM-NEXT: [[VEC_GEP46:%.*]] = getelementptr double, ptr [[C_PTR]], i64 2
+; RM-NEXT: store <2 x double> [[TMP46]], ptr [[VEC_GEP46]], align 8
; RM-NEXT: ret void
;
entry:
- %a = load <6 x double>, <6 x double>* %a.ptr, align 8
- %b = load <6 x double>, <6 x double>* %b.ptr, align 8
+ %a = load <6 x double>, ptr %a.ptr, align 8
+ %b = load <6 x double>, ptr %b.ptr, align 8
%add = fadd <6 x double> %a, %a
- store <6 x double> %add, <6 x double>* %a.ptr, align 8
+ store <6 x double> %add, ptr %a.ptr, align 8
%sub = fsub <6 x double> %b, <double 1.0, double 1.0, double 1.0, double 1.0, double 1.0, double 1.0>
- store <6 x double> %sub, <6 x double>* %b.ptr, align 8
+ store <6 x double> %sub, ptr %b.ptr, align 8
%mul = call <4 x double> @llvm.matrix.multiply.v4f64.v6f64.v6f64(<6 x double> %add, <6 x double> %sub, i32 2, i32 3, i32 2)
- %c = load <4 x double>, <4 x double>* %c.ptr, align 8
+ %c = load <4 x double>, ptr %c.ptr, align 8
%res = fsub <4 x double> %c, %mul
- store <4 x double> %res, <4 x double>* %c.ptr, align 8
+ store <4 x double> %res, ptr %c.ptr, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll
index d8109cc5f921e..7da58137ab87d 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll
@@ -6,12 +6,12 @@
target datalayout = "e-m:o-i64:64-f80:128-n8:4:32:64-S128"
target triple = "aarch64-apple-ios"
-define void @multiply_can_hoist_cast(<4 x double>* noalias %A, <4 x double> * %B, [4 x double]* %C) {
+define void @multiply_can_hoist_cast(ptr noalias %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply_can_hoist_cast(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint [4 x double]* [[C:%.*]] to i64
+; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[C:%.*]] to i64
; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 32
-; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint <4 x double>* [[B:%.*]] to i64
+; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[B:%.*]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
; CHECK: alias_cont:
@@ -19,94 +19,71 @@ define void @multiply_can_hoist_cast(<4 x double>* noalias %A, <4 x double> * %B
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
; CHECK: copy:
-; CHECK-NEXT: [[TMP2:%.*]] = alloca <4 x double>, align 8
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x double>* [[TMP2]] to i8*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(32) [[TMP3]], i8* noundef nonnull align 8 dereferenceable(32) [[TMP4]], i64 32, i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(32) [[B]], i64 32, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS]]
; CHECK: no_alias:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x double>* [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A:%.*]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast <4 x double>* [[TMP5]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST2]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD3]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast double* [[TMP7]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST5]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast double* [[TMP8]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST8]], align 8
-; CHECK-NEXT: [[TMP9:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD6]], <1 x double> [[COL_LOAD9]], <1 x double> [[TMP6]])
-; CHECK-NEXT: [[VEC_CAST15:%.*]] = bitcast [4 x double]* [[C]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP9]], <1 x double>* [[VEC_CAST15]], align 8
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast double* [[TMP10]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD18:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_CAST20:%.*]] = bitcast <4 x double>* [[TMP5]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD21:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST20]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = fmul contract <1 x double> [[COL_LOAD18]], [[COL_LOAD21]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST26:%.*]] = bitcast double* [[TMP12]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST26]], align 8
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST29:%.*]] = bitcast double* [[TMP13]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST29]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD27]], <1 x double> [[COL_LOAD30]], <1 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST36:%.*]] = bitcast double* [[TMP15]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP14]], <1 x double>* [[VEC_CAST36]], align 8
-; CHECK-NEXT: [[VEC_CAST38:%.*]] = bitcast <4 x double>* [[A]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD39:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST38]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST41:%.*]] = bitcast double* [[TMP16]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD42:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST41]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD39]], [[COL_LOAD42]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST47:%.*]] = bitcast double* [[TMP18]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST47]], align 8
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST50:%.*]] = bitcast double* [[TMP19]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST50]], align 8
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD48]], <1 x double> [[COL_LOAD51]], <1 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST57:%.*]] = bitcast double* [[TMP21]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP20]], <1 x double>* [[VEC_CAST57]], align 8
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST59:%.*]] = bitcast double* [[TMP22]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD60:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST59]], align 8
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST62:%.*]] = bitcast double* [[TMP23]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD63:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST62]], align 8
-; CHECK-NEXT: [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD60]], [[COL_LOAD63]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST68:%.*]] = bitcast double* [[TMP25]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD69:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST68]], align 8
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST71:%.*]] = bitcast double* [[TMP26]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD72:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST71]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD69]], <1 x double> [[COL_LOAD72]], <1 x double> [[TMP24]])
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST78:%.*]] = bitcast double* [[TMP28]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP27]], <1 x double>* [[VEC_CAST78]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])
+; CHECK-NEXT: store <1 x double> [[TMP7]], ptr [[C]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[C]], i64 1
+; CHECK-NEXT: store <1 x double> [[TMP12]], ptr [[TMP13]], align 8
+; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %c.cast = bitcast [4 x double]* %C to <4 x double>*
- store <4 x double> %c, <4 x double>* %c.cast, align 8
+ store <4 x double> %c, ptr %C, align 8
ret void
}
-define void @multiply_can_hoist_multiple_insts(<4 x double>* noalias %A, <4 x double> * %B, [4 x double]* %C) {
+define void @multiply_can_hoist_multiple_insts(ptr noalias %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply_can_hoist_multiple_insts(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x double], [4 x double]* [[C:%.*]], i64 2
-; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint [4 x double]* [[GEP]] to i64
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x double], ptr [[C:%.*]], i64 2
+; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[GEP]] to i64
; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 32
-; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint <4 x double>* [[B:%.*]] to i64
+; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[B:%.*]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
; CHECK: alias_cont:
@@ -114,96 +91,73 @@ define void @multiply_can_hoist_multiple_insts(<4 x double>* noalias %A, <4 x do
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
; CHECK: copy:
-; CHECK-NEXT: [[TMP2:%.*]] = alloca <4 x double>, align 8
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x double>* [[TMP2]] to i8*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(32) [[TMP3]], i8* noundef nonnull align 8 dereferenceable(32) [[TMP4]], i64 32, i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(32) [[B]], i64 32, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS]]
; CHECK: no_alias:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x double>* [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A:%.*]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast <4 x double>* [[TMP5]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST2]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD3]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast double* [[TMP7]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST5]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast double* [[TMP8]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST8]], align 8
-; CHECK-NEXT: [[TMP9:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD6]], <1 x double> [[COL_LOAD9]], <1 x double> [[TMP6]])
-; CHECK-NEXT: [[VEC_CAST15:%.*]] = bitcast [4 x double]* [[GEP]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP9]], <1 x double>* [[VEC_CAST15]], align 8
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast double* [[TMP10]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD18:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_CAST20:%.*]] = bitcast <4 x double>* [[TMP5]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD21:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST20]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = fmul contract <1 x double> [[COL_LOAD18]], [[COL_LOAD21]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST26:%.*]] = bitcast double* [[TMP12]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST26]], align 8
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST29:%.*]] = bitcast double* [[TMP13]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST29]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD27]], <1 x double> [[COL_LOAD30]], <1 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 2, i64 1
-; CHECK-NEXT: [[VEC_CAST36:%.*]] = bitcast double* [[TMP15]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP14]], <1 x double>* [[VEC_CAST36]], align 8
-; CHECK-NEXT: [[VEC_CAST38:%.*]] = bitcast <4 x double>* [[A]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD39:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST38]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST41:%.*]] = bitcast double* [[TMP16]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD42:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST41]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD39]], [[COL_LOAD42]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST47:%.*]] = bitcast double* [[TMP18]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST47]], align 8
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST50:%.*]] = bitcast double* [[TMP19]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST50]], align 8
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD48]], <1 x double> [[COL_LOAD51]], <1 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 2, i64 2
-; CHECK-NEXT: [[VEC_CAST57:%.*]] = bitcast double* [[TMP21]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP20]], <1 x double>* [[VEC_CAST57]], align 8
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST59:%.*]] = bitcast double* [[TMP22]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD60:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST59]], align 8
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST62:%.*]] = bitcast double* [[TMP23]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD63:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST62]], align 8
-; CHECK-NEXT: [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD60]], [[COL_LOAD63]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST68:%.*]] = bitcast double* [[TMP25]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD69:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST68]], align 8
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST71:%.*]] = bitcast double* [[TMP26]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD72:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST71]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD69]], <1 x double> [[COL_LOAD72]], <1 x double> [[TMP24]])
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 2, i64 3
-; CHECK-NEXT: [[VEC_CAST78:%.*]] = bitcast double* [[TMP28]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP27]], <1 x double>* [[VEC_CAST78]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])
+; CHECK-NEXT: store <1 x double> [[TMP7]], ptr [[GEP]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 1
+; CHECK-NEXT: store <1 x double> [[TMP12]], ptr [[TMP13]], align 8
+; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 2
+; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr [4 x double], ptr [[C]], i64 2, i64 3
+; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %gep = getelementptr [4 x double], [4 x double]* %C, i32 2
- %c.cast = bitcast [4 x double]* %gep to <4 x double>*
- store <4 x double> %c, <4 x double>* %c.cast, align 8
+ %gep = getelementptr [4 x double], ptr %C, i32 2
+ store <4 x double> %c, ptr %gep, align 8
ret void
}
; Make sure the correct instruction order is preserved when hoisting.
-define void @multiply_can_hoist_multiple_insts2(<4 x double>* noalias %A, <4 x double> * %B, [4 x double]* %C) {
+define void @multiply_can_hoist_multiple_insts2(ptr noalias %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply_can_hoist_multiple_insts2(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP_179:%.*]] = getelementptr [4 x double], [4 x double]* [[C:%.*]], i64 42
-; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint [4 x double]* [[GEP_179]] to i64
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr <4 x double>, ptr [[C:%.*]], i64 42
+; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[GEP_1]] to i64
; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 32
-; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint <4 x double>* [[B:%.*]] to i64
+; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[B:%.*]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
; CHECK: alias_cont:
@@ -211,127 +165,97 @@ define void @multiply_can_hoist_multiple_insts2(<4 x double>* noalias %A, <4 x d
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
; CHECK: copy:
-; CHECK-NEXT: [[TMP2:%.*]] = alloca <4 x double>, align 8
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x double>* [[TMP2]] to i8*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(32) [[TMP3]], i8* noundef nonnull align 8 dereferenceable(32) [[TMP4]], i64 32, i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(32) [[B]], i64 32, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS]]
; CHECK: no_alias:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x double>* [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A:%.*]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast <4 x double>* [[TMP5]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST2]], align 8
-; CHECK-NEXT: [[TMP6:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD3]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast double* [[TMP7]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST5]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast double* [[TMP8]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST8]], align 8
-; CHECK-NEXT: [[TMP9:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD6]], <1 x double> [[COL_LOAD9]], <1 x double> [[TMP6]])
-; CHECK-NEXT: [[VEC_CAST15:%.*]] = bitcast [4 x double]* [[GEP_179]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP9]], <1 x double>* [[VEC_CAST15]], align 8
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast double* [[TMP10]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD18:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_CAST20:%.*]] = bitcast <4 x double>* [[TMP5]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD21:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST20]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = fmul contract <1 x double> [[COL_LOAD18]], [[COL_LOAD21]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST26:%.*]] = bitcast double* [[TMP12]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST26]], align 8
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST29:%.*]] = bitcast double* [[TMP13]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST29]], align 8
-; CHECK-NEXT: [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD27]], <1 x double> [[COL_LOAD30]], <1 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 42, i64 1
-; CHECK-NEXT: [[VEC_CAST36:%.*]] = bitcast double* [[TMP15]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP14]], <1 x double>* [[VEC_CAST36]], align 8
-; CHECK-NEXT: [[VEC_CAST38:%.*]] = bitcast <4 x double>* [[A]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD39:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST38]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST41:%.*]] = bitcast double* [[TMP16]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD42:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST41]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD39]], [[COL_LOAD42]]
-; CHECK-NEXT: [[TMP18:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST47:%.*]] = bitcast double* [[TMP18]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST47]], align 8
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST50:%.*]] = bitcast double* [[TMP19]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST50]], align 8
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD48]], <1 x double> [[COL_LOAD51]], <1 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 42, i64 2
-; CHECK-NEXT: [[VEC_CAST57:%.*]] = bitcast double* [[TMP21]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP20]], <1 x double>* [[VEC_CAST57]], align 8
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 1
-; CHECK-NEXT: [[VEC_CAST59:%.*]] = bitcast double* [[TMP22]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD60:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST59]], align 8
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST62:%.*]] = bitcast double* [[TMP23]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD63:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST62]], align 8
-; CHECK-NEXT: [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD60]], [[COL_LOAD63]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST68:%.*]] = bitcast double* [[TMP25]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD69:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST68]], align 8
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr <4 x double>, <4 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST71:%.*]] = bitcast double* [[TMP26]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD72:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST71]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD69]], <1 x double> [[COL_LOAD72]], <1 x double> [[TMP24]])
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 42, i64 3
-; CHECK-NEXT: [[VEC_CAST78:%.*]] = bitcast double* [[TMP28]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP27]], <1 x double>* [[VEC_CAST78]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])
+; CHECK-NEXT: store <1 x double> [[TMP7]], ptr [[GEP_1]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr double, ptr [[TMP3]], i64 1
+; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr <4 x double>, ptr [[C]], i64 42, i64 1
+; CHECK-NEXT: store <1 x double> [[TMP12]], ptr [[TMP13]], align 8
+; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8
+; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr <4 x double>, ptr [[C]], i64 42, i64 2
+; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[TMP19]], align 8
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr <4 x double>, ptr [[C]], i64 42, i64 3
+; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %c.cast = bitcast [4 x double]* %C to <4 x double>*
%off.0 = add i32 10, 10
%off.1 = add i32 %off.0, 2
%off.2 = add i32 %off.0, %off.1
- %gep.1 = getelementptr <4 x double>, <4 x double>* %c.cast, i32 %off.2
- store <4 x double> %c, <4 x double>* %gep.1, align 8
+ %gep.1 = getelementptr <4 x double>, ptr %C, i32 %off.2
+ store <4 x double> %c, ptr %gep.1, align 8
ret void
}
-define void @multiply_dont_hoist_phi(<4 x double>* noalias %A, <4 x double> * %B, [4 x double]* %C) {
+define void @multiply_dont_hoist_phi(ptr noalias %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply_dont_hoist_phi(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[NEXT:%.*]]
; CHECK: next:
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <4 x double>, <4 x double>* [[A:%.*]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr <4 x double>, <4 x double>* [[B:%.*]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[VEC_GEP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST6]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT13]]
-; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP0]])
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast <4 x double>* [[B]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST3]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B:%.*]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP0]])
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT10]], <2 x double> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [4 x double], [4 x double]* [[C:%.*]], i64 26, i64 0
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP3]], <2 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_GEP18:%.*]] = getelementptr [4 x double], [4 x double]* [[C]], i64 26, i64 2
-; CHECK-NEXT: [[VEC_CAST19:%.*]] = bitcast double* [[VEC_GEP18]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP1]], <2 x double>* [[VEC_CAST19]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr <4 x double>, ptr [[C:%.*]], i64 26
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[GEP_1]], align 8
+; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr <4 x double>, ptr [[C]], i64 26, i64 2
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[VEC_GEP14]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %c.cast = bitcast [4 x double]* %C to <4 x double>*
br label %next
next:
@@ -339,132 +263,111 @@ next:
%off.0 = add i32 10, %p
%off.1 = add i32 %off.0, 2
%off.2 = add i32 %off.0, %off.1
- %gep.1 = getelementptr <4 x double>, <4 x double>* %c.cast, i32 %off.2
- store <4 x double> %c, <4 x double>* %gep.1, align 8
+ %gep.1 = getelementptr <4 x double>, ptr %C, i32 %off.2
+ store <4 x double> %c, ptr %gep.1, align 8
ret void
}
; The address load may alias, so avoid moving it for now.
-define void @multiply_dont_hoist_cast_due_to_operand(<4 x double>* noalias %A, <4 x double> * %B, [4 x double]** %C.ptr) {
+define void @multiply_dont_hoist_cast_due_to_operand(ptr noalias %A, ptr %B, ptr %C.ptr) {
; CHECK-LABEL: @multiply_dont_hoist_cast_due_to_operand(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast <4 x double>* [[B:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST3]], align 8
-; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr <4 x double>, <4 x double>* [[B]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[VEC_GEP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST6]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT10]], <2 x double> [[TMP0]])
-; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT13]]
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast [4 x double]** [[C_PTR:%.*]] to double**
-; CHECK-NEXT: [[C2021:%.*]] = load double*, double** [[TMP4]], align 8
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast double* [[C2021]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP1]], <2 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_GEP18:%.*]] = getelementptr double, double* [[C2021]], i64 2
-; CHECK-NEXT: [[VEC_CAST19:%.*]] = bitcast double* [[VEC_GEP18]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP3]], <2 x double>* [[VEC_CAST19]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8
+; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %C = load [4 x double]*, [4 x double]** %C.ptr
- %c.cast = bitcast [4 x double]* %C to <4 x double>*
- store <4 x double> %c, <4 x double>* %c.cast, align 8
+ %C = load ptr, ptr %C.ptr
+ store <4 x double> %c, ptr %C, align 8
ret void
}
; The address load may alias, so avoid moving it for now.
-define void @multiply_dont_hoist_load(<4 x double>* noalias %A, <4 x double> * %B, <4 x double>** %C.ptr) {
+define void @multiply_dont_hoist_load(ptr noalias %A, ptr %B, ptr %C.ptr) {
; CHECK-LABEL: @multiply_dont_hoist_load(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast <4 x double>* [[B:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST3]], align 8
-; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr <4 x double>, <4 x double>* [[B]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[VEC_GEP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST6]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT10]], <2 x double> [[TMP0]])
-; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT13]]
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>** [[C_PTR:%.*]] to double**
-; CHECK-NEXT: [[C20:%.*]] = load double*, double** [[TMP4]], align 8
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast double* [[C20]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP1]], <2 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_GEP18:%.*]] = getelementptr double, double* [[C20]], i64 2
-; CHECK-NEXT: [[VEC_CAST19:%.*]] = bitcast double* [[VEC_GEP18]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP3]], <2 x double>* [[VEC_CAST19]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8
+; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %C = load <4 x double>*, <4 x double>** %C.ptr
- store <4 x double> %c, <4 x double>* %C, align 8
+ %C = load ptr, ptr %C.ptr
+ store <4 x double> %c, ptr %C, align 8
ret void
}
; The call to @get_adress may clobber memory, avoid moving it for now.
-define void @multiply_dont_hoist_call(<4 x double>* noalias %A, <4 x double> * %B) {
+define void @multiply_dont_hoist_call(ptr noalias %A, ptr %B) {
; CHECK-LABEL: @multiply_dont_hoist_call(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <4 x double>* [[A:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <4 x double>, <4 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast <4 x double>* [[B:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST3]], align 8
-; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr <4 x double>, <4 x double>* [[B]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[VEC_GEP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST6]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[B]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT10]], <2 x double> [[TMP0]])
-; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT13]]
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[COL_LOAD7]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP2]])
-; CHECK-NEXT: [[C:%.*]] = call <4 x double>* @get_address()
-; CHECK-NEXT: [[VEC_CAST17:%.*]] = bitcast <4 x double>* [[C]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP1]], <2 x double>* [[VEC_CAST17]], align 8
-; CHECK-NEXT: [[VEC_GEP18:%.*]] = getelementptr <4 x double>, <4 x double>* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST19:%.*]] = bitcast double* [[VEC_GEP18]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP3]], <2 x double>* [[VEC_CAST19]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[C:%.*]] = call ptr @get_address()
+; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8
+; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- %C = call <4 x double>* @get_address()
- store <4 x double> %c, <4 x double>* %C, align 8
+ %C = call ptr @get_address()
+ store <4 x double> %c, ptr %C, align 8
ret void
}
-declare <4 x double>* @get_address()
+declare ptr @get_address()
declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
index f64f5b7842065..2798f114d6190 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
@@ -6,7 +6,7 @@
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "aarch64-apple-ios"
-define void @multiply_noalias_4x4(<16 x double>* noalias %A, <16 x double>* noalias %B, <16 x double>* noalias %C) {
+define void @multiply_noalias_4x4(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_noalias_4x4(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -28,28 +28,24 @@ define void @multiply_noalias_4x4(<16 x double>* noalias %A, <16 x double>* noal
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INNER_IV]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x double>, <16 x double>* [[A:%.*]], i64 0, i64 [[TMP1]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP2]], i64 4
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 4
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[COLS_IV]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, <16 x double>* [[B:%.*]], i64 0, i64 [[TMP4]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP5]], i64 4
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD5]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 4
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[RESULT_VEC_0]])
-; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[COL_LOAD5]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP7]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP6]])
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[COL_LOAD8]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[RESULT_VEC_1]])
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[COL_LOAD8]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP9]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP8]])
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP7]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP6]])
+; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[RESULT_VEC_1]])
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP9]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP8]])
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2
@@ -60,12 +56,10 @@ define void @multiply_noalias_4x4(<16 x double>* noalias %A, <16 x double>* noal
; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 4
; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[COLS_IV]], 2
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr <16 x double>, <16 x double>* [[C:%.*]], i64 0, i64 [[TMP11]]
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast double* [[TMP12]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP7]], <2 x double>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP12]], i64 4
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP9]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP11]]
+; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[TMP12]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP12]], i64 4
+; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -76,19 +70,19 @@ define void @multiply_noalias_4x4(<16 x double>* noalias %A, <16 x double>* noal
;
entry:
- %a = load <16 x double>, <16 x double>* %A, align 8
- %b = load <16 x double>, <16 x double>* %B, align 8
+ %a = load <16 x double>, ptr %A, align 8
+ %b = load <16 x double>, ptr %B, align 8
%c = call <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double> %a, <16 x double> %b, i32 4, i32 4, i32 4)
- store <16 x double> %c, <16 x double>* %C, align 8
+ store <16 x double> %c, ptr %C, align 8
ret void
}
declare <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double>, <16 x double>, i32, i32, i32)
-define void @multiply_noalias_2x4(<8 x i64>* noalias %A, <8 x i64>* noalias %B, <4 x i64>* noalias %C) {
+define void @multiply_noalias_2x4(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_noalias_2x4(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -110,31 +104,27 @@ define void @multiply_noalias_2x4(<8 x i64>* noalias %A, <8 x i64>* noalias %B,
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INNER_IV]], 1
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr <8 x i64>, <8 x i64>* [[A:%.*]], i64 0, i64 [[TMP1]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i64* [[TMP2]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i64, i64* [[TMP2]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast i64* [[VEC_GEP]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i64, ptr [[TMP2]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[COLS_IV]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr <8 x i64>, <8 x i64>* [[B:%.*]], i64 0, i64 [[TMP4]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast i64* [[TMP5]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i64, i64* [[TMP5]], i64 4
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast i64* [[VEC_GEP6]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i64, ptr [[TMP5]], i64 4
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i64>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i64> [[RESULT_VEC_0]], [[TMP6]]
-; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT8]]
; CHECK-NEXT: [[TMP9]] = add <2 x i64> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT12]]
; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i64> [[RESULT_VEC_1]], [[TMP10]]
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT15]]
; CHECK-NEXT: [[TMP13]] = add <2 x i64> [[TMP11]], [[TMP12]]
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
@@ -146,12 +136,10 @@ define void @multiply_noalias_2x4(<8 x i64>* noalias %A, <8 x i64>* noalias %B,
; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0
; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[COLS_IV]], 1
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP14]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <4 x i64>, <4 x i64>* [[C:%.*]], i64 0, i64 [[TMP15]]
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast i64* [[TMP16]] to <2 x i64>*
-; CHECK-NEXT: store <2 x i64> [[TMP9]], <2 x i64>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr i64, i64* [[TMP16]], i64 2
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast i64* [[VEC_GEP22]] to <2 x i64>*
-; CHECK-NEXT: store <2 x i64> [[TMP13]], <2 x i64>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[C:%.*]], i64 [[TMP15]]
+; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP16]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr i64, ptr [[TMP16]], i64 2
+; CHECK-NEXT: store <2 x i64> [[TMP13]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -168,19 +156,19 @@ define void @multiply_noalias_2x4(<8 x i64>* noalias %A, <8 x i64>* noalias %B,
; Store the current 2x2 tile.
entry:
- %a = load <8 x i64>, <8 x i64>* %A, align 8
- %b = load <8 x i64>, <8 x i64>* %B, align 8
+ %a = load <8 x i64>, ptr %A, align 8
+ %b = load <8 x i64>, ptr %B, align 8
%c = call <4 x i64> @llvm.matrix.multiply.v4i64.v8i64.v8i64(<8 x i64> %a, <8 x i64> %b, i32 2, i32 4, i32 2)
- store <4 x i64> %c, <4 x i64>* %C, align 8
+ store <4 x i64> %c, ptr %C, align 8
ret void
}
declare <4 x i64> @llvm.matrix.multiply.v4i64.v8i64.v8i64(<8 x i64>, <8 x i64>, i32, i32, i32)
-define void @multiply_noalias_4x2_2x8(<8 x i64>* noalias %A, <16 x i64>* noalias %B, <32 x i64>* noalias %C) {
+define void @multiply_noalias_4x2_2x8(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_noalias_4x2_2x8(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -202,31 +190,27 @@ define void @multiply_noalias_4x2_2x8(<8 x i64>* noalias %A, <16 x i64>* noalias
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INNER_IV]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr <8 x i64>, <8 x i64>* [[A:%.*]], i64 0, i64 [[TMP1]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i64* [[TMP2]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i64, i64* [[TMP2]], i64 4
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast i64* [[VEC_GEP]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i64, ptr [[TMP2]], i64 4
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[COLS_IV]], 1
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i64>, <16 x i64>* [[B:%.*]], i64 0, i64 [[TMP4]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast i64* [[TMP5]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i64, i64* [[TMP5]], i64 2
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast i64* [[VEC_GEP6]] to <2 x i64>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x i64>, <2 x i64>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i64, ptr [[TMP5]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i64>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i64> [[RESULT_VEC_0]], [[TMP6]]
-; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD5]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT12]]
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT8]]
; CHECK-NEXT: [[TMP9]] = add <2 x i64> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT16]]
+; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT12]]
; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i64> [[RESULT_VEC_1]], [[TMP10]]
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x i64> [[COL_LOAD8]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD2]], [[SPLAT_SPLAT19]]
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT15]]
; CHECK-NEXT: [[TMP13]] = add <2 x i64> [[TMP11]], [[TMP12]]
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
@@ -238,12 +222,10 @@ define void @multiply_noalias_4x2_2x8(<8 x i64>* noalias %A, <16 x i64>* noalias
; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 4
; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[COLS_IV]], 2
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP14]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <32 x i64>, <32 x i64>* [[C:%.*]], i64 0, i64 [[TMP15]]
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast i64* [[TMP16]] to <2 x i64>*
-; CHECK-NEXT: store <2 x i64> [[TMP9]], <2 x i64>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr i64, i64* [[TMP16]], i64 4
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast i64* [[VEC_GEP22]] to <2 x i64>*
-; CHECK-NEXT: store <2 x i64> [[TMP13]], <2 x i64>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[C:%.*]], i64 [[TMP15]]
+; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP16]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr i64, ptr [[TMP16]], i64 4
+; CHECK-NEXT: store <2 x i64> [[TMP13]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -260,12 +242,12 @@ define void @multiply_noalias_4x2_2x8(<8 x i64>* noalias %A, <16 x i64>* noalias
; Store the current 2x2 tile.
entry:
- %a = load <8 x i64>, <8 x i64>* %A, align 8
- %b = load <16 x i64>, <16 x i64>* %B, align 8
+ %a = load <8 x i64>, ptr %A, align 8
+ %b = load <16 x i64>, ptr %B, align 8
%c = call <32 x i64> @llvm.matrix.multiply.v32i64.v8i64.v16i64(<8 x i64> %a, <16 x i64> %b, i32 4, i32 2, i32 8)
- store <32 x i64> %c, <32 x i64>* %C, align 8
+ store <32 x i64> %c, ptr %C, align 8
ret void
}
@@ -273,12 +255,12 @@ declare <32 x i64> @llvm.matrix.multiply.v32i64.v8i64.v16i64(<8 x i64>, <16 x i6
; Check the runtime aliasing checks.
-define void @multiply_alias_2x2(<4 x float>* %A, <4 x float>* %B, <4 x float>* %C) {
+define void @multiply_alias_2x2(ptr %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply_alias_2x2(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint <4 x float>* [[C:%.*]] to i64
+; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[C:%.*]] to i64
; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 16
-; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint <4 x float>* [[A:%.*]] to i64
+; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[A:%.*]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
; CHECK: alias_cont:
@@ -286,30 +268,26 @@ define void @multiply_alias_2x2(<4 x float>* %A, <4 x float>* %B, <4 x float>* %
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
; CHECK: copy:
-; CHECK-NEXT: [[TMP2:%.*]] = alloca <4 x float>, align 4
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x float>* [[TMP2]] to i8*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x float>* [[A]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) [[TMP3]], i8* noundef nonnull align 8 dereferenceable(16) [[TMP4]], i64 16, i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x float], align 4
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(16) [[A]], i64 16, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS]]
; CHECK: no_alias:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x float>* [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
-; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint <4 x float>* [[C]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint ptr [[C]] to i64
; CHECK-NEXT: [[STORE_END5:%.*]] = add nuw nsw i64 [[STORE_BEGIN4]], 16
-; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint <4 x float>* [[B:%.*]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
+; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint ptr [[B:%.*]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
; CHECK: alias_cont1:
; CHECK-NEXT: [[LOAD_END7:%.*]] = add nuw nsw i64 [[LOAD_BEGIN6]], 16
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
; CHECK: copy2:
-; CHECK-NEXT: [[TMP8:%.*]] = alloca <4 x float>, align 4
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x float>* [[TMP8]] to i8*
-; CHECK-NEXT: [[TMP10:%.*]] = bitcast <4 x float>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 4 dereferenceable(16) [[TMP9]], i8* noundef nonnull align 8 dereferenceable(16) [[TMP10]], i64 16, i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = alloca [4 x float], align 4
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[TMP6]], ptr noundef nonnull align 8 dereferenceable(16) [[B]], i64 16, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS3]]
; CHECK: no_alias3:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <4 x float>* [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP8]], [[COPY2]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi ptr [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP6]], [[COPY2]] ]
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
; CHECK: cols.header:
; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[NO_ALIAS3]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]
@@ -323,34 +301,30 @@ define void @multiply_alias_2x2(<4 x float>* %A, <4 x float>* %B, <4 x float>* %
; CHECK-NEXT: br label [[INNER_HEADER:%.*]]
; CHECK: inner.header:
; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP19:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
; CHECK-NEXT: br label [[INNER_BODY:%.*]]
; CHECK: inner.body:
-; CHECK-NEXT: [[TMP12:%.*]] = shl i64 [[INNER_IV]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr <4 x float>, <4 x float>* [[TMP5]], i64 0, i64 [[TMP13]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[TMP14]] to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[TMP14]], i64 2
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast float* [[VEC_GEP]] to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST8]], align 4
-; CHECK-NEXT: [[TMP15:%.*]] = shl i64 [[COLS_IV]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP15]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr <4 x float>, <4 x float>* [[TMP11]], i64 0, i64 [[TMP16]]
-; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast float* [[TMP17]] to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD12:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST11]], align 4
-; CHECK-NEXT: [[VEC_GEP13:%.*]] = getelementptr float, float* [[TMP17]], i64 2
-; CHECK-NEXT: [[VEC_CAST14:%.*]] = bitcast float* [[VEC_GEP13]] to <2 x float>*
-; CHECK-NEXT: [[COL_LOAD15:%.*]] = load <2 x float>, <2 x float>* [[VEC_CAST14]], align 4
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x float> [[COL_LOAD12]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP18:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT]], <2 x float> [[RESULT_VEC_0]])
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x float> [[COL_LOAD12]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP19]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD9]], <2 x float> [[SPLAT_SPLAT19]], <2 x float> [[TMP18]])
-; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <2 x float> [[COL_LOAD15]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT23]], <2 x float> [[RESULT_VEC_1]])
-; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <2 x float> [[COL_LOAD15]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP21]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD9]], <2 x float> [[SPLAT_SPLAT26]], <2 x float> [[TMP20]])
+; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[INNER_IV]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], [[ROWS_IV]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[TMP3]], i64 [[TMP9]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], [[INNER_IV]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[TMP7]], i64 [[TMP12]]
+; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x float>, ptr [[TMP13]], align 4
+; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr float, ptr [[TMP13]], i64 2
+; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x float>, ptr [[VEC_GEP10]], align 4
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT]], <2 x float> [[RESULT_VEC_0]])
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP15]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT15]], <2 x float> [[TMP14]])
+; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT19]], <2 x float> [[RESULT_VEC_1]])
+; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP17]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT22]], <2 x float> [[TMP16]])
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2
@@ -359,14 +333,12 @@ define void @multiply_alias_2x2(<4 x float>* %A, <4 x float>* %B, <4 x float>* %
; CHECK: rows.latch:
; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0
-; CHECK-NEXT: [[TMP22:%.*]] = shl i64 [[COLS_IV]], 1
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr <4 x float>, <4 x float>* [[C]], i64 0, i64 [[TMP23]]
-; CHECK-NEXT: [[VEC_CAST28:%.*]] = bitcast float* [[TMP24]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[TMP19]], <2 x float>* [[VEC_CAST28]], align 8
-; CHECK-NEXT: [[VEC_GEP29:%.*]] = getelementptr float, float* [[TMP24]], i64 2
-; CHECK-NEXT: [[VEC_CAST30:%.*]] = bitcast float* [[VEC_GEP29]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[TMP21]], <2 x float>* [[VEC_CAST30]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[COLS_IV]], 1
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr float, ptr [[C]], i64 [[TMP19]]
+; CHECK-NEXT: store <2 x float> [[TMP15]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[VEC_GEP23:%.*]] = getelementptr float, ptr [[TMP20]], i64 2
+; CHECK-NEXT: store <2 x float> [[TMP17]], ptr [[VEC_GEP23]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -378,12 +350,12 @@ define void @multiply_alias_2x2(<4 x float>* %A, <4 x float>* %B, <4 x float>* %
; First, check for aliasing at runtime, create non-aliasing copies if required.
entry:
- %a = load <4 x float>, <4 x float>* %A, align 8
- %b = load <4 x float>, <4 x float>* %B, align 8
+ %a = load <4 x float>, ptr %A, align 8
+ %b = load <4 x float>, ptr %B, align 8
%c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)
- store <4 x float> %c, <4 x float>* %C, align 8
+ store <4 x float> %c, ptr %C, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll
index 1ff6afd4aa7d2..2f4846a2840e5 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll
@@ -6,25 +6,20 @@
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "aarch64-apple-ios"
-define void @test(<6 x double> * %A, <6 x double> * %B, <9 x double>* %C, i1 %cond) {
+define void @test(ptr %A, ptr %B, ptr %C, i1 %cond) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST195:%.*]] = bitcast <6 x double>* [[A:%.*]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD196:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST195]], align 8
-; CHECK-NEXT: [[VEC_GEP197:%.*]] = getelementptr <6 x double>, <6 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST198:%.*]] = bitcast double* [[VEC_GEP197]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD199:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST198]], align 8
-; CHECK-NEXT: [[VEC_CAST200:%.*]] = bitcast <6 x double>* [[B:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD201:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST200]], align 8
-; CHECK-NEXT: [[VEC_GEP202:%.*]] = getelementptr <6 x double>, <6 x double>* [[B]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST203:%.*]] = bitcast double* [[VEC_GEP202]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD204:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST203]], align 8
-; CHECK-NEXT: [[VEC_GEP205:%.*]] = getelementptr <6 x double>, <6 x double>* [[B]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST206:%.*]] = bitcast double* [[VEC_GEP205]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD207:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST206]], align 8
-; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint <9 x double>* [[C:%.*]] to i64
+; CHECK-NEXT: [[COL_LOAD133:%.*]] = load <3 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP134:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD135:%.*]] = load <3 x double>, ptr [[VEC_GEP134]], align 8
+; CHECK-NEXT: [[COL_LOAD136:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP137:%.*]] = getelementptr double, ptr [[B]], i64 2
+; CHECK-NEXT: [[COL_LOAD138:%.*]] = load <2 x double>, ptr [[VEC_GEP137]], align 8
+; CHECK-NEXT: [[VEC_GEP139:%.*]] = getelementptr double, ptr [[B]], i64 4
+; CHECK-NEXT: [[COL_LOAD140:%.*]] = load <2 x double>, ptr [[VEC_GEP139]], align 8
+; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[C:%.*]] to i64
; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 72
-; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint <6 x double>* [[A]] to i64
+; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[A]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
; CHECK: alias_cont:
@@ -32,275 +27,218 @@ define void @test(<6 x double> * %A, <6 x double> * %B, <9 x double>* %C, i1 %co
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
; CHECK: copy:
-; CHECK-NEXT: [[TMP2:%.*]] = alloca [6 x double], align 8
-; CHECK-NEXT: [[BC:%.+]] = bitcast [6 x double]* [[TMP2]] to <6 x double>*
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast [6 x double]* [[TMP2]] to i8*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <6 x double>* [[A]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(48) [[TMP3]], i8* noundef nonnull align 8 dereferenceable(48) [[TMP4]], i64 48, i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [6 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(48) [[A]], i64 48, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS]]
; CHECK: no_alias:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <6 x double>* [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[BC]], [[COPY]] ]
-; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint <9 x double>* [[C]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint ptr [[C]] to i64
; CHECK-NEXT: [[STORE_END5:%.*]] = add nuw nsw i64 [[STORE_BEGIN4]], 72
-; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint <6 x double>* [[B]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
+; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
; CHECK: alias_cont1:
; CHECK-NEXT: [[LOAD_END7:%.*]] = add nuw nsw i64 [[LOAD_BEGIN6]], 48
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
; CHECK: copy2:
-; CHECK-NEXT: [[TMP8:%.*]] = alloca [6 x double], align 8
-; CHECK-NEXT: [[BC2:%.+]] = bitcast [6 x double]* [[TMP8]] to <6 x double>*
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast [6 x double]* [[TMP8]] to i8*
-; CHECK-NEXT: [[TMP10:%.*]] = bitcast <6 x double>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(48) [[TMP9]], i8* noundef nonnull align 8 dereferenceable(48) [[TMP10]], i64 48, i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = alloca [6 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP6]], ptr noundef nonnull align 8 dereferenceable(48) [[B]], i64 48, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS3]]
; CHECK: no_alias3:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <6 x double>* [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[BC2]], [[COPY2]] ]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <6 x double>* [[TMP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST8]], align 8
-; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast <6 x double>* [[TMP11]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD12:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST11]], align 8
-; CHECK-NEXT: [[VEC_GEP13:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP11]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST14:%.*]] = bitcast double* [[VEC_GEP13]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD15:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST14]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT18:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD9]], <2 x double> [[SPLAT_SPLAT18]], <2 x double> [[TMP12]])
-; CHECK-NEXT: [[SPLAT_SPLAT21:%.*]] = shufflevector <2 x double> [[COL_LOAD15]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT21]]
-; CHECK-NEXT: [[SPLAT_SPLAT24:%.*]] = shufflevector <2 x double> [[COL_LOAD15]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD9]], <2 x double> [[SPLAT_SPLAT24]], <2 x double> [[TMP14]])
-; CHECK-NEXT: [[VEC_CAST26:%.*]] = bitcast <9 x double>* [[C]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP13]], <2 x double>* [[VEC_CAST26]], align 8
-; CHECK-NEXT: [[VEC_GEP27:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST28:%.*]] = bitcast double* [[VEC_GEP27]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP15]], <2 x double>* [[VEC_CAST28]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST30:%.*]] = bitcast double* [[TMP16]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST30]], align 8
-; CHECK-NEXT: [[VEC_GEP32:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP5]], i64 0, i64 5
-; CHECK-NEXT: [[VEC_CAST33:%.*]] = bitcast double* [[VEC_GEP32]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD34:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST33]], align 8
-; CHECK-NEXT: [[VEC_CAST36:%.*]] = bitcast <6 x double>* [[TMP11]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD37:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST36]], align 8
-; CHECK-NEXT: [[VEC_GEP38:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP11]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST39:%.*]] = bitcast double* [[VEC_GEP38]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST39]], align 8
-; CHECK-NEXT: [[SPLAT_SPLATINSERT42:%.*]] = shufflevector <2 x double> [[COL_LOAD37]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = fmul contract <1 x double> [[COL_LOAD31]], [[SPLAT_SPLATINSERT42]]
-; CHECK-NEXT: [[SPLAT_SPLATINSERT45:%.*]] = shufflevector <2 x double> [[COL_LOAD37]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD34]], <1 x double> [[SPLAT_SPLATINSERT45]], <1 x double> [[TMP17]])
-; CHECK-NEXT: [[SPLAT_SPLATINSERT48:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = fmul contract <1 x double> [[COL_LOAD31]], [[SPLAT_SPLATINSERT48]]
-; CHECK-NEXT: [[SPLAT_SPLATINSERT51:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD34]], <1 x double> [[SPLAT_SPLATINSERT51]], <1 x double> [[TMP19]])
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST54:%.*]] = bitcast double* [[TMP21]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP18]], <1 x double>* [[VEC_CAST54]], align 8
-; CHECK-NEXT: [[VEC_GEP55:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 5
-; CHECK-NEXT: [[VEC_CAST56:%.*]] = bitcast double* [[VEC_GEP55]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP20]], <1 x double>* [[VEC_CAST56]], align 8
-; CHECK-NEXT: [[VEC_CAST58:%.*]] = bitcast <6 x double>* [[TMP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD59:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST58]], align 8
-; CHECK-NEXT: [[VEC_GEP60:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP5]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST61:%.*]] = bitcast double* [[VEC_GEP60]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD62:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST61]], align 8
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP11]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST64:%.*]] = bitcast double* [[TMP22]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD65:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST64]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD59]], [[SPLAT_SPLAT68]]
-; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP24:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT71]], <2 x double> [[TMP23]])
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST73:%.*]] = bitcast double* [[TMP25]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP24]], <2 x double>* [[VEC_CAST73]], align 8
-; CHECK-NEXT: [[TMP26:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST75:%.*]] = bitcast double* [[TMP26]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD76:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST75]], align 8
-; CHECK-NEXT: [[VEC_GEP77:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP5]], i64 0, i64 5
-; CHECK-NEXT: [[VEC_CAST78:%.*]] = bitcast double* [[VEC_GEP77]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD79:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST78]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP11]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST81:%.*]] = bitcast double* [[TMP27]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD82:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST81]], align 8
-; CHECK-NEXT: [[SPLAT_SPLATINSERT84:%.*]] = shufflevector <2 x double> [[COL_LOAD82]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP28:%.*]] = fmul contract <1 x double> [[COL_LOAD76]], [[SPLAT_SPLATINSERT84]]
-; CHECK-NEXT: [[SPLAT_SPLATINSERT87:%.*]] = shufflevector <2 x double> [[COL_LOAD82]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP29:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD79]], <1 x double> [[SPLAT_SPLATINSERT87]], <1 x double> [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST90:%.*]] = bitcast double* [[TMP30]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP29]], <1 x double>* [[VEC_CAST90]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = phi ptr [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP6]], [[COPY2]] ]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
+; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x double>, ptr [[VEC_GEP10]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP8]])
+; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT20]], <2 x double> [[TMP10]])
+; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[C]], align 8
+; CHECK-NEXT: [[VEC_GEP21:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT: store <2 x double> [[TMP11]], ptr [[VEC_GEP21]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD22:%.*]] = load <1 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[VEC_GEP23:%.*]] = getelementptr double, ptr [[TMP3]], i64 5
+; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[VEC_GEP23]], align 8
+; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[VEC_GEP26:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
+; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <2 x double>, ptr [[VEC_GEP26]], align 8
+; CHECK-NEXT: [[SPLAT_SPLATINSERT29:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT29]]
+; CHECK-NEXT: [[SPLAT_SPLATINSERT32:%.*]] = shufflevector <2 x double> [[COL_LOAD25]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT32]], <1 x double> [[TMP13]])
+; CHECK-NEXT: [[SPLAT_SPLATINSERT35:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD22]], [[SPLAT_SPLATINSERT35]]
+; CHECK-NEXT: [[SPLAT_SPLATINSERT38:%.*]] = shufflevector <2 x double> [[COL_LOAD27]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP16:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[SPLAT_SPLATINSERT38]], <1 x double> [[TMP15]])
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <1 x double> [[TMP14]], ptr [[TMP17]], align 8
+; CHECK-NEXT: [[VEC_GEP40:%.*]] = getelementptr double, ptr [[C]], i64 5
+; CHECK-NEXT: store <1 x double> [[TMP16]], ptr [[VEC_GEP40]], align 8
+; CHECK-NEXT: [[COL_LOAD41:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[VEC_GEP42:%.*]] = getelementptr double, ptr [[TMP3]], i64 3
+; CHECK-NEXT: [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[VEC_GEP42]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[COL_LOAD44:%.*]] = load <2 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = fmul contract <2 x double> [[COL_LOAD41]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD44]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD43]], <2 x double> [[SPLAT_SPLAT50]], <2 x double> [[TMP19]])
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT: store <2 x double> [[TMP20]], ptr [[TMP21]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <1 x double>, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[VEC_GEP52:%.*]] = getelementptr double, ptr [[TMP3]], i64 5
+; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <1 x double>, ptr [[VEC_GEP52]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[COL_LOAD54:%.*]] = load <2 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[SPLAT_SPLATINSERT56:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = fmul contract <1 x double> [[COL_LOAD51]], [[SPLAT_SPLATINSERT56]]
+; CHECK-NEXT: [[SPLAT_SPLATINSERT59:%.*]] = shufflevector <2 x double> [[COL_LOAD54]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD53]], <1 x double> [[SPLAT_SPLATINSERT59]], <1 x double> [[TMP24]])
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8
; CHECK-NEXT: br i1 [[COND:%.*]], label [[TRUE:%.*]], label [[FALSE:%.*]]
; CHECK: true:
-; CHECK-NEXT: [[TMP31:%.*]] = fadd contract <3 x double> [[COL_LOAD196]], [[COL_LOAD196]]
-; CHECK-NEXT: [[TMP32:%.*]] = fadd contract <3 x double> [[COL_LOAD199]], [[COL_LOAD199]]
-; CHECK-NEXT: [[VEC_CAST213:%.*]] = bitcast <6 x double>* [[A]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP31]], <3 x double>* [[VEC_CAST213]], align 8
-; CHECK-NEXT: [[VEC_GEP214:%.*]] = getelementptr <6 x double>, <6 x double>* [[A]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST215:%.*]] = bitcast double* [[VEC_GEP214]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP32]], <3 x double>* [[VEC_CAST215]], align 8
+; CHECK-NEXT: [[TMP27:%.*]] = fadd contract <3 x double> [[COL_LOAD133]], [[COL_LOAD133]]
+; CHECK-NEXT: [[TMP28:%.*]] = fadd contract <3 x double> [[COL_LOAD135]], [[COL_LOAD135]]
+; CHECK-NEXT: store <3 x double> [[TMP27]], ptr [[A]], align 8
+; CHECK-NEXT: [[VEC_GEP143:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: store <3 x double> [[TMP28]], ptr [[VEC_GEP143]], align 8
; CHECK-NEXT: br label [[END:%.*]]
; CHECK: false:
-; CHECK-NEXT: [[TMP33:%.*]] = fadd contract <2 x double> [[COL_LOAD201]], [[COL_LOAD201]]
-; CHECK-NEXT: [[TMP34:%.*]] = fadd contract <2 x double> [[COL_LOAD204]], [[COL_LOAD204]]
-; CHECK-NEXT: [[TMP35:%.*]] = fadd contract <2 x double> [[COL_LOAD207]], [[COL_LOAD207]]
-; CHECK-NEXT: [[VEC_CAST208:%.*]] = bitcast <6 x double>* [[B]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP33]], <2 x double>* [[VEC_CAST208]], align 8
-; CHECK-NEXT: [[VEC_GEP209:%.*]] = getelementptr <6 x double>, <6 x double>* [[B]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST210:%.*]] = bitcast double* [[VEC_GEP209]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP34]], <2 x double>* [[VEC_CAST210]], align 8
-; CHECK-NEXT: [[VEC_GEP211:%.*]] = getelementptr <6 x double>, <6 x double>* [[B]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST212:%.*]] = bitcast double* [[VEC_GEP211]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP35]], <2 x double>* [[VEC_CAST212]], align 8
+; CHECK-NEXT: [[TMP29:%.*]] = fadd contract <2 x double> [[COL_LOAD136]], [[COL_LOAD136]]
+; CHECK-NEXT: [[TMP30:%.*]] = fadd contract <2 x double> [[COL_LOAD138]], [[COL_LOAD138]]
+; CHECK-NEXT: [[TMP31:%.*]] = fadd contract <2 x double> [[COL_LOAD140]], [[COL_LOAD140]]
+; CHECK-NEXT: store <2 x double> [[TMP29]], ptr [[B]], align 8
+; CHECK-NEXT: [[VEC_GEP141:%.*]] = getelementptr double, ptr [[B]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP30]], ptr [[VEC_GEP141]], align 8
+; CHECK-NEXT: [[VEC_GEP142:%.*]] = getelementptr double, ptr [[B]], i64 4
+; CHECK-NEXT: store <2 x double> [[TMP31]], ptr [[VEC_GEP142]], align 8
; CHECK-NEXT: br label [[END]]
; CHECK: end:
-; CHECK-NEXT: [[STORE_BEGIN94:%.*]] = ptrtoint <9 x double>* [[C]] to i64
-; CHECK-NEXT: [[STORE_END95:%.*]] = add nuw nsw i64 [[STORE_BEGIN94]], 72
-; CHECK-NEXT: [[LOAD_BEGIN96:%.*]] = ptrtoint <6 x double>* [[A]] to i64
-; CHECK-NEXT: [[TMP36:%.*]] = icmp ugt i64 [[STORE_END95]], [[LOAD_BEGIN96]]
-; CHECK-NEXT: br i1 [[TMP36]], label [[ALIAS_CONT91:%.*]], label [[NO_ALIAS93:%.*]]
-; CHECK: alias_cont91:
-; CHECK-NEXT: [[LOAD_END97:%.*]] = add nuw nsw i64 [[LOAD_BEGIN96]], 48
-; CHECK-NEXT: [[TMP37:%.*]] = icmp ugt i64 [[LOAD_END97]], [[STORE_BEGIN94]]
-; CHECK-NEXT: br i1 [[TMP37]], label [[COPY92:%.*]], label [[NO_ALIAS93]]
-; CHECK: copy92:
+; CHECK-NEXT: [[STORE_BEGIN64:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT: [[STORE_END65:%.*]] = add nuw nsw i64 [[STORE_BEGIN64]], 72
+; CHECK-NEXT: [[LOAD_BEGIN66:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT: [[TMP32:%.*]] = icmp ugt i64 [[STORE_END65]], [[LOAD_BEGIN66]]
+; CHECK-NEXT: br i1 [[TMP32]], label [[ALIAS_CONT61:%.*]], label [[NO_ALIAS63:%.*]]
+; CHECK: alias_cont61:
+; CHECK-NEXT: [[LOAD_END67:%.*]] = add nuw nsw i64 [[LOAD_BEGIN66]], 48
+; CHECK-NEXT: [[TMP33:%.*]] = icmp ugt i64 [[LOAD_END67]], [[STORE_BEGIN64]]
+; CHECK-NEXT: br i1 [[TMP33]], label [[COPY62:%.*]], label [[NO_ALIAS63]]
+; CHECK: copy62:
+; CHECK-NEXT: [[TMP34:%.*]] = alloca [6 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP34]], ptr noundef nonnull align 8 dereferenceable(48) [[A]], i64 48, i1 false)
+; CHECK-NEXT: br label [[NO_ALIAS63]]
+; CHECK: no_alias63:
+; CHECK-NEXT: [[TMP35:%.*]] = phi ptr [ [[A]], [[END]] ], [ [[A]], [[ALIAS_CONT61]] ], [ [[TMP34]], [[COPY62]] ]
+; CHECK-NEXT: [[STORE_BEGIN71:%.*]] = ptrtoint ptr [[C]] to i64
+; CHECK-NEXT: [[STORE_END72:%.*]] = add nuw nsw i64 [[STORE_BEGIN71]], 72
+; CHECK-NEXT: [[LOAD_BEGIN73:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT: [[TMP36:%.*]] = icmp ugt i64 [[STORE_END72]], [[LOAD_BEGIN73]]
+; CHECK-NEXT: br i1 [[TMP36]], label [[ALIAS_CONT68:%.*]], label [[NO_ALIAS70:%.*]]
+; CHECK: alias_cont68:
+; CHECK-NEXT: [[LOAD_END74:%.*]] = add nuw nsw i64 [[LOAD_BEGIN73]], 48
+; CHECK-NEXT: [[TMP37:%.*]] = icmp ugt i64 [[LOAD_END74]], [[STORE_BEGIN71]]
+; CHECK-NEXT: br i1 [[TMP37]], label [[COPY69:%.*]], label [[NO_ALIAS70]]
+; CHECK: copy69:
; CHECK-NEXT: [[TMP38:%.*]] = alloca [6 x double], align 8
-; CHECK-NEXT: [[BC3:%.+]] = bitcast [6 x double]* [[TMP38]] to <6 x double>*
-; CHECK-NEXT: [[TMP39:%.*]] = bitcast [6 x double]* [[TMP38]] to i8*
-; CHECK-NEXT: [[TMP40:%.*]] = bitcast <6 x double>* [[A]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(48) [[TMP39]], i8* noundef nonnull align 8 dereferenceable(48) [[TMP40]], i64 48, i1 false)
-; CHECK-NEXT: br label [[NO_ALIAS93]]
-; CHECK: no_alias93:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <6 x double>* [ [[A]], [[END]] ], [ [[A]], [[ALIAS_CONT91]] ], [ [[BC3]], [[COPY92]] ]
-; CHECK-NEXT: [[STORE_BEGIN101:%.*]] = ptrtoint <9 x double>* [[C]] to i64
-; CHECK-NEXT: [[STORE_END102:%.*]] = add nuw nsw i64 [[STORE_BEGIN101]], 72
-; CHECK-NEXT: [[LOAD_BEGIN103:%.*]] = ptrtoint <6 x double>* [[B]] to i64
-; CHECK-NEXT: [[TMP42:%.*]] = icmp ugt i64 [[STORE_END102]], [[LOAD_BEGIN103]]
-; CHECK-NEXT: br i1 [[TMP42]], label [[ALIAS_CONT98:%.*]], label [[NO_ALIAS100:%.*]]
-; CHECK: alias_cont98:
-; CHECK-NEXT: [[LOAD_END104:%.*]] = add nuw nsw i64 [[LOAD_BEGIN103]], 48
-; CHECK-NEXT: [[TMP43:%.*]] = icmp ugt i64 [[LOAD_END104]], [[STORE_BEGIN101]]
-; CHECK-NEXT: br i1 [[TMP43]], label [[COPY99:%.*]], label [[NO_ALIAS100]]
-; CHECK: copy99:
-; CHECK-NEXT: [[TMP44:%.*]] = alloca [6 x double], align 8
-; CHECK-NEXT: [[BC4:%.+]] = bitcast [6 x double]* [[TMP44]] to <6 x double>*
-; CHECK-NEXT: [[TMP45:%.*]] = bitcast [6 x double]* [[TMP44]] to i8*
-; CHECK-NEXT: [[TMP46:%.*]] = bitcast <6 x double>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(48) [[TMP45]], i8* noundef nonnull align 8 dereferenceable(48) [[TMP46]], i64 48, i1 false)
-; CHECK-NEXT: br label [[NO_ALIAS100]]
-; CHECK: no_alias100:
-; CHECK-NEXT: [[TMP47:%.*]] = phi <6 x double>* [ [[B]], [[NO_ALIAS93]] ], [ [[B]], [[ALIAS_CONT98]] ], [ [[BC4]], [[COPY99]] ]
-; CHECK-NEXT: [[VEC_CAST106:%.*]] = bitcast <6 x double>* [[TMP41]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD107:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST106]], align 8
-; CHECK-NEXT: [[VEC_GEP108:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP41]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST109:%.*]] = bitcast double* [[VEC_GEP108]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD110:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST109]], align 8
-; CHECK-NEXT: [[VEC_CAST112:%.*]] = bitcast <6 x double>* [[TMP47]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD113:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST112]], align 8
-; CHECK-NEXT: [[VEC_GEP114:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP47]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST115:%.*]] = bitcast double* [[VEC_GEP114]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD116:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST115]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT119:%.*]] = shufflevector <2 x double> [[COL_LOAD113]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP48:%.*]] = fmul contract <2 x double> [[COL_LOAD107]], [[SPLAT_SPLAT119]]
-; CHECK-NEXT: [[SPLAT_SPLAT122:%.*]] = shufflevector <2 x double> [[COL_LOAD113]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP49:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD110]], <2 x double> [[SPLAT_SPLAT122]], <2 x double> [[TMP48]])
-; CHECK-NEXT: [[SPLAT_SPLAT125:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP50:%.*]] = fmul contract <2 x double> [[COL_LOAD107]], [[SPLAT_SPLAT125]]
-; CHECK-NEXT: [[SPLAT_SPLAT128:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP51:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD110]], <2 x double> [[SPLAT_SPLAT128]], <2 x double> [[TMP50]])
-; CHECK-NEXT: [[VEC_CAST130:%.*]] = bitcast <9 x double>* [[C]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP49]], <2 x double>* [[VEC_CAST130]], align 8
-; CHECK-NEXT: [[VEC_GEP131:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST132:%.*]] = bitcast double* [[VEC_GEP131]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP51]], <2 x double>* [[VEC_CAST132]], align 8
-; CHECK-NEXT: [[TMP52:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP41]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST134:%.*]] = bitcast double* [[TMP52]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD135:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST134]], align 8
-; CHECK-NEXT: [[VEC_GEP136:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP41]], i64 0, i64 5
-; CHECK-NEXT: [[VEC_CAST137:%.*]] = bitcast double* [[VEC_GEP136]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD138:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST137]], align 8
-; CHECK-NEXT: [[VEC_CAST140:%.*]] = bitcast <6 x double>* [[TMP47]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD141:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST140]], align 8
-; CHECK-NEXT: [[VEC_GEP142:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP47]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST143:%.*]] = bitcast double* [[VEC_GEP142]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD144:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST143]], align 8
-; CHECK-NEXT: [[SPLAT_SPLATINSERT146:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP53:%.*]] = fmul contract <1 x double> [[COL_LOAD135]], [[SPLAT_SPLATINSERT146]]
-; CHECK-NEXT: [[SPLAT_SPLATINSERT149:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP54:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD138]], <1 x double> [[SPLAT_SPLATINSERT149]], <1 x double> [[TMP53]])
-; CHECK-NEXT: [[SPLAT_SPLATINSERT152:%.*]] = shufflevector <2 x double> [[COL_LOAD144]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP55:%.*]] = fmul contract <1 x double> [[COL_LOAD135]], [[SPLAT_SPLATINSERT152]]
-; CHECK-NEXT: [[SPLAT_SPLATINSERT155:%.*]] = shufflevector <2 x double> [[COL_LOAD144]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP56:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD138]], <1 x double> [[SPLAT_SPLATINSERT155]], <1 x double> [[TMP55]])
-; CHECK-NEXT: [[TMP57:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST158:%.*]] = bitcast double* [[TMP57]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP54]], <1 x double>* [[VEC_CAST158]], align 8
-; CHECK-NEXT: [[VEC_GEP159:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 5
-; CHECK-NEXT: [[VEC_CAST160:%.*]] = bitcast double* [[VEC_GEP159]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP56]], <1 x double>* [[VEC_CAST160]], align 8
-; CHECK-NEXT: [[VEC_CAST162:%.*]] = bitcast <6 x double>* [[TMP41]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD163:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST162]], align 8
-; CHECK-NEXT: [[VEC_GEP164:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP41]], i64 0, i64 3
-; CHECK-NEXT: [[VEC_CAST165:%.*]] = bitcast double* [[VEC_GEP164]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD166:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST165]], align 8
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP47]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST168:%.*]] = bitcast double* [[TMP58]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD169:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST168]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT172:%.*]] = shufflevector <2 x double> [[COL_LOAD169]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP59:%.*]] = fmul contract <2 x double> [[COL_LOAD163]], [[SPLAT_SPLAT172]]
-; CHECK-NEXT: [[SPLAT_SPLAT175:%.*]] = shufflevector <2 x double> [[COL_LOAD169]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP60:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD166]], <2 x double> [[SPLAT_SPLAT175]], <2 x double> [[TMP59]])
-; CHECK-NEXT: [[TMP61:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST177:%.*]] = bitcast double* [[TMP61]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP60]], <2 x double>* [[VEC_CAST177]], align 8
-; CHECK-NEXT: [[TMP62:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP41]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST179:%.*]] = bitcast double* [[TMP62]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD180:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST179]], align 8
-; CHECK-NEXT: [[VEC_GEP181:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP41]], i64 0, i64 5
-; CHECK-NEXT: [[VEC_CAST182:%.*]] = bitcast double* [[VEC_GEP181]] to <1 x double>*
-; CHECK-NEXT: [[COL_LOAD183:%.*]] = load <1 x double>, <1 x double>* [[VEC_CAST182]], align 8
-; CHECK-NEXT: [[TMP63:%.*]] = getelementptr <6 x double>, <6 x double>* [[TMP47]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST185:%.*]] = bitcast double* [[TMP63]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD186:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST185]], align 8
-; CHECK-NEXT: [[SPLAT_SPLATINSERT188:%.*]] = shufflevector <2 x double> [[COL_LOAD186]], <2 x double> undef, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP64:%.*]] = fmul contract <1 x double> [[COL_LOAD180]], [[SPLAT_SPLATINSERT188]]
-; CHECK-NEXT: [[SPLAT_SPLATINSERT191:%.*]] = shufflevector <2 x double> [[COL_LOAD186]], <2 x double> undef, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP65:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD183]], <1 x double> [[SPLAT_SPLATINSERT191]], <1 x double> [[TMP64]])
-; CHECK-NEXT: [[TMP66:%.*]] = getelementptr <9 x double>, <9 x double>* [[C]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST194:%.*]] = bitcast double* [[TMP66]] to <1 x double>*
-; CHECK-NEXT: store <1 x double> [[TMP65]], <1 x double>* [[VEC_CAST194]], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(48) [[TMP38]], ptr noundef nonnull align 8 dereferenceable(48) [[B]], i64 48, i1 false)
+; CHECK-NEXT: br label [[NO_ALIAS70]]
+; CHECK: no_alias70:
+; CHECK-NEXT: [[TMP39:%.*]] = phi ptr [ [[B]], [[NO_ALIAS63]] ], [ [[B]], [[ALIAS_CONT68]] ], [ [[TMP38]], [[COPY69]] ]
+; CHECK-NEXT: [[COL_LOAD75:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
+; CHECK-NEXT: [[VEC_GEP76:%.*]] = getelementptr double, ptr [[TMP35]], i64 3
+; CHECK-NEXT: [[COL_LOAD77:%.*]] = load <2 x double>, ptr [[VEC_GEP76]], align 8
+; CHECK-NEXT: [[COL_LOAD78:%.*]] = load <2 x double>, ptr [[TMP39]], align 8
+; CHECK-NEXT: [[VEC_GEP79:%.*]] = getelementptr double, ptr [[TMP39]], i64 2
+; CHECK-NEXT: [[COL_LOAD80:%.*]] = load <2 x double>, ptr [[VEC_GEP79]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT83:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP40:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT: [[SPLAT_SPLAT86:%.*]] = shufflevector <2 x double> [[COL_LOAD78]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT86]], <2 x double> [[TMP40]])
+; CHECK-NEXT: [[SPLAT_SPLAT89:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP42:%.*]] = fmul contract <2 x double> [[COL_LOAD75]], [[SPLAT_SPLAT89]]
+; CHECK-NEXT: [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT92]], <2 x double> [[TMP42]])
+; CHECK-NEXT: store <2 x double> [[TMP41]], ptr [[C]], align 8
+; CHECK-NEXT: [[VEC_GEP93:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT: store <2 x double> [[TMP43]], ptr [[VEC_GEP93]], align 8
+; CHECK-NEXT: [[TMP44:%.*]] = getelementptr double, ptr [[TMP35]], i64 2
+; CHECK-NEXT: [[COL_LOAD94:%.*]] = load <1 x double>, ptr [[TMP44]], align 8
+; CHECK-NEXT: [[VEC_GEP95:%.*]] = getelementptr double, ptr [[TMP35]], i64 5
+; CHECK-NEXT: [[COL_LOAD96:%.*]] = load <1 x double>, ptr [[VEC_GEP95]], align 8
+; CHECK-NEXT: [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[TMP39]], align 8
+; CHECK-NEXT: [[VEC_GEP98:%.*]] = getelementptr double, ptr [[TMP39]], i64 2
+; CHECK-NEXT: [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[VEC_GEP98]], align 8
+; CHECK-NEXT: [[SPLAT_SPLATINSERT101:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP45:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT101]]
+; CHECK-NEXT: [[SPLAT_SPLATINSERT104:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP46:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT104]], <1 x double> [[TMP45]])
+; CHECK-NEXT: [[SPLAT_SPLATINSERT107:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP47:%.*]] = fmul contract <1 x double> [[COL_LOAD94]], [[SPLAT_SPLATINSERT107]]
+; CHECK-NEXT: [[SPLAT_SPLATINSERT110:%.*]] = shufflevector <2 x double> [[COL_LOAD99]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP48:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD96]], <1 x double> [[SPLAT_SPLATINSERT110]], <1 x double> [[TMP47]])
+; CHECK-NEXT: [[TMP49:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <1 x double> [[TMP46]], ptr [[TMP49]], align 8
+; CHECK-NEXT: [[VEC_GEP112:%.*]] = getelementptr double, ptr [[C]], i64 5
+; CHECK-NEXT: store <1 x double> [[TMP48]], ptr [[VEC_GEP112]], align 8
+; CHECK-NEXT: [[COL_LOAD113:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
+; CHECK-NEXT: [[VEC_GEP114:%.*]] = getelementptr double, ptr [[TMP35]], i64 3
+; CHECK-NEXT: [[COL_LOAD115:%.*]] = load <2 x double>, ptr [[VEC_GEP114]], align 8
+; CHECK-NEXT: [[TMP50:%.*]] = getelementptr double, ptr [[TMP39]], i64 4
+; CHECK-NEXT: [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP50]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT119:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP51:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT119]]
+; CHECK-NEXT: [[SPLAT_SPLAT122:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP52:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT122]], <2 x double> [[TMP51]])
+; CHECK-NEXT: [[TMP53:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT: store <2 x double> [[TMP52]], ptr [[TMP53]], align 8
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr double, ptr [[TMP35]], i64 2
+; CHECK-NEXT: [[COL_LOAD123:%.*]] = load <1 x double>, ptr [[TMP54]], align 8
+; CHECK-NEXT: [[VEC_GEP124:%.*]] = getelementptr double, ptr [[TMP35]], i64 5
+; CHECK-NEXT: [[COL_LOAD125:%.*]] = load <1 x double>, ptr [[VEC_GEP124]], align 8
+; CHECK-NEXT: [[TMP55:%.*]] = getelementptr double, ptr [[TMP39]], i64 4
+; CHECK-NEXT: [[COL_LOAD126:%.*]] = load <2 x double>, ptr [[TMP55]], align 8
+; CHECK-NEXT: [[SPLAT_SPLATINSERT128:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP56:%.*]] = fmul contract <1 x double> [[COL_LOAD123]], [[SPLAT_SPLATINSERT128]]
+; CHECK-NEXT: [[SPLAT_SPLATINSERT131:%.*]] = shufflevector <2 x double> [[COL_LOAD126]], <2 x double> undef, <1 x i32> <i32 1>
+; CHECK-NEXT: [[TMP57:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD125]], <1 x double> [[SPLAT_SPLATINSERT131]], <1 x double> [[TMP56]])
+; CHECK-NEXT: [[TMP58:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT: store <1 x double> [[TMP57]], ptr [[TMP58]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <6 x double>, <6 x double>* %A, align 8
- %b = load <6 x double>, <6 x double>* %B, align 8
+ %a = load <6 x double>, ptr %A, align 8
+ %b = load <6 x double>, ptr %B, align 8
%c = call <9 x double> @llvm.matrix.multiply(<6 x double> %a, <6 x double> %b, i32 3, i32 2, i32 3)
- store <9 x double> %c, <9 x double>* %C, align 8
+ store <9 x double> %c, ptr %C, align 8
br i1 %cond, label %true, label %false
true:
%a.add = fadd <6 x double> %a, %a
- store <6 x double> %a.add, <6 x double>* %A, align 8
+ store <6 x double> %a.add, ptr %A, align 8
br label %end
false:
%b.add = fadd <6 x double> %b, %b
- store <6 x double> %b.add, <6 x double>* %B, align 8
+ store <6 x double> %b.add, ptr %B, align 8
br label %end
end:
- %a.2 = load <6 x double>, <6 x double>* %A, align 8
- %b.2 = load <6 x double>, <6 x double>* %B, align 8
+ %a.2 = load <6 x double>, ptr %A, align 8
+ %b.2 = load <6 x double>, ptr %B, align 8
%c.2 = call <9 x double> @llvm.matrix.multiply(<6 x double> %a.2, <6 x double> %b.2, i32 3, i32 2, i32 3)
- store <9 x double> %c.2, <9 x double>* %C, align 8
+ store <9 x double> %c.2, ptr %C, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
index d915ece9f8ff0..8bc37f6eddf4c 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
@@ -6,7 +6,7 @@
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "aarch64-apple-ios"
-define void @multiply_all_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
+define void @multiply_all_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_all_volatile(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -22,58 +22,48 @@ define void @multiply_all_volatile(<4 x double>* noalias %A, <4 x double>* noali
; CHECK-NEXT: br label [[INNER_HEADER:%.*]]
; CHECK: inner.header:
; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
; CHECK-NEXT: br label [[INNER_BODY:%.*]]
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP2]], i64 [[TMP1]]
-; CHECK-NEXT: [[COL_CAST:%.*]] = bitcast double* [[TMP3]] to <4 x double>*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP4]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, double* [[TMP7]], i64 [[TMP6]]
-; CHECK-NEXT: [[COL_CAST3:%.*]] = bitcast double* [[TMP8]] to <4 x double>*
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[TMP9]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP9]], i64 2
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP10]], i32 0
+; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
-; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]])
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP15]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP14]], <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> poison, double [[TMP16]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
-; CHECK-NEXT: [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> poison, double [[TMP18]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP21]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]])
+; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]])
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2
@@ -82,17 +72,12 @@ define void @multiply_all_volatile(<4 x double>* noalias %A, <4 x double>* noali
; CHECK: rows.latch:
; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
-; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr double, double* [[TMP24]], i64 [[TMP23]]
-; CHECK-NEXT: [[COL_CAST20:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
-; CHECK-NEXT: [[TMP26:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast double* [[TMP26]] to <2 x double>*
-; CHECK-NEXT: store volatile <2 x double> [[TMP15]], <2 x double>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP26]], i64 2
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
-; CHECK-NEXT: store volatile <2 x double> [[TMP21]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]]
+; CHECK-NEXT: store volatile <2 x double> [[TMP11]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2
+; CHECK-NEXT: store volatile <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -104,17 +89,17 @@ define void @multiply_all_volatile(<4 x double>* noalias %A, <4 x double>* noali
entry:
- %a = load volatile <4 x double>, <4 x double>* %A, align 8
- %b = load volatile <4 x double>, <4 x double>* %B, align 8
+ %a = load volatile <4 x double>, ptr %A, align 8
+ %b = load volatile <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- store volatile <4 x double> %c, <4 x double>* %C, align 8
+ store volatile <4 x double> %c, ptr %C, align 8
ret void
}
-define void @multiply_load0_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
+define void @multiply_load0_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_load0_volatile(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -130,58 +115,48 @@ define void @multiply_load0_volatile(<4 x double>* noalias %A, <4 x double>* noa
; CHECK-NEXT: br label [[INNER_HEADER:%.*]]
; CHECK: inner.header:
; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
; CHECK-NEXT: br label [[INNER_BODY:%.*]]
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP2]], i64 [[TMP1]]
-; CHECK-NEXT: [[COL_CAST:%.*]] = bitcast double* [[TMP3]] to <4 x double>*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP4]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, double* [[TMP7]], i64 [[TMP6]]
-; CHECK-NEXT: [[COL_CAST3:%.*]] = bitcast double* [[TMP8]] to <4 x double>*
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[TMP9]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP9]], i64 2
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP10]], i32 0
+; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
-; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]])
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP15]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP14]], <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> poison, double [[TMP16]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
-; CHECK-NEXT: [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> poison, double [[TMP18]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP21]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]])
+; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]])
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2
@@ -190,17 +165,12 @@ define void @multiply_load0_volatile(<4 x double>* noalias %A, <4 x double>* noa
; CHECK: rows.latch:
; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
-; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr double, double* [[TMP24]], i64 [[TMP23]]
-; CHECK-NEXT: [[COL_CAST20:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
-; CHECK-NEXT: [[TMP26:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast double* [[TMP26]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP15]], <2 x double>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP26]], i64 2
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP21]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]]
+; CHECK-NEXT: store <2 x double> [[TMP11]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -212,16 +182,16 @@ define void @multiply_load0_volatile(<4 x double>* noalias %A, <4 x double>* noa
entry:
- %a = load volatile <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load volatile <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- store <4 x double> %c, <4 x double>* %C, align 8
+ store <4 x double> %c, ptr %C, align 8
ret void
}
-define void @multiply_load1_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
+define void @multiply_load1_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_load1_volatile(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -237,58 +207,48 @@ define void @multiply_load1_volatile(<4 x double>* noalias %A, <4 x double>* noa
; CHECK-NEXT: br label [[INNER_HEADER:%.*]]
; CHECK: inner.header:
; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
; CHECK-NEXT: br label [[INNER_BODY:%.*]]
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP2]], i64 [[TMP1]]
-; CHECK-NEXT: [[COL_CAST:%.*]] = bitcast double* [[TMP3]] to <4 x double>*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP4]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, double* [[TMP7]], i64 [[TMP6]]
-; CHECK-NEXT: [[COL_CAST3:%.*]] = bitcast double* [[TMP8]] to <4 x double>*
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[TMP9]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP9]], i64 2
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP10]], i32 0
+; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
-; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]])
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP15]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP14]], <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> poison, double [[TMP16]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
-; CHECK-NEXT: [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> poison, double [[TMP18]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP21]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]])
+; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]])
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2
@@ -297,17 +257,12 @@ define void @multiply_load1_volatile(<4 x double>* noalias %A, <4 x double>* noa
; CHECK: rows.latch:
; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
-; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr double, double* [[TMP24]], i64 [[TMP23]]
-; CHECK-NEXT: [[COL_CAST20:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
-; CHECK-NEXT: [[TMP26:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast double* [[TMP26]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP15]], <2 x double>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP26]], i64 2
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP21]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]]
+; CHECK-NEXT: store <2 x double> [[TMP11]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -319,16 +274,16 @@ define void @multiply_load1_volatile(<4 x double>* noalias %A, <4 x double>* noa
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load volatile <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load volatile <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- store <4 x double> %c, <4 x double>* %C, align 8
+ store <4 x double> %c, ptr %C, align 8
ret void
}
-define void @multiply_store_volatile(<4 x double>* noalias %A, <4 x double>* noalias %B, <4 x double>* noalias %C) {
+define void @multiply_store_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_store_volatile(
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[COLS_HEADER:%.*]]
@@ -344,58 +299,48 @@ define void @multiply_store_volatile(<4 x double>* noalias %A, <4 x double>* noa
; CHECK-NEXT: br label [[INNER_HEADER:%.*]]
; CHECK: inner.header:
; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]
-; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]
-; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP21:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]
+; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]
; CHECK-NEXT: br label [[INNER_BODY:%.*]]
; CHECK: inner.body:
; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, double* [[TMP2]], i64 [[TMP1]]
-; CHECK-NEXT: [[COL_CAST:%.*]] = bitcast double* [[TMP3]] to <4 x double>*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x double>* [[COL_CAST]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP4]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], [[INNER_IV]]
-; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, double* [[TMP7]], i64 [[TMP6]]
-; CHECK-NEXT: [[COL_CAST3:%.*]] = bitcast double* [[TMP8]] to <4 x double>*
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x double>* [[COL_CAST3]] to double*
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[TMP9]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP9]], i64 2
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]]
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP10]], i32 0
+; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK9]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
-; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]])
+; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]])
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[TMP11]])
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP15]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP14]], <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[BLOCK14:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT15:%.*]] = insertelement <2 x double> poison, double [[TMP16]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT15]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK14]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[BLOCK13]])
-; CHECK-NEXT: [[BLOCK17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[COL_LOAD8]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT18:%.*]] = insertelement <2 x double> poison, double [[TMP18]], i32 0
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT18]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK17]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP17]])
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[TMP21]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]])
+; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i32 0
+; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]])
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br label [[INNER_LATCH]]
; CHECK: inner.latch:
; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2
@@ -404,17 +349,12 @@ define void @multiply_store_volatile(<4 x double>* noalias %A, <4 x double>* noa
; CHECK: rows.latch:
; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2
; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2
-; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[COLS_IV]], 2
-; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[ROWS_IV]]
-; CHECK-NEXT: [[TMP24:%.*]] = bitcast <4 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr double, double* [[TMP24]], i64 [[TMP23]]
-; CHECK-NEXT: [[COL_CAST20:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
-; CHECK-NEXT: [[TMP26:%.*]] = bitcast <4 x double>* [[COL_CAST20]] to double*
-; CHECK-NEXT: [[VEC_CAST21:%.*]] = bitcast double* [[TMP26]] to <2 x double>*
-; CHECK-NEXT: store volatile <2 x double> [[TMP15]], <2 x double>* [[VEC_CAST21]], align 8
-; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr double, double* [[TMP26]], i64 2
-; CHECK-NEXT: [[VEC_CAST23:%.*]] = bitcast double* [[VEC_GEP22]] to <2 x double>*
-; CHECK-NEXT: store volatile <2 x double> [[TMP21]], <2 x double>* [[VEC_CAST23]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]]
+; CHECK-NEXT: store volatile <2 x double> [[TMP11]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2
+; CHECK-NEXT: store volatile <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8
; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]]
; CHECK: cols.latch:
; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2
@@ -425,12 +365,12 @@ define void @multiply_store_volatile(<4 x double>* noalias %A, <4 x double>* noa
;
entry:
- %a = load <4 x double>, <4 x double>* %A, align 8
- %b = load <4 x double>, <4 x double>* %B, align 8
+ %a = load <4 x double>, ptr %A, align 8
+ %b = load <4 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
- store volatile <4 x double> %c, <4 x double>* %C, align 8
+ store volatile <4 x double> %c, ptr %C, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
index 31340f9db56c0..3dd04b23f272d 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused.ll
@@ -8,12 +8,12 @@ target triple = "aarch64-apple-ios"
; Test tiling without generating explicit loops.
-define void @multiply(<16 x double> * %A, <16 x double> * %B, <16 x double>* %C) {
+define void @multiply(ptr %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint <16 x double>* [[C:%.*]] to i64
+; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[C:%.*]] to i64
; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 128
-; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint <16 x double>* [[A:%.*]] to i64
+; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[A:%.*]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]
; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]
; CHECK: alias_cont:
@@ -21,209 +21,165 @@ define void @multiply(<16 x double> * %A, <16 x double> * %B, <16 x double>* %C)
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]
; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]
; CHECK: copy:
-; CHECK-NEXT: [[TMP2:%.*]] = alloca <16 x double>, align 8
-; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x double>* [[TMP2]] to i8*
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x double>* [[A]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(128) [[TMP3]], i8* noundef nonnull align 8 dereferenceable(128) [[TMP4]], i64 128, i1 false)
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [16 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(128) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(128) [[A]], i64 128, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS]]
; CHECK: no_alias:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <16 x double>* [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
-; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint <16 x double>* [[C]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]
+; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint ptr [[C]] to i64
; CHECK-NEXT: [[STORE_END5:%.*]] = add nuw nsw i64 [[STORE_BEGIN4]], 128
-; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint <16 x double>* [[B:%.*]] to i64
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
-; CHECK-NEXT: br i1 [[TMP6]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
+; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint ptr [[B:%.*]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]
+; CHECK-NEXT: br i1 [[TMP4]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]
; CHECK: alias_cont1:
; CHECK-NEXT: [[LOAD_END7:%.*]] = add nuw nsw i64 [[LOAD_BEGIN6]], 128
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
-; CHECK-NEXT: br i1 [[TMP7]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]
+; CHECK-NEXT: br i1 [[TMP5]], label [[COPY2:%.*]], label [[NO_ALIAS3]]
; CHECK: copy2:
-; CHECK-NEXT: [[TMP8:%.*]] = alloca <16 x double>, align 8
-; CHECK-NEXT: [[TMP9:%.*]] = bitcast <16 x double>* [[TMP8]] to i8*
-; CHECK-NEXT: [[TMP10:%.*]] = bitcast <16 x double>* [[B]] to i8*
-; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* noundef nonnull align 8 dereferenceable(128) [[TMP9]], i8* noundef nonnull align 8 dereferenceable(128) [[TMP10]], i64 128, i1 false)
+; CHECK-NEXT: [[TMP6:%.*]] = alloca [16 x double], align 8
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(128) [[TMP6]], ptr noundef nonnull align 8 dereferenceable(128) [[B]], i64 128, i1 false)
; CHECK-NEXT: br label [[NO_ALIAS3]]
; CHECK: no_alias3:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <16 x double>* [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP8]], [[COPY2]] ]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <16 x double>* [[TMP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST8]], align 8
-; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast <16 x double>* [[TMP11]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD12:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST11]], align 8
-; CHECK-NEXT: [[VEC_GEP13:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST14:%.*]] = bitcast double* [[VEC_GEP13]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD15:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST14]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP12:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT18:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD9]], <2 x double> [[SPLAT_SPLAT18]], <2 x double> [[TMP12]])
-; CHECK-NEXT: [[SPLAT_SPLAT21:%.*]] = shufflevector <2 x double> [[COL_LOAD15]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT21]]
-; CHECK-NEXT: [[SPLAT_SPLAT24:%.*]] = shufflevector <2 x double> [[COL_LOAD15]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD9]], <2 x double> [[SPLAT_SPLAT24]], <2 x double> [[TMP14]])
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST26:%.*]] = bitcast double* [[TMP16]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD27:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST26]], align 8
-; CHECK-NEXT: [[VEC_GEP28:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST29:%.*]] = bitcast double* [[VEC_GEP28]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST29]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST32:%.*]] = bitcast double* [[TMP17]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD33:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST32]], align 8
-; CHECK-NEXT: [[VEC_GEP34:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST35:%.*]] = bitcast double* [[VEC_GEP34]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST35]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <2 x double> [[COL_LOAD33]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP18:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD27]], <2 x double> [[SPLAT_SPLAT40]], <2 x double> [[TMP13]])
-; CHECK-NEXT: [[SPLAT_SPLAT43:%.*]] = shufflevector <2 x double> [[COL_LOAD33]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD30]], <2 x double> [[SPLAT_SPLAT43]], <2 x double> [[TMP18]])
-; CHECK-NEXT: [[SPLAT_SPLAT47:%.*]] = shufflevector <2 x double> [[COL_LOAD36]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD27]], <2 x double> [[SPLAT_SPLAT47]], <2 x double> [[TMP15]])
-; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD36]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP21:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD30]], <2 x double> [[SPLAT_SPLAT50]], <2 x double> [[TMP20]])
-; CHECK-NEXT: [[VEC_CAST52:%.*]] = bitcast <16 x double>* [[C]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP19]], <2 x double>* [[VEC_CAST52]], align 8
-; CHECK-NEXT: [[VEC_GEP53:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST54:%.*]] = bitcast double* [[VEC_GEP53]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP21]], <2 x double>* [[VEC_CAST54]], align 8
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST56:%.*]] = bitcast double* [[TMP22]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD57:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST56]], align 8
-; CHECK-NEXT: [[VEC_GEP58:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST59:%.*]] = bitcast double* [[VEC_GEP58]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD60:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST59]], align 8
-; CHECK-NEXT: [[VEC_CAST62:%.*]] = bitcast <16 x double>* [[TMP11]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD63:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST62]], align 8
-; CHECK-NEXT: [[VEC_GEP64:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST65:%.*]] = bitcast double* [[VEC_GEP64]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD66:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST65]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = phi ptr [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP6]], [[COPY2]] ]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP3]], i64 4
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x double>, ptr [[VEC_GEP10]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
+; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD9]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP8]])
+; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <2 x double> [[COL_LOAD11]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP11:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT20]], <2 x double> [[TMP10]])
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[COL_LOAD21:%.*]] = load <2 x double>, ptr [[TMP12]], align 8
+; CHECK-NEXT: [[VEC_GEP22:%.*]] = getelementptr double, ptr [[TMP3]], i64 12
+; CHECK-NEXT: [[COL_LOAD23:%.*]] = load <2 x double>, ptr [[VEC_GEP22]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
+; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <2 x double>, ptr [[TMP13]], align 8
+; CHECK-NEXT: [[VEC_GEP25:%.*]] = getelementptr double, ptr [[TMP7]], i64 6
+; CHECK-NEXT: [[COL_LOAD26:%.*]] = load <2 x double>, ptr [[VEC_GEP25]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD21]], <2 x double> [[SPLAT_SPLAT30]], <2 x double> [[TMP9]])
+; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD24]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP14]])
+; CHECK-NEXT: [[SPLAT_SPLAT37:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD21]], <2 x double> [[SPLAT_SPLAT37]], <2 x double> [[TMP11]])
+; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT40]], <2 x double> [[TMP16]])
+; CHECK-NEXT: store <2 x double> [[TMP15]], ptr [[C]], align 8
+; CHECK-NEXT: [[VEC_GEP41:%.*]] = getelementptr double, ptr [[C]], i64 4
+; CHECK-NEXT: store <2 x double> [[TMP17]], ptr [[VEC_GEP41]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD42:%.*]] = load <2 x double>, ptr [[TMP18]], align 8
+; CHECK-NEXT: [[VEC_GEP43:%.*]] = getelementptr double, ptr [[TMP3]], i64 6
+; CHECK-NEXT: [[COL_LOAD44:%.*]] = load <2 x double>, ptr [[VEC_GEP43]], align 8
+; CHECK-NEXT: [[COL_LOAD45:%.*]] = load <2 x double>, ptr [[TMP7]], align 8
+; CHECK-NEXT: [[VEC_GEP46:%.*]] = getelementptr double, ptr [[TMP7]], i64 4
+; CHECK-NEXT: [[COL_LOAD47:%.*]] = load <2 x double>, ptr [[VEC_GEP46]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT50:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT: [[SPLAT_SPLAT53:%.*]] = shufflevector <2 x double> [[COL_LOAD45]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD44]], <2 x double> [[SPLAT_SPLAT53]], <2 x double> [[TMP19]])
+; CHECK-NEXT: [[SPLAT_SPLAT56:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <2 x double> [[COL_LOAD42]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT: [[SPLAT_SPLAT59:%.*]] = shufflevector <2 x double> [[COL_LOAD47]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP22:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD44]], <2 x double> [[SPLAT_SPLAT59]], <2 x double> [[TMP21]])
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP3]], i64 10
+; CHECK-NEXT: [[COL_LOAD60:%.*]] = load <2 x double>, ptr [[TMP23]], align 8
+; CHECK-NEXT: [[VEC_GEP61:%.*]] = getelementptr double, ptr [[TMP3]], i64 14
+; CHECK-NEXT: [[COL_LOAD62:%.*]] = load <2 x double>, ptr [[VEC_GEP61]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr double, ptr [[TMP7]], i64 2
+; CHECK-NEXT: [[COL_LOAD63:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT: [[VEC_GEP64:%.*]] = getelementptr double, ptr [[TMP7]], i64 6
+; CHECK-NEXT: [[COL_LOAD65:%.*]] = load <2 x double>, ptr [[VEC_GEP64]], align 8
; CHECK-NEXT: [[SPLAT_SPLAT69:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD57]], [[SPLAT_SPLAT69]]
+; CHECK-NEXT: [[TMP25:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT69]], <2 x double> [[TMP20]])
; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD63]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP24:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP23]])
-; CHECK-NEXT: [[SPLAT_SPLAT75:%.*]] = shufflevector <2 x double> [[COL_LOAD66]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD57]], [[SPLAT_SPLAT75]]
-; CHECK-NEXT: [[SPLAT_SPLAT78:%.*]] = shufflevector <2 x double> [[COL_LOAD66]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT78]], <2 x double> [[TMP25]])
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST80:%.*]] = bitcast double* [[TMP27]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD81:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST80]], align 8
-; CHECK-NEXT: [[VEC_GEP82:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST83:%.*]] = bitcast double* [[VEC_GEP82]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD84:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST83]], align 8
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST86:%.*]] = bitcast double* [[TMP28]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD87:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST86]], align 8
-; CHECK-NEXT: [[VEC_GEP88:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST89:%.*]] = bitcast double* [[VEC_GEP88]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD90:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST89]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT94:%.*]] = shufflevector <2 x double> [[COL_LOAD87]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP29:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD81]], <2 x double> [[SPLAT_SPLAT94]], <2 x double> [[TMP24]])
-; CHECK-NEXT: [[SPLAT_SPLAT97:%.*]] = shufflevector <2 x double> [[COL_LOAD87]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP30:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD84]], <2 x double> [[SPLAT_SPLAT97]], <2 x double> [[TMP29]])
-; CHECK-NEXT: [[SPLAT_SPLAT101:%.*]] = shufflevector <2 x double> [[COL_LOAD90]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD81]], <2 x double> [[SPLAT_SPLAT101]], <2 x double> [[TMP26]])
-; CHECK-NEXT: [[SPLAT_SPLAT104:%.*]] = shufflevector <2 x double> [[COL_LOAD90]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD84]], <2 x double> [[SPLAT_SPLAT104]], <2 x double> [[TMP31]])
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST106:%.*]] = bitcast double* [[TMP33]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP30]], <2 x double>* [[VEC_CAST106]], align 8
-; CHECK-NEXT: [[VEC_GEP107:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST108:%.*]] = bitcast double* [[VEC_GEP107]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP32]], <2 x double>* [[VEC_CAST108]], align 8
-; CHECK-NEXT: [[VEC_CAST110:%.*]] = bitcast <16 x double>* [[TMP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD111:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST110]], align 8
-; CHECK-NEXT: [[VEC_GEP112:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST113:%.*]] = bitcast double* [[VEC_GEP112]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD114:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST113]], align 8
-; CHECK-NEXT: [[TMP34:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST116:%.*]] = bitcast double* [[TMP34]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD117:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST116]], align 8
-; CHECK-NEXT: [[VEC_GEP118:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST119:%.*]] = bitcast double* [[VEC_GEP118]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD120:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST119]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT123:%.*]] = shufflevector <2 x double> [[COL_LOAD117]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP35:%.*]] = fmul contract <2 x double> [[COL_LOAD111]], [[SPLAT_SPLAT123]]
-; CHECK-NEXT: [[SPLAT_SPLAT126:%.*]] = shufflevector <2 x double> [[COL_LOAD117]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP36:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD114]], <2 x double> [[SPLAT_SPLAT126]], <2 x double> [[TMP35]])
-; CHECK-NEXT: [[SPLAT_SPLAT129:%.*]] = shufflevector <2 x double> [[COL_LOAD120]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP37:%.*]] = fmul contract <2 x double> [[COL_LOAD111]], [[SPLAT_SPLAT129]]
-; CHECK-NEXT: [[SPLAT_SPLAT132:%.*]] = shufflevector <2 x double> [[COL_LOAD120]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP38:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD114]], <2 x double> [[SPLAT_SPLAT132]], <2 x double> [[TMP37]])
-; CHECK-NEXT: [[TMP39:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST134:%.*]] = bitcast double* [[TMP39]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD135:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST134]], align 8
-; CHECK-NEXT: [[VEC_GEP136:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST137:%.*]] = bitcast double* [[VEC_GEP136]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD138:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST137]], align 8
-; CHECK-NEXT: [[TMP40:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST140:%.*]] = bitcast double* [[TMP40]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD141:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST140]], align 8
-; CHECK-NEXT: [[VEC_GEP142:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST143:%.*]] = bitcast double* [[VEC_GEP142]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD144:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST143]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT148:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD135]], <2 x double> [[SPLAT_SPLAT148]], <2 x double> [[TMP36]])
-; CHECK-NEXT: [[SPLAT_SPLAT151:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP42:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT151]], <2 x double> [[TMP41]])
-; CHECK-NEXT: [[SPLAT_SPLAT155:%.*]] = shufflevector <2 x double> [[COL_LOAD144]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD135]], <2 x double> [[SPLAT_SPLAT155]], <2 x double> [[TMP38]])
-; CHECK-NEXT: [[SPLAT_SPLAT158:%.*]] = shufflevector <2 x double> [[COL_LOAD144]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP44:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT158]], <2 x double> [[TMP43]])
-; CHECK-NEXT: [[TMP45:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST160:%.*]] = bitcast double* [[TMP45]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP42]], <2 x double>* [[VEC_CAST160]], align 8
-; CHECK-NEXT: [[VEC_GEP161:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST162:%.*]] = bitcast double* [[VEC_GEP161]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP44]], <2 x double>* [[VEC_CAST162]], align 8
-; CHECK-NEXT: [[TMP46:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST164:%.*]] = bitcast double* [[TMP46]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD165:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST164]], align 8
-; CHECK-NEXT: [[VEC_GEP166:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST167:%.*]] = bitcast double* [[VEC_GEP166]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD168:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST167]], align 8
-; CHECK-NEXT: [[TMP47:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST170:%.*]] = bitcast double* [[TMP47]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD171:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST170]], align 8
-; CHECK-NEXT: [[VEC_GEP172:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST173:%.*]] = bitcast double* [[VEC_GEP172]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD174:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST173]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT177:%.*]] = shufflevector <2 x double> [[COL_LOAD171]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP48:%.*]] = fmul contract <2 x double> [[COL_LOAD165]], [[SPLAT_SPLAT177]]
-; CHECK-NEXT: [[SPLAT_SPLAT180:%.*]] = shufflevector <2 x double> [[COL_LOAD171]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP49:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD168]], <2 x double> [[SPLAT_SPLAT180]], <2 x double> [[TMP48]])
-; CHECK-NEXT: [[SPLAT_SPLAT183:%.*]] = shufflevector <2 x double> [[COL_LOAD174]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP50:%.*]] = fmul contract <2 x double> [[COL_LOAD165]], [[SPLAT_SPLAT183]]
-; CHECK-NEXT: [[SPLAT_SPLAT186:%.*]] = shufflevector <2 x double> [[COL_LOAD174]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP51:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD168]], <2 x double> [[SPLAT_SPLAT186]], <2 x double> [[TMP50]])
-; CHECK-NEXT: [[TMP52:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST188:%.*]] = bitcast double* [[TMP52]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD189:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST188]], align 8
-; CHECK-NEXT: [[VEC_GEP190:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP5]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST191:%.*]] = bitcast double* [[VEC_GEP190]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD192:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST191]], align 8
-; CHECK-NEXT: [[TMP53:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST194:%.*]] = bitcast double* [[TMP53]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD195:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST194]], align 8
-; CHECK-NEXT: [[VEC_GEP196:%.*]] = getelementptr <16 x double>, <16 x double>* [[TMP11]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST197:%.*]] = bitcast double* [[VEC_GEP196]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD198:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST197]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT202:%.*]] = shufflevector <2 x double> [[COL_LOAD195]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP54:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD189]], <2 x double> [[SPLAT_SPLAT202]], <2 x double> [[TMP49]])
-; CHECK-NEXT: [[SPLAT_SPLAT205:%.*]] = shufflevector <2 x double> [[COL_LOAD195]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP55:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD192]], <2 x double> [[SPLAT_SPLAT205]], <2 x double> [[TMP54]])
-; CHECK-NEXT: [[SPLAT_SPLAT209:%.*]] = shufflevector <2 x double> [[COL_LOAD198]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP56:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD189]], <2 x double> [[SPLAT_SPLAT209]], <2 x double> [[TMP51]])
-; CHECK-NEXT: [[SPLAT_SPLAT212:%.*]] = shufflevector <2 x double> [[COL_LOAD198]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP57:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD192]], <2 x double> [[SPLAT_SPLAT212]], <2 x double> [[TMP56]])
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST214:%.*]] = bitcast double* [[TMP58]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP55]], <2 x double>* [[VEC_CAST214]], align 8
-; CHECK-NEXT: [[VEC_GEP215:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST216:%.*]] = bitcast double* [[VEC_GEP215]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP57]], <2 x double>* [[VEC_CAST216]], align 8
+; CHECK-NEXT: [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP25]])
+; CHECK-NEXT: [[SPLAT_SPLAT76:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD60]], <2 x double> [[SPLAT_SPLAT76]], <2 x double> [[TMP22]])
+; CHECK-NEXT: [[SPLAT_SPLAT79:%.*]] = shufflevector <2 x double> [[COL_LOAD65]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP28:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD62]], <2 x double> [[SPLAT_SPLAT79]], <2 x double> [[TMP27]])
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP26]], ptr [[TMP29]], align 8
+; CHECK-NEXT: [[VEC_GEP80:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT: store <2 x double> [[TMP28]], ptr [[VEC_GEP80]], align 8
+; CHECK-NEXT: [[COL_LOAD81:%.*]] = load <2 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[VEC_GEP82:%.*]] = getelementptr double, ptr [[TMP3]], i64 4
+; CHECK-NEXT: [[COL_LOAD83:%.*]] = load <2 x double>, ptr [[VEC_GEP82]], align 8
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr double, ptr [[TMP7]], i64 8
+; CHECK-NEXT: [[COL_LOAD84:%.*]] = load <2 x double>, ptr [[TMP30]], align 8
+; CHECK-NEXT: [[VEC_GEP85:%.*]] = getelementptr double, ptr [[TMP7]], i64 12
+; CHECK-NEXT: [[COL_LOAD86:%.*]] = load <2 x double>, ptr [[VEC_GEP85]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT89:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT89]]
+; CHECK-NEXT: [[SPLAT_SPLAT92:%.*]] = shufflevector <2 x double> [[COL_LOAD84]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD83]], <2 x double> [[SPLAT_SPLAT92]], <2 x double> [[TMP31]])
+; CHECK-NEXT: [[SPLAT_SPLAT95:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = fmul contract <2 x double> [[COL_LOAD81]], [[SPLAT_SPLAT95]]
+; CHECK-NEXT: [[SPLAT_SPLAT98:%.*]] = shufflevector <2 x double> [[COL_LOAD86]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP34:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD83]], <2 x double> [[SPLAT_SPLAT98]], <2 x double> [[TMP33]])
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr double, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[COL_LOAD99:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
+; CHECK-NEXT: [[VEC_GEP100:%.*]] = getelementptr double, ptr [[TMP3]], i64 12
+; CHECK-NEXT: [[COL_LOAD101:%.*]] = load <2 x double>, ptr [[VEC_GEP100]], align 8
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr double, ptr [[TMP7]], i64 10
+; CHECK-NEXT: [[COL_LOAD102:%.*]] = load <2 x double>, ptr [[TMP36]], align 8
+; CHECK-NEXT: [[VEC_GEP103:%.*]] = getelementptr double, ptr [[TMP7]], i64 14
+; CHECK-NEXT: [[COL_LOAD104:%.*]] = load <2 x double>, ptr [[VEC_GEP103]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT108:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD99]], <2 x double> [[SPLAT_SPLAT108]], <2 x double> [[TMP32]])
+; CHECK-NEXT: [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD102]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP38:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD101]], <2 x double> [[SPLAT_SPLAT111]], <2 x double> [[TMP37]])
+; CHECK-NEXT: [[SPLAT_SPLAT115:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD99]], <2 x double> [[SPLAT_SPLAT115]], <2 x double> [[TMP34]])
+; CHECK-NEXT: [[SPLAT_SPLAT118:%.*]] = shufflevector <2 x double> [[COL_LOAD104]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP40:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD101]], <2 x double> [[SPLAT_SPLAT118]], <2 x double> [[TMP39]])
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT: store <2 x double> [[TMP38]], ptr [[TMP41]], align 8
+; CHECK-NEXT: [[VEC_GEP119:%.*]] = getelementptr double, ptr [[C]], i64 12
+; CHECK-NEXT: store <2 x double> [[TMP40]], ptr [[VEC_GEP119]], align 8
+; CHECK-NEXT: [[TMP42:%.*]] = getelementptr double, ptr [[TMP3]], i64 2
+; CHECK-NEXT: [[COL_LOAD120:%.*]] = load <2 x double>, ptr [[TMP42]], align 8
+; CHECK-NEXT: [[VEC_GEP121:%.*]] = getelementptr double, ptr [[TMP3]], i64 6
+; CHECK-NEXT: [[COL_LOAD122:%.*]] = load <2 x double>, ptr [[VEC_GEP121]], align 8
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr double, ptr [[TMP7]], i64 8
+; CHECK-NEXT: [[COL_LOAD123:%.*]] = load <2 x double>, ptr [[TMP43]], align 8
+; CHECK-NEXT: [[VEC_GEP124:%.*]] = getelementptr double, ptr [[TMP7]], i64 12
+; CHECK-NEXT: [[COL_LOAD125:%.*]] = load <2 x double>, ptr [[VEC_GEP124]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT128:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT128]]
+; CHECK-NEXT: [[SPLAT_SPLAT131:%.*]] = shufflevector <2 x double> [[COL_LOAD123]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP45:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD122]], <2 x double> [[SPLAT_SPLAT131]], <2 x double> [[TMP44]])
+; CHECK-NEXT: [[SPLAT_SPLAT134:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP46:%.*]] = fmul contract <2 x double> [[COL_LOAD120]], [[SPLAT_SPLAT134]]
+; CHECK-NEXT: [[SPLAT_SPLAT137:%.*]] = shufflevector <2 x double> [[COL_LOAD125]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP47:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD122]], <2 x double> [[SPLAT_SPLAT137]], <2 x double> [[TMP46]])
+; CHECK-NEXT: [[TMP48:%.*]] = getelementptr double, ptr [[TMP3]], i64 10
+; CHECK-NEXT: [[COL_LOAD138:%.*]] = load <2 x double>, ptr [[TMP48]], align 8
+; CHECK-NEXT: [[VEC_GEP139:%.*]] = getelementptr double, ptr [[TMP3]], i64 14
+; CHECK-NEXT: [[COL_LOAD140:%.*]] = load <2 x double>, ptr [[VEC_GEP139]], align 8
+; CHECK-NEXT: [[TMP49:%.*]] = getelementptr double, ptr [[TMP7]], i64 10
+; CHECK-NEXT: [[COL_LOAD141:%.*]] = load <2 x double>, ptr [[TMP49]], align 8
+; CHECK-NEXT: [[VEC_GEP142:%.*]] = getelementptr double, ptr [[TMP7]], i64 14
+; CHECK-NEXT: [[COL_LOAD143:%.*]] = load <2 x double>, ptr [[VEC_GEP142]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT147:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP50:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT147]], <2 x double> [[TMP45]])
+; CHECK-NEXT: [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD141]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP51:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD140]], <2 x double> [[SPLAT_SPLAT150]], <2 x double> [[TMP50]])
+; CHECK-NEXT: [[SPLAT_SPLAT154:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP52:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD138]], <2 x double> [[SPLAT_SPLAT154]], <2 x double> [[TMP47]])
+; CHECK-NEXT: [[SPLAT_SPLAT157:%.*]] = shufflevector <2 x double> [[COL_LOAD143]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP53:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD140]], <2 x double> [[SPLAT_SPLAT157]], <2 x double> [[TMP52]])
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr double, ptr [[C]], i64 10
+; CHECK-NEXT: store <2 x double> [[TMP51]], ptr [[TMP54]], align 8
+; CHECK-NEXT: [[VEC_GEP158:%.*]] = getelementptr double, ptr [[C]], i64 14
+; CHECK-NEXT: store <2 x double> [[TMP53]], ptr [[VEC_GEP158]], align 8
; CHECK-NEXT: ret void
;
@@ -264,193 +220,157 @@ define void @multiply(<16 x double> * %A, <16 x double> * %B, <16 x double>* %C)
;; -> c[2:4, 2:4]
entry:
- %a = load <16 x double>, <16 x double>* %A, align 8
- %b = load <16 x double>, <16 x double>* %B, align 8
+ %a = load <16 x double>, ptr %A, align 8
+ %b = load <16 x double>, ptr %B, align 8
%c = call <16 x double> @llvm.matrix.multiply(<16 x double> %a, <16 x double> %b, i32 4, i32 4, i32 4)
- store <16 x double> %c, <16 x double>* %C, align 8
+ store <16 x double> %c, ptr %C, align 8
ret void
}
; The same load is used for both operands of the multiply.
-define void @multiply_reuse_load(<16 x double>* noalias %A, <16 x double>* noalias %B, <16 x double>* noalias %C) {
+define void @multiply_reuse_load(ptr noalias %A, ptr noalias %B, ptr noalias %C) {
; CHECK-LABEL: @multiply_reuse_load(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <16 x double>* [[A:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT11]], <2 x double> [[TMP0]])
-; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT14]]
-; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT17]], <2 x double> [[TMP2]])
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST19:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST19]], align 8
-; CHECK-NEXT: [[VEC_GEP21:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST22:%.*]] = bitcast double* [[VEC_GEP21]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD23:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST22]], align 8
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST25:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD26:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST25]], align 8
-; CHECK-NEXT: [[VEC_GEP27:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST28:%.*]] = bitcast double* [[VEC_GEP27]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD29:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST28]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD20]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP1]])
-; CHECK-NEXT: [[SPLAT_SPLAT36:%.*]] = shufflevector <2 x double> [[COL_LOAD26]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT36]], <2 x double> [[TMP6]])
-; CHECK-NEXT: [[SPLAT_SPLAT40:%.*]] = shufflevector <2 x double> [[COL_LOAD29]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD20]], <2 x double> [[SPLAT_SPLAT40]], <2 x double> [[TMP3]])
-; CHECK-NEXT: [[SPLAT_SPLAT43:%.*]] = shufflevector <2 x double> [[COL_LOAD29]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD23]], <2 x double> [[SPLAT_SPLAT43]], <2 x double> [[TMP8]])
-; CHECK-NEXT: [[VEC_CAST45:%.*]] = bitcast <16 x double>* [[C:%.*]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP7]], <2 x double>* [[VEC_CAST45]], align 8
-; CHECK-NEXT: [[VEC_GEP46:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST47:%.*]] = bitcast double* [[VEC_GEP46]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP9]], <2 x double>* [[VEC_CAST47]], align 8
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST49:%.*]] = bitcast double* [[TMP10]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD50:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST49]], align 8
-; CHECK-NEXT: [[VEC_GEP51:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST52:%.*]] = bitcast double* [[VEC_GEP51]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST52]], align 8
-; CHECK-NEXT: [[VEC_CAST55:%.*]] = bitcast <16 x double>* [[A]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD56:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST55]], align 8
-; CHECK-NEXT: [[VEC_GEP57:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST58:%.*]] = bitcast double* [[VEC_GEP57]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD59:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST58]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])
+; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]
+; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <2 x double>, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[VEC_GEP15:%.*]] = getelementptr double, ptr [[A]], i64 12
+; CHECK-NEXT: [[COL_LOAD16:%.*]] = load <2 x double>, ptr [[VEC_GEP15]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD17:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[VEC_GEP18:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <2 x double>, ptr [[VEC_GEP18]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD14]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP1]])
+; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <2 x double> [[COL_LOAD17]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD16]], <2 x double> [[SPLAT_SPLAT26]], <2 x double> [[TMP6]])
+; CHECK-NEXT: [[SPLAT_SPLAT30:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD14]], <2 x double> [[SPLAT_SPLAT30]], <2 x double> [[TMP3]])
+; CHECK-NEXT: [[SPLAT_SPLAT33:%.*]] = shufflevector <2 x double> [[COL_LOAD19]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD16]], <2 x double> [[SPLAT_SPLAT33]], <2 x double> [[TMP8]])
+; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[C:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP34:%.*]] = getelementptr double, ptr [[C]], i64 4
+; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[VEC_GEP34]], align 8
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <2 x double>, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[VEC_GEP36:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD37:%.*]] = load <2 x double>, ptr [[VEC_GEP36]], align 8
+; CHECK-NEXT: [[COL_LOAD38:%.*]] = load <2 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[VEC_GEP39:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, ptr [[VEC_GEP39]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT43:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT43]]
+; CHECK-NEXT: [[SPLAT_SPLAT46:%.*]] = shufflevector <2 x double> [[COL_LOAD38]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD37]], <2 x double> [[SPLAT_SPLAT46]], <2 x double> [[TMP11]])
+; CHECK-NEXT: [[SPLAT_SPLAT49:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <2 x double> [[COL_LOAD35]], [[SPLAT_SPLAT49]]
+; CHECK-NEXT: [[SPLAT_SPLAT52:%.*]] = shufflevector <2 x double> [[COL_LOAD40]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD37]], <2 x double> [[SPLAT_SPLAT52]], <2 x double> [[TMP13]])
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr double, ptr [[A]], i64 10
+; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <2 x double>, ptr [[TMP15]], align 8
+; CHECK-NEXT: [[VEC_GEP54:%.*]] = getelementptr double, ptr [[A]], i64 14
+; CHECK-NEXT: [[COL_LOAD55:%.*]] = load <2 x double>, ptr [[VEC_GEP54]], align 8
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD56:%.*]] = load <2 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[VEC_GEP57:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD58:%.*]] = load <2 x double>, ptr [[VEC_GEP57]], align 8
; CHECK-NEXT: [[SPLAT_SPLAT62:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = fmul contract <2 x double> [[COL_LOAD50]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT62]], <2 x double> [[TMP12]])
; CHECK-NEXT: [[SPLAT_SPLAT65:%.*]] = shufflevector <2 x double> [[COL_LOAD56]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP12:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT65]], <2 x double> [[TMP11]])
-; CHECK-NEXT: [[SPLAT_SPLAT68:%.*]] = shufflevector <2 x double> [[COL_LOAD59]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = fmul contract <2 x double> [[COL_LOAD50]], [[SPLAT_SPLAT68]]
-; CHECK-NEXT: [[SPLAT_SPLAT71:%.*]] = shufflevector <2 x double> [[COL_LOAD59]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT71]], <2 x double> [[TMP13]])
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST73:%.*]] = bitcast double* [[TMP15]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD74:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST73]], align 8
-; CHECK-NEXT: [[VEC_GEP75:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST76:%.*]] = bitcast double* [[VEC_GEP75]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD77:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST76]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST79:%.*]] = bitcast double* [[TMP16]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD80:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST79]], align 8
-; CHECK-NEXT: [[VEC_GEP81:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST82:%.*]] = bitcast double* [[VEC_GEP81]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD83:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST82]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT87:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD74]], <2 x double> [[SPLAT_SPLAT87]], <2 x double> [[TMP12]])
-; CHECK-NEXT: [[SPLAT_SPLAT90:%.*]] = shufflevector <2 x double> [[COL_LOAD80]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP18:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT90]], <2 x double> [[TMP17]])
-; CHECK-NEXT: [[SPLAT_SPLAT94:%.*]] = shufflevector <2 x double> [[COL_LOAD83]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD74]], <2 x double> [[SPLAT_SPLAT94]], <2 x double> [[TMP14]])
-; CHECK-NEXT: [[SPLAT_SPLAT97:%.*]] = shufflevector <2 x double> [[COL_LOAD83]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD77]], <2 x double> [[SPLAT_SPLAT97]], <2 x double> [[TMP19]])
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST99:%.*]] = bitcast double* [[TMP21]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP18]], <2 x double>* [[VEC_CAST99]], align 8
-; CHECK-NEXT: [[VEC_GEP100:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST101:%.*]] = bitcast double* [[VEC_GEP100]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP20]], <2 x double>* [[VEC_CAST101]], align 8
-; CHECK-NEXT: [[VEC_CAST103:%.*]] = bitcast <16 x double>* [[A]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD104:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST103]], align 8
-; CHECK-NEXT: [[VEC_GEP105:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST106:%.*]] = bitcast double* [[VEC_GEP105]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD107:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST106]], align 8
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST109:%.*]] = bitcast double* [[TMP22]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD110:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST109]], align 8
-; CHECK-NEXT: [[VEC_GEP111:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST112:%.*]] = bitcast double* [[VEC_GEP111]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD113:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST112]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT116:%.*]] = shufflevector <2 x double> [[COL_LOAD110]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD104]], [[SPLAT_SPLAT116]]
-; CHECK-NEXT: [[SPLAT_SPLAT119:%.*]] = shufflevector <2 x double> [[COL_LOAD110]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP24:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD107]], <2 x double> [[SPLAT_SPLAT119]], <2 x double> [[TMP23]])
-; CHECK-NEXT: [[SPLAT_SPLAT122:%.*]] = shufflevector <2 x double> [[COL_LOAD113]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD104]], [[SPLAT_SPLAT122]]
-; CHECK-NEXT: [[SPLAT_SPLAT125:%.*]] = shufflevector <2 x double> [[COL_LOAD113]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD107]], <2 x double> [[SPLAT_SPLAT125]], <2 x double> [[TMP25]])
-; CHECK-NEXT: [[TMP27:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST127:%.*]] = bitcast double* [[TMP27]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD128:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST127]], align 8
-; CHECK-NEXT: [[VEC_GEP129:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST130:%.*]] = bitcast double* [[VEC_GEP129]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD131:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST130]], align 8
-; CHECK-NEXT: [[TMP28:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST133:%.*]] = bitcast double* [[TMP28]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD134:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST133]], align 8
-; CHECK-NEXT: [[VEC_GEP135:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST136:%.*]] = bitcast double* [[VEC_GEP135]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD137:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST136]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT141:%.*]] = shufflevector <2 x double> [[COL_LOAD134]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP29:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD128]], <2 x double> [[SPLAT_SPLAT141]], <2 x double> [[TMP24]])
-; CHECK-NEXT: [[SPLAT_SPLAT144:%.*]] = shufflevector <2 x double> [[COL_LOAD134]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP30:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT144]], <2 x double> [[TMP29]])
-; CHECK-NEXT: [[SPLAT_SPLAT148:%.*]] = shufflevector <2 x double> [[COL_LOAD137]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP31:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD128]], <2 x double> [[SPLAT_SPLAT148]], <2 x double> [[TMP26]])
-; CHECK-NEXT: [[SPLAT_SPLAT151:%.*]] = shufflevector <2 x double> [[COL_LOAD137]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT151]], <2 x double> [[TMP31]])
-; CHECK-NEXT: [[TMP33:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST153:%.*]] = bitcast double* [[TMP33]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP30]], <2 x double>* [[VEC_CAST153]], align 8
-; CHECK-NEXT: [[VEC_GEP154:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST155:%.*]] = bitcast double* [[VEC_GEP154]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP32]], <2 x double>* [[VEC_CAST155]], align 8
-; CHECK-NEXT: [[TMP34:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST157:%.*]] = bitcast double* [[TMP34]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD158:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST157]], align 8
-; CHECK-NEXT: [[VEC_GEP159:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST160:%.*]] = bitcast double* [[VEC_GEP159]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD161:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST160]], align 8
-; CHECK-NEXT: [[TMP35:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 8
-; CHECK-NEXT: [[VEC_CAST163:%.*]] = bitcast double* [[TMP35]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD164:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST163]], align 8
-; CHECK-NEXT: [[VEC_GEP165:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 12
-; CHECK-NEXT: [[VEC_CAST166:%.*]] = bitcast double* [[VEC_GEP165]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD167:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST166]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT170:%.*]] = shufflevector <2 x double> [[COL_LOAD164]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP36:%.*]] = fmul contract <2 x double> [[COL_LOAD158]], [[SPLAT_SPLAT170]]
-; CHECK-NEXT: [[SPLAT_SPLAT173:%.*]] = shufflevector <2 x double> [[COL_LOAD164]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD161]], <2 x double> [[SPLAT_SPLAT173]], <2 x double> [[TMP36]])
-; CHECK-NEXT: [[SPLAT_SPLAT176:%.*]] = shufflevector <2 x double> [[COL_LOAD167]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP38:%.*]] = fmul contract <2 x double> [[COL_LOAD158]], [[SPLAT_SPLAT176]]
-; CHECK-NEXT: [[SPLAT_SPLAT179:%.*]] = shufflevector <2 x double> [[COL_LOAD167]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD161]], <2 x double> [[SPLAT_SPLAT179]], <2 x double> [[TMP38]])
-; CHECK-NEXT: [[TMP40:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST181:%.*]] = bitcast double* [[TMP40]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD182:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST181]], align 8
-; CHECK-NEXT: [[VEC_GEP183:%.*]] = getelementptr <16 x double>, <16 x double>* [[A]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST184:%.*]] = bitcast double* [[VEC_GEP183]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD185:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST184]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT195:%.*]] = shufflevector <2 x double> [[COL_LOAD182]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD182]], <2 x double> [[SPLAT_SPLAT195]], <2 x double> [[TMP37]])
-; CHECK-NEXT: [[SPLAT_SPLAT198:%.*]] = shufflevector <2 x double> [[COL_LOAD182]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP42:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD185]], <2 x double> [[SPLAT_SPLAT198]], <2 x double> [[TMP41]])
-; CHECK-NEXT: [[SPLAT_SPLAT202:%.*]] = shufflevector <2 x double> [[COL_LOAD185]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD182]], <2 x double> [[SPLAT_SPLAT202]], <2 x double> [[TMP39]])
-; CHECK-NEXT: [[SPLAT_SPLAT205:%.*]] = shufflevector <2 x double> [[COL_LOAD185]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP44:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD185]], <2 x double> [[SPLAT_SPLAT205]], <2 x double> [[TMP43]])
-; CHECK-NEXT: [[TMP45:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 10
-; CHECK-NEXT: [[VEC_CAST207:%.*]] = bitcast double* [[TMP45]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP42]], <2 x double>* [[VEC_CAST207]], align 8
-; CHECK-NEXT: [[VEC_GEP208:%.*]] = getelementptr <16 x double>, <16 x double>* [[C]], i64 0, i64 14
-; CHECK-NEXT: [[VEC_CAST209:%.*]] = bitcast double* [[VEC_GEP208]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP44]], <2 x double>* [[VEC_CAST209]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD55]], <2 x double> [[SPLAT_SPLAT65]], <2 x double> [[TMP17]])
+; CHECK-NEXT: [[SPLAT_SPLAT69:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD53]], <2 x double> [[SPLAT_SPLAT69]], <2 x double> [[TMP14]])
+; CHECK-NEXT: [[SPLAT_SPLAT72:%.*]] = shufflevector <2 x double> [[COL_LOAD58]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP20:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD55]], <2 x double> [[SPLAT_SPLAT72]], <2 x double> [[TMP19]])
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP18]], ptr [[TMP21]], align 8
+; CHECK-NEXT: [[VEC_GEP73:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT: store <2 x double> [[TMP20]], ptr [[VEC_GEP73]], align 8
+; CHECK-NEXT: [[COL_LOAD74:%.*]] = load <2 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[VEC_GEP75:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD76:%.*]] = load <2 x double>, ptr [[VEC_GEP75]], align 8
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT: [[COL_LOAD77:%.*]] = load <2 x double>, ptr [[TMP22]], align 8
+; CHECK-NEXT: [[VEC_GEP78:%.*]] = getelementptr double, ptr [[A]], i64 12
+; CHECK-NEXT: [[COL_LOAD79:%.*]] = load <2 x double>, ptr [[VEC_GEP78]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT82:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT82]]
+; CHECK-NEXT: [[SPLAT_SPLAT85:%.*]] = shufflevector <2 x double> [[COL_LOAD77]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP24:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD76]], <2 x double> [[SPLAT_SPLAT85]], <2 x double> [[TMP23]])
+; CHECK-NEXT: [[SPLAT_SPLAT88:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = fmul contract <2 x double> [[COL_LOAD74]], [[SPLAT_SPLAT88]]
+; CHECK-NEXT: [[SPLAT_SPLAT91:%.*]] = shufflevector <2 x double> [[COL_LOAD79]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP26:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD76]], <2 x double> [[SPLAT_SPLAT91]], <2 x double> [[TMP25]])
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT: [[COL_LOAD92:%.*]] = load <2 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT: [[VEC_GEP93:%.*]] = getelementptr double, ptr [[A]], i64 12
+; CHECK-NEXT: [[COL_LOAD94:%.*]] = load <2 x double>, ptr [[VEC_GEP93]], align 8
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr double, ptr [[A]], i64 10
+; CHECK-NEXT: [[COL_LOAD95:%.*]] = load <2 x double>, ptr [[TMP28]], align 8
+; CHECK-NEXT: [[VEC_GEP96:%.*]] = getelementptr double, ptr [[A]], i64 14
+; CHECK-NEXT: [[COL_LOAD97:%.*]] = load <2 x double>, ptr [[VEC_GEP96]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT101:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD92]], <2 x double> [[SPLAT_SPLAT101]], <2 x double> [[TMP24]])
+; CHECK-NEXT: [[SPLAT_SPLAT104:%.*]] = shufflevector <2 x double> [[COL_LOAD95]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP30:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD94]], <2 x double> [[SPLAT_SPLAT104]], <2 x double> [[TMP29]])
+; CHECK-NEXT: [[SPLAT_SPLAT108:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD92]], <2 x double> [[SPLAT_SPLAT108]], <2 x double> [[TMP26]])
+; CHECK-NEXT: [[SPLAT_SPLAT111:%.*]] = shufflevector <2 x double> [[COL_LOAD97]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP32:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD94]], <2 x double> [[SPLAT_SPLAT111]], <2 x double> [[TMP31]])
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr double, ptr [[C]], i64 8
+; CHECK-NEXT: store <2 x double> [[TMP30]], ptr [[TMP33]], align 8
+; CHECK-NEXT: [[VEC_GEP112:%.*]] = getelementptr double, ptr [[C]], i64 12
+; CHECK-NEXT: store <2 x double> [[TMP32]], ptr [[VEC_GEP112]], align 8
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD113:%.*]] = load <2 x double>, ptr [[TMP34]], align 8
+; CHECK-NEXT: [[VEC_GEP114:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD115:%.*]] = load <2 x double>, ptr [[VEC_GEP114]], align 8
+; CHECK-NEXT: [[TMP35:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT: [[COL_LOAD116:%.*]] = load <2 x double>, ptr [[TMP35]], align 8
+; CHECK-NEXT: [[VEC_GEP117:%.*]] = getelementptr double, ptr [[A]], i64 12
+; CHECK-NEXT: [[COL_LOAD118:%.*]] = load <2 x double>, ptr [[VEC_GEP117]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT121:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP36:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT121]]
+; CHECK-NEXT: [[SPLAT_SPLAT124:%.*]] = shufflevector <2 x double> [[COL_LOAD116]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP37:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT124]], <2 x double> [[TMP36]])
+; CHECK-NEXT: [[SPLAT_SPLAT127:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP38:%.*]] = fmul contract <2 x double> [[COL_LOAD113]], [[SPLAT_SPLAT127]]
+; CHECK-NEXT: [[SPLAT_SPLAT130:%.*]] = shufflevector <2 x double> [[COL_LOAD118]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP39:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD115]], <2 x double> [[SPLAT_SPLAT130]], <2 x double> [[TMP38]])
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr double, ptr [[A]], i64 10
+; CHECK-NEXT: [[COL_LOAD131:%.*]] = load <2 x double>, ptr [[TMP40]], align 8
+; CHECK-NEXT: [[VEC_GEP132:%.*]] = getelementptr double, ptr [[A]], i64 14
+; CHECK-NEXT: [[COL_LOAD133:%.*]] = load <2 x double>, ptr [[VEC_GEP132]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT140:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP41:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT140]], <2 x double> [[TMP37]])
+; CHECK-NEXT: [[SPLAT_SPLAT143:%.*]] = shufflevector <2 x double> [[COL_LOAD131]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP42:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD133]], <2 x double> [[SPLAT_SPLAT143]], <2 x double> [[TMP41]])
+; CHECK-NEXT: [[SPLAT_SPLAT147:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP43:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD131]], <2 x double> [[SPLAT_SPLAT147]], <2 x double> [[TMP39]])
+; CHECK-NEXT: [[SPLAT_SPLAT150:%.*]] = shufflevector <2 x double> [[COL_LOAD133]], <2 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP44:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD133]], <2 x double> [[SPLAT_SPLAT150]], <2 x double> [[TMP43]])
+; CHECK-NEXT: [[TMP45:%.*]] = getelementptr double, ptr [[C]], i64 10
+; CHECK-NEXT: store <2 x double> [[TMP42]], ptr [[TMP45]], align 8
+; CHECK-NEXT: [[VEC_GEP151:%.*]] = getelementptr double, ptr [[C]], i64 14
+; CHECK-NEXT: store <2 x double> [[TMP44]], ptr [[VEC_GEP151]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <16 x double>, <16 x double>* %A, align 8
+ %a = load <16 x double>, ptr %A, align 8
%c = call <16 x double> @llvm.matrix.multiply(<16 x double> %a, <16 x double> %a, i32 4, i32 4, i32 4)
- store <16 x double> %c, <16 x double>* %C, align 8
+ store <16 x double> %c, ptr %C, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
index 9b4e0b70c8209..2dac1e51cfef3 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-minimal.ll
@@ -10,55 +10,47 @@
target datalayout = "e-m:o-i64:64-f80:128-n8:8:32:64-S128"
target triple = "aarch64-apple-ios"
-define void @multiply(<8 x double> * %A, <8 x double> * %B, <4 x double>* %C) {
+define void @multiply(ptr %A, ptr %B, ptr %C) {
; CHECK-LABEL: @multiply(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <8 x double>* [[A:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr <8 x double>, <8 x double>* [[A]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr <8 x double>, <8 x double>* [[A]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr <8 x double>, <8 x double>* [[A]], i64 0, i64 6
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast <8 x double>* [[B:%.*]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST9]], align 8
-; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr <8 x double>, <8 x double>* [[B]], i64 0, i64 4
-; CHECK-NEXT: [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST12]], align 8
-; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <2 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[VEC_GEP4:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, ptr [[VEC_GEP4]], align 8
+; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <4 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B]], i64 4
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <4 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]
-; CHECK-NEXT: [[SPLAT_SPLAT16:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT16]], <2 x double> [[TMP0]])
-; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
-; CHECK-NEXT: [[TMP2:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT19]], <2 x double> [[TMP1]])
-; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <4 x double> [[COL_LOAD10]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT22]], <2 x double> [[TMP2]])
-; CHECK-NEXT: [[SPLAT_SPLAT25:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT25]]
-; CHECK-NEXT: [[SPLAT_SPLAT28:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT: [[TMP5:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD2]], <2 x double> [[SPLAT_SPLAT28]], <2 x double> [[TMP4]])
-; CHECK-NEXT: [[SPLAT_SPLAT31:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
-; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT31]], <2 x double> [[TMP5]])
-; CHECK-NEXT: [[SPLAT_SPLAT34:%.*]] = shufflevector <4 x double> [[COL_LOAD13]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD8]], <2 x double> [[SPLAT_SPLAT34]], <2 x double> [[TMP6]])
-; CHECK-NEXT: [[VEC_CAST35:%.*]] = bitcast <4 x double>* [[C:%.*]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP3]], <2 x double>* [[VEC_CAST35]], align 8
-; CHECK-NEXT: [[VEC_GEP36:%.*]] = getelementptr <4 x double>, <4 x double>* [[C]], i64 0, i64 2
-; CHECK-NEXT: [[VEC_CAST37:%.*]] = bitcast double* [[VEC_GEP36]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP7]], <2 x double>* [[VEC_CAST37]], align 8
+; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT11]], <2 x double> [[TMP0]])
+; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT14]], <2 x double> [[TMP1]])
+; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <4 x double> [[COL_LOAD6]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT17]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP5:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT23]], <2 x double> [[TMP4]])
+; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD3]], <2 x double> [[SPLAT_SPLAT26]], <2 x double> [[TMP5]])
+; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <4 x double> [[COL_LOAD8]], <4 x double> undef, <2 x i32> <i32 3, i32 3>
+; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD5]], <2 x double> [[SPLAT_SPLAT29]], <2 x double> [[TMP6]])
+; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[C:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP30:%.*]] = getelementptr double, ptr [[C]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[VEC_GEP30]], align 8
; CHECK-NEXT: ret void
;
entry:
- %a = load <8 x double>, <8 x double>* %A, align 8
- %b = load <8 x double>, <8 x double>* %B, align 8
+ %a = load <8 x double>, ptr %A, align 8
+ %b = load <8 x double>, ptr %B, align 8
%c = call <4 x double> @llvm.matrix.multiply(<8 x double> %a, <8 x double> %b, i32 2, i32 4, i32 2)
- store <4 x double> %c, <4 x double>* %C, align 8
+ store <4 x double> %c, ptr %C, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
index 9a7948d166451..92f0f1a7e2f38 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-right-transpose.ll
@@ -70,7 +70,7 @@ entry:
}
; Lower transpose if not all uses are folded.
-define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double> %b, <6 x double>* %p) {
+define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double> %b, ptr %p) {
; CHECK-LABEL: @multiply_right_transpose_2x3x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SPLIT39:%.*]] = shufflevector <6 x double> [[B:%.*]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
@@ -88,12 +88,9 @@ define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[SPLIT41]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
-; CHECK-NEXT: [[TMP12:%.*]] = bitcast <6 x double>* [[P:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP12]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP5]], <3 x double>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP12]], i64 3
-; CHECK-NEXT: [[VEC_CAST42:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP11]], <3 x double>* [[VEC_CAST42]], align 8
+; CHECK-NEXT: store <3 x double> [[TMP5]], ptr [[P:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[P]], i64 3
+; CHECK-NEXT: store <3 x double> [[TMP11]], ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[A:%.*]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[A]], <6 x double> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <6 x double> [[A]], <6 x double> poison, <2 x i32> <i32 4, i32 5>
@@ -101,87 +98,87 @@ define <4 x double> @multiply_right_transpose_2x3x2(<6 x double> %a, <6 x double
; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <6 x double> [[B]], <6 x double> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <6 x double> [[B]], <6 x double> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP13]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP12]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP14:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT: [[TMP13:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> poison, double [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <1 x double> poison, double [[TMP14]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT7]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP16:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
-; CHECK-NEXT: [[TMP17:%.*]] = fadd <1 x double> [[TMP14]], [[TMP16]]
+; CHECK-NEXT: [[TMP15:%.*]] = fmul <1 x double> [[BLOCK6]], [[SPLAT_SPLAT8]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd <1 x double> [[TMP13]], [[TMP15]]
; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP18]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP17]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP19:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
-; CHECK-NEXT: [[TMP20:%.*]] = fadd <1 x double> [[TMP17]], [[TMP19]]
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <1 x double> [[TMP20]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP21]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP18:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
+; CHECK-NEXT: [[TMP19:%.*]] = fadd <1 x double> [[TMP16]], [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <1 x double> [[TMP19]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK12:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP23]], i32 0
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP22]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT: [[TMP23:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
; CHECK-NEXT: [[BLOCK15:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP25]], i32 0
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP24]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP26:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
-; CHECK-NEXT: [[TMP27:%.*]] = fadd <1 x double> [[TMP24]], [[TMP26]]
+; CHECK-NEXT: [[TMP25:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT: [[TMP26:%.*]] = fadd <1 x double> [[TMP23]], [[TMP25]]
; CHECK-NEXT: [[BLOCK18:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 0
-; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP28]], i32 0
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 0
+; CHECK-NEXT: [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP27]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP29:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
-; CHECK-NEXT: [[TMP30:%.*]] = fadd <1 x double> [[TMP27]], [[TMP29]]
-; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <1 x double> [[TMP30]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <2 x double> [[TMP22]], <2 x double> [[TMP31]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP28:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT: [[TMP29:%.*]] = fadd <1 x double> [[TMP26]], [[TMP28]]
+; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <1 x double> [[TMP29]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> [[TMP30]], <2 x i32> <i32 0, i32 2>
; CHECK-NEXT: [[BLOCK21:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP33]], i32 0
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP32]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT: [[TMP33:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
; CHECK-NEXT: [[BLOCK24:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP35:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP35]], i32 0
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP34]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP36:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
-; CHECK-NEXT: [[TMP37:%.*]] = fadd <1 x double> [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP35:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT: [[TMP36:%.*]] = fadd <1 x double> [[TMP33]], [[TMP35]]
; CHECK-NEXT: [[BLOCK27:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP38]], i32 0
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP37]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP39:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
-; CHECK-NEXT: [[TMP40:%.*]] = fadd <1 x double> [[TMP37]], [[TMP39]]
-; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <1 x double> [[TMP40]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP41]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP38:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT: [[TMP39:%.*]] = fadd <1 x double> [[TMP36]], [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <1 x double> [[TMP39]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP40]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[BLOCK30:%.*]] = shufflevector <2 x double> [[SPLIT]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP43]], i32 0
+; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
+; CHECK-NEXT: [[TMP43:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
; CHECK-NEXT: [[BLOCK33:%.*]] = shufflevector <2 x double> [[SPLIT1]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP45:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP45]], i32 0
+; CHECK-NEXT: [[TMP44:%.*]] = extractelement <2 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP44]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP46:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
-; CHECK-NEXT: [[TMP47:%.*]] = fadd <1 x double> [[TMP44]], [[TMP46]]
+; CHECK-NEXT: [[TMP45:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
+; CHECK-NEXT: [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
; CHECK-NEXT: [[BLOCK36:%.*]] = shufflevector <2 x double> [[SPLIT2]], <2 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 1
-; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP48]], i32 0
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <2 x double> [[SPLIT5]], i64 1
+; CHECK-NEXT: [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP47]], i32 0
; CHECK-NEXT: [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP49:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
-; CHECK-NEXT: [[TMP50:%.*]] = fadd <1 x double> [[TMP47]], [[TMP49]]
-; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <1 x double> [[TMP50]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
-; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <2 x double> [[TMP42]], <2 x double> [[TMP51]], <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT: [[TMP53:%.*]] = shufflevector <2 x double> [[TMP32]], <2 x double> [[TMP52]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: ret <4 x double> [[TMP53]]
+; CHECK-NEXT: [[TMP48:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
+; CHECK-NEXT: [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]]
+; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <2 x double> [[TMP41]], <2 x double> [[TMP50]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> [[TMP51]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <4 x double> [[TMP52]]
;
entry:
%bt = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 2, i32 3)
- store <6 x double> %bt, <6 x double>* %p, align 16
+ store <6 x double> %bt, ptr %p, align 16
%c = call <4 x double> @llvm.matrix.multiply.v4f64.v6f64.v6f64(<6 x double> %a, <6 x double> %bt, i32 2, i32 3, i32 2)
ret <4 x double> %c
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
index e2d450df451a9..33a338dbc4ea0 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backward.ll
@@ -41,97 +41,87 @@ entry:
ret <8 x double> %c
}
-define <8 x double> @load_fadd_transpose(<8 x double>* %A.Ptr, <8 x double> %b) {
+define <8 x double> @load_fadd_transpose(ptr %A.Ptr, <8 x double> %b) {
; CHECK-LABEL: @load_fadd_transpose(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x double>* [[A_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 4
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 4
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <2 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[VEC_GEP4:%.*]] = getelementptr double, ptr [[A_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, ptr [[VEC_GEP4]], align 8
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[SPLIT9:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[SPLIT10:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[SPLIT11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
-; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[COL_LOAD]], [[SPLIT]]
-; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[COL_LOAD2]], [[SPLIT9]]
-; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[COL_LOAD5]], [[SPLIT10]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[COL_LOAD8]], [[SPLIT11]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> [[TMP6]], double [[TMP7]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP8]], double [[TMP9]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP10]], double [[TMP11]], i64 3
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> poison, double [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP14]], double [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP17]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP19]], i64 3
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: ret <8 x double> [[TMP21]]
+; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[SPLIT8:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP0:%.*]] = fadd <2 x double> [[COL_LOAD]], [[SPLIT]]
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[COL_LOAD1]], [[SPLIT6]]
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[COL_LOAD3]], [[SPLIT7]]
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[COL_LOAD5]], [[SPLIT8]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> poison, double [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP7]], double [[TMP8]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> poison, double [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 3
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <8 x double> [[TMP20]]
;
entry:
- %a = load <8 x double>, <8 x double>* %A.Ptr, align 8
+ %a = load <8 x double>, ptr %A.Ptr, align 8
%add = fadd <8 x double> %a, %b
%c = call <8 x double> @llvm.matrix.transpose(<8 x double> %add, i32 2, i32 4)
ret <8 x double> %c
}
-define <8 x double> @load_fneg_transpose(<8 x double>* %A.Ptr) {
+define <8 x double> @load_fneg_transpose(ptr %A.Ptr) {
; CHECK-LABEL: @load_fneg_transpose(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x double>* [[A_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 2
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 4
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = fneg <2 x double> [[COL_LOAD]]
-; CHECK-NEXT: [[TMP2:%.*]] = fneg <2 x double> [[COL_LOAD2]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 2
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 4
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <2 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[VEC_GEP4:%.*]] = getelementptr double, ptr [[A_PTR]], i64 6
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, ptr [[VEC_GEP4]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = fneg <2 x double> [[COL_LOAD]]
+; CHECK-NEXT: [[TMP1:%.*]] = fneg <2 x double> [[COL_LOAD1]]
+; CHECK-NEXT: [[TMP2:%.*]] = fneg <2 x double> [[COL_LOAD3]]
; CHECK-NEXT: [[TMP3:%.*]] = fneg <2 x double> [[COL_LOAD5]]
-; CHECK-NEXT: [[TMP4:%.*]] = fneg <2 x double> [[COL_LOAD8]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> [[TMP6]], double [[TMP7]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP8]], double [[TMP9]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP10]], double [[TMP11]], i64 3
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> poison, double [[TMP13]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP14]], double [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP16]], double [[TMP17]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP19]], i64 3
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> [[TMP20]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: ret <8 x double> [[TMP21]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> poison, double [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP7]], double [[TMP8]], i64 2
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP9]], double [[TMP10]], i64 3
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> poison, double [[TMP12]], i64 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP18]], i64 3
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <8 x double> [[TMP20]]
;
entry:
- %a = load <8 x double>, <8 x double>* %A.Ptr, align 8
+ %a = load <8 x double>, ptr %A.Ptr, align 8
%neg = fneg <8 x double> %a
%c = call <8 x double> @llvm.matrix.transpose(<8 x double> %neg, i32 2, i32 4)
ret <8 x double> %c
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
index 2e1c654bff5ff..36669e4cdf10f 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-forward.ll
@@ -3,7 +3,7 @@
; Check that we do not emit shufflevectors to flatten the result of the
; transpose and store the columns directly.
-define void @transpose_store(<8 x double> %a, <8 x double>* %Ptr) {
+define void @transpose_store(<8 x double> %a, ptr %Ptr) {
; CHECK-LABEL: @transpose_store(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
@@ -26,17 +26,14 @@ define void @transpose_store(<8 x double> %a, <8 x double>* %Ptr) {
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 2
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[SPLIT3]], i64 1
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 3
-; CHECK-NEXT: [[TMP16:%.*]] = bitcast <8 x double>* [[PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
-; CHECK-NEXT: store <4 x double> [[TMP7]], <4 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP16]], i64 4
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP]] to <4 x double>*
-; CHECK-NEXT: store <4 x double> [[TMP15]], <4 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT: store <4 x double> [[TMP7]], ptr [[PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[PTR]], i64 4
+; CHECK-NEXT: store <4 x double> [[TMP15]], ptr [[VEC_GEP]], align 8
; CHECK-NEXT: ret void
;
entry:
%c = call <8 x double> @llvm.matrix.transpose(<8 x double> %a, i32 2, i32 4)
- store <8 x double> %c, <8 x double>* %Ptr, align 8
+ store <8 x double> %c, ptr %Ptr, align 8
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
index c6da1e4fad02e..cee7890b43174 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-mixed-users.ll
@@ -3,7 +3,7 @@
; Currently we only lower stores with shape information, but need to embed the
; matrix in a flat vector for function calls and returns.
-define <8 x double> @strided_load_4x4(<8 x double> %in, <8 x double>* %Ptr) {
+define <8 x double> @strided_load_4x4(<8 x double> %in, ptr %Ptr) {
; CHECK-LABEL: @strided_load_4x4(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <8 x double> [[IN:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <8 x double> [[IN]], <8 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
@@ -26,23 +26,18 @@ define <8 x double> @strided_load_4x4(<8 x double> %in, <8 x double>* %Ptr) {
; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> [[TMP18]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP20:%.*]] = bitcast <8 x double>* [[PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP20]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP4]], <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP20]], i64 2
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP8]], <2 x double>* [[VEC_CAST2]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP20]], i64 4
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP12]], <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, double* [[TMP20]], i64 6
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[VEC_GEP5]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP16]], <2 x double>* [[VEC_CAST6]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP4]], ptr [[PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[PTR]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP8]], ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[PTR]], i64 4
+; CHECK-NEXT: store <2 x double> [[TMP12]], ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[PTR]], i64 6
+; CHECK-NEXT: store <2 x double> [[TMP16]], ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: call void @foo(<8 x double> [[TMP19]])
; CHECK-NEXT: ret <8 x double> [[TMP19]]
;
%transposed = call <8 x double> @llvm.matrix.transpose(<8 x double> %in, i32 4, i32 2)
- store <8 x double> %transposed, <8 x double>* %Ptr, align 8
+ store <8 x double> %transposed, ptr %Ptr, align 8
call void @foo(<8 x double> %transposed)
ret <8 x double> %transposed
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
index b757f49bdde10..8f8dfd88f9f0d 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-multiple-iterations.ll
@@ -5,76 +5,66 @@
; Make sure we propagate in multiple iterations. First, we back-propagate the
; shape information from the transpose to %A, in the next iteration we
; forward-propagate it to %Mul, and then back to %B.
-define <16 x double> @backpropagation_iterations(<16 x double>* %A.Ptr, <16 x double>* %B.Ptr) {
+define <16 x double> @backpropagation_iterations(ptr %A.Ptr, ptr %B.Ptr) {
; CHECK-LABEL: @backpropagation_iterations(
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x double>* [[A_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP1]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP1]], i64 4
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP1]], i64 8
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[TMP1]], i64 12
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP3]], double [[TMP4]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP5]], double [[TMP6]], i64 2
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[COL_LOAD8]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP7]], double [[TMP8]], i64 3
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> poison, double [[TMP10]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP12]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 1
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP14]], i64 2
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x double> [[COL_LOAD8]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP16]], i64 3
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 2
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> poison, double [[TMP18]], i64 0
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 2
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP20]], i64 1
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 2
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x double> [[TMP21]], double [[TMP22]], i64 2
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <4 x double> [[COL_LOAD8]], i64 2
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x double> [[TMP23]], double [[TMP24]], i64 3
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 3
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x double> poison, double [[TMP26]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x double> [[COL_LOAD2]], i64 3
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x double> [[TMP27]], double [[TMP28]], i64 1
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 3
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x double> [[TMP29]], double [[TMP30]], i64 2
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x double> [[COL_LOAD8]], i64 3
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x double> [[TMP31]], double [[TMP32]], i64 3
-; CHECK-NEXT: [[TMP34:%.*]] = bitcast <16 x double>* [[B_PTR:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast double* [[TMP34]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST9]], align 8
-; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr double, double* [[TMP34]], i64 4
-; CHECK-NEXT: [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST12]], align 8
-; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr double, double* [[TMP34]], i64 8
-; CHECK-NEXT: [[VEC_CAST15:%.*]] = bitcast double* [[VEC_GEP14]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD16:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST15]], align 8
-; CHECK-NEXT: [[VEC_GEP17:%.*]] = getelementptr double, double* [[TMP34]], i64 12
-; CHECK-NEXT: [[VEC_CAST18:%.*]] = bitcast double* [[VEC_GEP17]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST18]], align 8
-; CHECK-NEXT: [[TMP35:%.*]] = fmul <4 x double> [[COL_LOAD]], [[COL_LOAD10]]
-; CHECK-NEXT: [[TMP36:%.*]] = fmul <4 x double> [[COL_LOAD2]], [[COL_LOAD13]]
-; CHECK-NEXT: [[TMP37:%.*]] = fmul <4 x double> [[COL_LOAD5]], [[COL_LOAD16]]
-; CHECK-NEXT: [[TMP38:%.*]] = fmul <4 x double> [[COL_LOAD8]], [[COL_LOAD19]]
-; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <4 x double> [[TMP35]], <4 x double> [[TMP36]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> [[TMP38]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <8 x double> [[TMP39]], <8 x double> [[TMP40]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: ret <16 x double> [[TMP41]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, ptr [[A_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 4
+; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <4 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 8
+; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <4 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT: [[VEC_GEP4:%.*]] = getelementptr double, ptr [[A_PTR]], i64 12
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <4 x double>, ptr [[VEC_GEP4]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> poison, double [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP2]], double [[TMP3]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[COL_LOAD3]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> [[TMP4]], double [[TMP5]], i64 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> [[TMP6]], double [[TMP7]], i64 3
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 1
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> poison, double [[TMP9]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP10]], double [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x double> [[COL_LOAD3]], i64 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> [[TMP12]], double [[TMP13]], i64 2
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP14]], double [[TMP15]], i64 3
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 2
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> poison, double [[TMP17]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 2
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x double> [[TMP18]], double [[TMP19]], i64 1
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x double> [[COL_LOAD3]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x double> [[TMP20]], double [[TMP21]], i64 2
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 2
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x double> [[TMP22]], double [[TMP23]], i64 3
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x double> [[COL_LOAD]], i64 3
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x double> poison, double [[TMP25]], i64 0
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x double> [[COL_LOAD1]], i64 3
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x double> [[TMP26]], double [[TMP27]], i64 1
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x double> [[COL_LOAD3]], i64 3
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x double> [[TMP28]], double [[TMP29]], i64 2
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[COL_LOAD5]], i64 3
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x double> [[TMP30]], double [[TMP31]], i64 3
+; CHECK-NEXT: [[COL_LOAD6:%.*]] = load <4 x double>, ptr [[B_PTR:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B_PTR]], i64 4
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <4 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT: [[VEC_GEP9:%.*]] = getelementptr double, ptr [[B_PTR]], i64 8
+; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <4 x double>, ptr [[VEC_GEP9]], align 8
+; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr double, ptr [[B_PTR]], i64 12
+; CHECK-NEXT: [[COL_LOAD12:%.*]] = load <4 x double>, ptr [[VEC_GEP11]], align 8
+; CHECK-NEXT: [[TMP33:%.*]] = fmul <4 x double> [[COL_LOAD]], [[COL_LOAD6]]
+; CHECK-NEXT: [[TMP34:%.*]] = fmul <4 x double> [[COL_LOAD1]], [[COL_LOAD8]]
+; CHECK-NEXT: [[TMP35:%.*]] = fmul <4 x double> [[COL_LOAD3]], [[COL_LOAD10]]
+; CHECK-NEXT: [[TMP36:%.*]] = fmul <4 x double> [[COL_LOAD5]], [[COL_LOAD12]]
+; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <4 x double> [[TMP33]], <4 x double> [[TMP34]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <4 x double> [[TMP35]], <4 x double> [[TMP36]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <8 x double> [[TMP37]], <8 x double> [[TMP38]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: ret <16 x double> [[TMP39]]
;
- %A = load <16 x double>, <16 x double>* %A.Ptr, align 8
+ %A = load <16 x double>, ptr %A.Ptr, align 8
%A.trans = tail call <16 x double> @llvm.matrix.transpose.v16f64(<16 x double> %A, i32 4, i32 4)
- %B = load <16 x double>, <16 x double>* %B.Ptr, align 8
+ %B = load <16 x double>, ptr %B.Ptr, align 8
%Mul = fmul <16 x double> %A, %B
ret <16 x double> %Mul
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll
index d0f9ee7ec1650..aaabd18958fae 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-inlining.ll
@@ -92,20 +92,20 @@ target triple = "aarch64-apple-ios"
; CHECK-LABEL: remark: transpose.h:13:11: Lowered with 0 stores, 0 loads, 4 compute ops, 1 exposed transposes
; CHECK-NEXT: transpose.2x1.float(addr %D)
-define void @toplevel(<15 x double>* %A, double* %B, <15 x double>* %C, <2 x float>* %D) !dbg !16 {
+define void @toplevel(ptr %A, ptr %B, ptr %C, ptr %D) !dbg !16 {
entry:
- %a = load <15 x double>, <15 x double> *%A, align 16, !dbg !3791
- %b = call <15 x double> @llvm.matrix.column.major.load(double* %B, i64 5, i1 false, i32 3, i32 5), !dbg !3793
+ %a = load <15 x double>, ptr %A, align 16, !dbg !3791
+ %b = call <15 x double> @llvm.matrix.column.major.load(ptr %B, i64 5, i1 false, i32 3, i32 5), !dbg !3793
%c = fadd <15 x double> %a, %b, !dbg !100
- store <15 x double> %c, <15 x double> *%C, align 16, !dbg !102
+ store <15 x double> %c, ptr %C, align 16, !dbg !102
- %load = load <2 x float>, <2 x float>* %D, !dbg !104
+ %load = load <2 x float>, ptr %D, !dbg !104
%t1 = call <2 x float> @llvm.matrix.transpose(<2 x float> %load, i32 2, i32 1), !dbg !106
- store <2 x float> %t1, <2 x float>* %D, !dbg !108
+ store <2 x float> %t1, ptr %D, !dbg !108
ret void
}
-declare <15 x double> @llvm.matrix.column.major.load(double*, i64, i1, i32, i32)
+declare <15 x double> @llvm.matrix.column.major.load(ptr, i64, i1, i32, i32)
declare <2 x float> @llvm.matrix.transpose(<2 x float>, i32, i32)
!llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
index 2395d5cdc9b67..0fcbdcc43a447 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks-shared-subtrees.ll
@@ -89,25 +89,25 @@
; STDERR-NEXT: column.major.load.2x15.double(addr %arg3, scalar))),
; STDERR-NEXT: addr %arg2,
; STDERR-NEXT: 10)
-define void @test_2leafs(double* %arg1, double* %arg2, double* %arg3, i64 %stride) !dbg !8 {
+define void @test_2leafs(ptr %arg1, ptr %arg2, ptr %arg3, i64 %stride) !dbg !8 {
bb:
- %shared.load = tail call <8 x double> @llvm.matrix.column.major.load.v8f64.p0f64(double* %arg1, i64 %stride, i1 false, i32 2, i32 4), !dbg !10
- %shared.load.2 = tail call <30 x double> @llvm.matrix.column.major.load.v30f64.p0f64(double* %arg3, i64 %stride, i1 false, i32 2, i32 15), !dbg !10
+ %shared.load = tail call <8 x double> @llvm.matrix.column.major.load.v8f64.p0(ptr %arg1, i64 %stride, i1 false, i32 2, i32 4), !dbg !10
+ %shared.load.2 = tail call <30 x double> @llvm.matrix.column.major.load.v30f64.p0(ptr %arg3, i64 %stride, i1 false, i32 2, i32 15), !dbg !10
%tmp17 = tail call <8 x double> @llvm.matrix.transpose.v8f64(<8 x double> %shared.load, i32 2, i32 4), !dbg !10
- tail call void @llvm.matrix.column.major.store.v8f64.p0f64(<8 x double> %tmp17, double* %arg3, i64 10, i1 false, i32 4, i32 2), !dbg !10
- %tmp18 = tail call <60 x double> @llvm.matrix.column.major.load.v60f64.p0f64(double* %arg2, i64 20, i1 false, i32 4, i32 15), !dbg !11
+ tail call void @llvm.matrix.column.major.store.v8f64.p0(<8 x double> %tmp17, ptr %arg3, i64 10, i1 false, i32 4, i32 2), !dbg !10
+ %tmp18 = tail call <60 x double> @llvm.matrix.column.major.load.v60f64.p0(ptr %arg2, i64 20, i1 false, i32 4, i32 15), !dbg !11
%tmp48 = tail call <60 x double> @llvm.matrix.multiply.v60f64.v8f64.v30f64(<8 x double> %tmp17, <30 x double> %shared.load.2, i32 4, i32 2, i32 15), !dbg !11
%tmp49 = fsub <60 x double> %tmp18, %tmp48, !dbg !11
- tail call void @llvm.matrix.column.major.store.v60f64.p0f64(<60 x double> %tmp49, double* %arg2, i64 10, i1 false, i32 4, i32 15), !dbg !11
+ tail call void @llvm.matrix.column.major.store.v60f64.p0(<60 x double> %tmp49, ptr %arg2, i64 10, i1 false, i32 4, i32 15), !dbg !11
ret void
}
declare <8 x double> @llvm.matrix.transpose.v8f64(<8 x double>, i32 immarg, i32 immarg)
-declare <8 x double> @llvm.matrix.column.major.load.v8f64.p0f64(double*, i64, i1 immarg, i32 immarg, i32 immarg)
-declare <30 x double> @llvm.matrix.column.major.load.v30f64.p0f64(double*, i64, i1 immarg, i32 immarg, i32 immarg)
-declare <60 x double> @llvm.matrix.column.major.load.v60f64.p0f64(double*, i64, i1 immarg, i32 immarg, i32 immarg)
-declare void @llvm.matrix.column.major.store.v60f64.p0f64(<60 x double>, double* writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
-declare void @llvm.matrix.column.major.store.v8f64.p0f64(<8 x double>, double* writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
+declare <8 x double> @llvm.matrix.column.major.load.v8f64.p0(ptr, i64, i1 immarg, i32 immarg, i32 immarg)
+declare <30 x double> @llvm.matrix.column.major.load.v30f64.p0(ptr, i64, i1 immarg, i32 immarg, i32 immarg)
+declare <60 x double> @llvm.matrix.column.major.load.v60f64.p0(ptr, i64, i1 immarg, i32 immarg, i32 immarg)
+declare void @llvm.matrix.column.major.store.v60f64.p0(<60 x double>, ptr writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
+declare void @llvm.matrix.column.major.store.v8f64.p0(<8 x double>, ptr writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
declare <60 x double> @llvm.matrix.multiply.v60f64.v8f64.v30f64(<8 x double>, <30 x double>, i32 immarg, i32 immarg, i32 immarg)
!llvm.module.flags = !{!0, !1, !2, !3}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll
index 4c067023cb101..628ff08b81679 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/remarks.ll
@@ -8,10 +8,10 @@
; CHECK-NEXT: store(
; CHECK-NEXT: transpose.2x6.double(load(addr %A)),
; CHECK-NEXT: addr %B)
-define void @transpose(<12 x double>* %A, <12 x double>* %B) !dbg !23 {
- %load = load <12 x double>, <12 x double>* %A, !dbg !24
+define void @transpose(ptr %A, ptr %B) !dbg !23 {
+ %load = load <12 x double>, ptr %A, !dbg !24
%t = call <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double> %load, i32 2, i32 6), !dbg !24
- store <12 x double> %t, <12 x double>* %B, !dbg !24
+ store <12 x double> %t, ptr %B, !dbg !24
ret void
}
@@ -21,11 +21,11 @@ define void @transpose(<12 x double>* %A, <12 x double>* %B) !dbg !23 {
; CHECK-NEXT: load(addr %A),
; CHECK-NEXT: load(addr %B)),
; CHECK-NEXT: addr %C)
-define void @multiply(<12 x double>* %A, <12 x double>* %B, <4 x double>* %C) !dbg !25 {
- %A.matrix = load <12 x double>, <12 x double>* %A, !dbg !26
- %B.matrix = load <12 x double>, <12 x double>* %B, !dbg !26
+define void @multiply(ptr %A, ptr %B, ptr %C) !dbg !25 {
+ %A.matrix = load <12 x double>, ptr %A, !dbg !26
+ %B.matrix = load <12 x double>, ptr %B, !dbg !26
%t = call <4 x double> @llvm.matrix.multiply(<12 x double> %A.matrix, <12 x double> %B.matrix, i32 2, i32 6, i32 2), !dbg !26
- store <4 x double> %t, <4 x double>* %C, !dbg !26
+ store <4 x double> %t, ptr %C, !dbg !26
ret void
}
@@ -33,9 +33,9 @@ define void @multiply(<12 x double>* %A, <12 x double>* %B, <4 x double>* %C) !d
; CHECK-NEXT: store(
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5),
; CHECK-NEXT: addr %B)
-define void @column.major.load(double* %A, <9 x double>* %B) !dbg !27 {
- %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !28
- store <9 x double> %A.matrix, <9 x double>* %B, !dbg !28
+define void @column.major.load(ptr %A, ptr %B) !dbg !27 {
+ %A.matrix = call <9 x double> @llvm.matrix.column.major.load(ptr %A, i64 5, i1 false, i32 3, i32 3), !dbg !28
+ store <9 x double> %A.matrix, ptr %B, !dbg !28
ret void
}
@@ -44,9 +44,9 @@ define void @column.major.load(double* %A, <9 x double>* %B) !dbg !27 {
; CHECK-NEXT: column.major.load.3x3.double(addr %A, 5),
; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10)
-define void @column.major.store(double* %A, double* %B) !dbg !29 {
- %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !30
- call void @llvm.matrix.column.major.store(<9 x double> %A.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !30
+define void @column.major.store(ptr %A, ptr %B) !dbg !29 {
+ %A.matrix = call <9 x double> @llvm.matrix.column.major.load(ptr %A, i64 5, i1 false, i32 3, i32 3), !dbg !30
+ call void @llvm.matrix.column.major.store(<9 x double> %A.matrix, ptr %B, i64 10, i1 false, i32 3, i32 3), !dbg !30
ret void
}
@@ -60,11 +60,11 @@ define void @column.major.store(double* %A, double* %B) !dbg !29 {
; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10)
-define void @binaryops(double* %A, double* %B) !dbg !31 {
- %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !32
+define void @binaryops(ptr %A, ptr %B) !dbg !31 {
+ %A.matrix = call <9 x double> @llvm.matrix.column.major.load(ptr %A, i64 5, i1 false, i32 3, i32 3), !dbg !32
%R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !32
%R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !32
- call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !32
+ call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, ptr %B, i64 10, i1 false, i32 3, i32 3), !dbg !32
ret void
}
@@ -84,16 +84,16 @@ define void @binaryops(double* %A, double* %B) !dbg !31 {
; CHECK-NEXT: load(addr %D)),
; CHECK-NEXT: addr %E)
-define void @multiple_expressions(double* %A, double* %B, <12 x double>* %C, <12 x double>* %D, <4 x double>* %E) !dbg !33 {
- %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !34
+define void @multiple_expressions(ptr %A, ptr %B, ptr %C, ptr %D, ptr %E) !dbg !33 {
+ %A.matrix = call <9 x double> @llvm.matrix.column.major.load(ptr %A, i64 5, i1 false, i32 3, i32 3), !dbg !34
%R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !34
%R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !34
- call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !34
+ call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, ptr %B, i64 10, i1 false, i32 3, i32 3), !dbg !34
- %C.matrix = load <12 x double>, <12 x double>* %C, !dbg !34
- %D.matrix = load <12 x double>, <12 x double>* %D, !dbg !34
+ %C.matrix = load <12 x double>, ptr %C, !dbg !34
+ %D.matrix = load <12 x double>, ptr %D, !dbg !34
%Mult.matrix = call <4 x double> @llvm.matrix.multiply(<12 x double> %C.matrix, <12 x double> %D.matrix, i32 2, i32 6, i32 2), !dbg !34
- store <4 x double> %Mult.matrix, <4 x double>* %E, !dbg !34
+ store <4 x double> %Mult.matrix, ptr %E, !dbg !34
ret void
}
@@ -107,11 +107,11 @@ define void @multiple_expressions(double* %A, double* %B, <12 x double>* %C, <12
; CHECK-NEXT: (reused) column.major.load.3x3.double(addr %A, 5)),
; CHECK-NEXT: addr %B,
; CHECK-NEXT: 10)
-define void @stackaddresses(double* %A, double* %B) !dbg !35 {
- %A.matrix = call <9 x double> @llvm.matrix.column.major.load(double* %A, i64 5, i1 false, i32 3, i32 3), !dbg !36
+define void @stackaddresses(ptr %A, ptr %B) !dbg !35 {
+ %A.matrix = call <9 x double> @llvm.matrix.column.major.load(ptr %A, i64 5, i1 false, i32 3, i32 3), !dbg !36
%R1.matrix = fadd <9 x double> %A.matrix, %A.matrix, !dbg !36
%R2.matrix = fmul <9 x double> %R1.matrix, %A.matrix, !dbg !36
- call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, double* %B, i64 10, i1 false, i32 3, i32 3), !dbg !36
+ call void @llvm.matrix.column.major.store(<9 x double> %R2.matrix, ptr %B, i64 10, i1 false, i32 3, i32 3), !dbg !36
ret void
}
@@ -119,28 +119,25 @@ define void @stackaddresses(double* %A, double* %B) !dbg !35 {
; CHECK-NEXT: store(
; CHECK-NEXT: transpose.5x3.double(load(addr %A)),
; CHECK-NEXT: stack addr %s1)
-%S1 = type {<15 x double>*}
-define void @get_underlying_object(%S1* %A) !dbg !21 {
+%S1 = type {ptr}
+define void @get_underlying_object(ptr %A) !dbg !21 {
entry:
%s1 = alloca <15 x double>, !dbg !22
- %a1 = getelementptr %S1, %S1* %A, i32 0, i32 0, !dbg !22
- %a2 = load <15 x double>*, <15 x double>** %a1, !dbg !22
- %av = load <15 x double>, <15 x double>* %a2, !dbg !22
+ %a2 = load ptr, ptr %A, !dbg !22
+ %av = load <15 x double>, ptr %a2, !dbg !22
- %s2 = bitcast <15 x double>* %s1 to i64*, !dbg !22
- %s3 = bitcast i64* %s2 to <15 x double>*, !dbg !22
%t = call <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double> %av, i32 5, i32 3), !dbg !22
- store <15 x double> %t, <15 x double>* %s3, !dbg !22
+ store <15 x double> %t, ptr %s1, !dbg !22
ret void
}
declare <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double>, i32, i32)
declare <4 x double> @llvm.matrix.multiply(<12 x double>, <12 x double>, i32, i32, i32)
-declare <9 x double> @llvm.matrix.column.major.load(double*, i64, i1, i32, i32)
+declare <9 x double> @llvm.matrix.column.major.load(ptr, i64, i1, i32, i32)
declare <15 x double> @llvm.matrix.transpose.v15f64.v15f64(<15 x double>, i32, i32)
-declare void @llvm.matrix.column.major.store(<9 x double>, double*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store(<9 x double>, ptr, i64, i1, i32, i32)
!llvm.dbg.cu = !{!0}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/store-align-volatile.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/store-align-volatile.ll
index 7b36b098f14c7..02071f4b066ed 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/store-align-volatile.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/store-align-volatile.ll
@@ -1,106 +1,91 @@
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define void @strided_store_volatile(<6 x i32> %in, i32* %out) {
+define void @strided_store_volatile(<6 x i32> %in, ptr %out) {
; CHECK-LABEL: @strided_store_volatile(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[OUT:%.*]] to <3 x i32>*
-; CHECK-NEXT: store volatile <3 x i32> [[SPLIT]], <3 x i32>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT]], i64 5
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
-; CHECK-NEXT: store volatile <3 x i32> [[SPLIT1]], <3 x i32>* [[VEC_CAST2]], align 4
+; CHECK-NEXT: store volatile <3 x i32> [[SPLIT]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT]], i64 5
+; CHECK-NEXT: store volatile <3 x i32> [[SPLIT1]], ptr [[VEC_GEP]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x i32> %in, i32* %out, i64 5, i1 true, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x i32> %in, ptr %out, i64 5, i1 true, i32 3, i32 2)
ret void
}
-declare void @llvm.matrix.column.major.store(<6 x i32>, i32*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store(<6 x i32>, ptr, i64, i1, i32, i32)
-define void @multiply_store_volatile(<4 x i32> %in, <4 x i32>* %out) {
+define void @multiply_store_volatile(<4 x i32> %in, ptr %out) {
; CHECK-LABEL: @multiply_store_volatile(
-; CHECK: [[TMP29:%.*]] = bitcast <4 x i32>* %out to i32*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[TMP29]] to <2 x i32>*
-; CHECK-NEXT: store volatile <2 x i32> {{.*}}, <2 x i32>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[TMP29]], i64 2
-; CHECK-NEXT: [[VEC_CAST25:%.*]] = bitcast i32* [[VEC_GEP]] to <2 x i32>*
-; CHECK-NEXT: store volatile <2 x i32> {{.*}}, <2 x i32>* [[VEC_CAST25]], align 4
+; CHECK: store volatile <2 x i32> {{.*}}, ptr %out, align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr %out, i64 2
+; CHECK-NEXT: store volatile <2 x i32> {{.*}}, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: ret void
;
%res = call <4 x i32> @llvm.matrix.multiply(<4 x i32> %in, <4 x i32> %in, i32 2, i32 2, i32 2)
- store volatile <4 x i32> %res, <4 x i32>* %out, align 4
+ store volatile <4 x i32> %res, ptr %out, align 4
ret void
}
declare <4 x i32> @llvm.matrix.multiply(<4 x i32>, <4 x i32>, i32, i32, i32)
-define void @strided_store_align32(<6 x i32> %in, i64 %stride, i32* %out) {
+define void @strided_store_align32(<6 x i32> %in, i64 %stride, ptr %out) {
; CHECK-LABEL: @strided_store_align32(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
-; CHECK-NEXT: store volatile <3 x i32> [[SPLIT]], <3 x i32>* [[VEC_CAST]], align 32
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: store volatile <3 x i32> [[SPLIT]], ptr [[VEC_GEP]], align 32
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, i32* [[OUT]], i64 [[VEC_START2]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast i32* [[VEC_GEP3]] to <3 x i32>*
-; CHECK-NEXT: store volatile <3 x i32> [[SPLIT1]], <3 x i32>* [[VEC_CAST4]], align 4
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[VEC_START2]]
+; CHECK-NEXT: store volatile <3 x i32> [[SPLIT1]], ptr [[VEC_GEP3]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x i32> %in, i32* align 32 %out, i64 %stride, i1 true, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x i32> %in, ptr align 32 %out, i64 %stride, i1 true, i32 3, i32 2)
ret void
}
-define void @strided_store_align2(<6 x i32> %in, i64 %stride, i32* %out) {
+define void @strided_store_align2(<6 x i32> %in, i64 %stride, ptr %out) {
; CHECK-LABEL: @strided_store_align2(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
-; CHECK-NEXT: store volatile <3 x i32> [[SPLIT]], <3 x i32>* [[VEC_CAST]], align 2
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: store volatile <3 x i32> [[SPLIT]], ptr [[VEC_GEP]], align 2
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, i32* [[OUT]], i64 [[VEC_START2]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast i32* [[VEC_GEP3]] to <3 x i32>*
-; CHECK-NEXT: store volatile <3 x i32> [[SPLIT1]], <3 x i32>* [[VEC_CAST4]], align 2
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[VEC_START2]]
+; CHECK-NEXT: store volatile <3 x i32> [[SPLIT1]], ptr [[VEC_GEP3]], align 2
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x i32> %in, i32* align 2 %out, i64 %stride, i1 true, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x i32> %in, ptr align 2 %out, i64 %stride, i1 true, i32 3, i32 2)
ret void
}
-define void @multiply_store_align16_stride8(<4 x i32> %in, <4 x i32>* %out) {
+define void @multiply_store_align16_stride8(<4 x i32> %in, ptr %out) {
; CHECK-LABEL: @multiply_store_align16_stride8(
-; CHECK: [[TMP29:%.*]] = bitcast <4 x i32>* %out to i32*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[TMP29]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> {{.*}}, <2 x i32>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[TMP29]], i64 2
-; CHECK-NEXT: [[VEC_CAST25:%.*]] = bitcast i32* [[VEC_GEP]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> {{.*}}, <2 x i32>* [[VEC_CAST25]], align 8
+; CHECK: store <2 x i32> {{.*}}, ptr %out, align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr %out, i64 2
+; CHECK-NEXT: store <2 x i32> {{.*}}, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: ret void
;
%res = call <4 x i32> @llvm.matrix.multiply(<4 x i32> %in, <4 x i32> %in, i32 2, i32 2, i32 2)
- store <4 x i32> %res, <4 x i32>* %out, align 16
+ store <4 x i32> %res, ptr %out, align 16
ret void
}
-define void @strided_store_align8_stride12(<6 x i32> %in, i32* %out) {
+define void @strided_store_align8_stride12(<6 x i32> %in, ptr %out) {
; CHECK-LABEL: @strided_store_align8_stride12(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[OUT:%.*]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT]], <2 x i32>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT]], i64 3
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT1]], <2 x i32>* [[VEC_CAST3]], align 4
-; CHECK-NEXT: [[VEC_GEP4:%.*]] = getelementptr i32, i32* [[OUT]], i64 6
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast i32* [[VEC_GEP4]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT2]], <2 x i32>* [[VEC_CAST5]], align 8
+; CHECK-NEXT: store <2 x i32> [[SPLIT]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT]], i64 3
+; CHECK-NEXT: store <2 x i32> [[SPLIT1]], ptr [[VEC_GEP]], align 4
+; CHECK-NEXT: [[VEC_GEP4:%.*]] = getelementptr i32, ptr [[OUT]], i64 6
+; CHECK-NEXT: store <2 x i32> [[SPLIT2]], ptr [[VEC_GEP4]], align 8
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x i32> %in, i32* align 8 %out, i64 3, i1 false, i32 2, i32 3)
+ call void @llvm.matrix.column.major.store(<6 x i32> %in, ptr align 8 %out, i64 3, i1 false, i32 2, i32 3)
ret void
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
index 2023a1f164979..ab600462aead9 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-double.ll
@@ -1,85 +1,77 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define <9 x double> @strided_load_3x3(double* %in, i64 %stride) {
+define <9 x double> @strided_load_3x3(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[IN]], i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST3]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN]], i64 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[IN]], i64 [[VEC_START5]]
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, ptr [[IN]], i64 [[VEC_START5]]
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP6]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x double> [[COL_LOAD]], <3 x double> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <6 x double> [[TMP0]], <6 x double> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: ret <9 x double> [[TMP2]]
;
entry:
- %load = call <9 x double> @llvm.matrix.column.major.load.v9f64.i64(double* %in, i64 %stride, i1 false, i32 3, i32 3)
+ %load = call <9 x double> @llvm.matrix.column.major.load.v9f64.i64(ptr %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x double> %load
}
-declare <9 x double> @llvm.matrix.column.major.load.v9f64.i64(double*, i64, i1, i32, i32)
+declare <9 x double> @llvm.matrix.column.major.load.v9f64.i64(ptr, i64, i1, i32, i32)
-define <9 x double> @strided_load_9x1(double* %in, i64 %stride) {
+define <9 x double> @strided_load_9x1(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_9x1(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <9 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x double>, <9 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: ret <9 x double> [[COL_LOAD]]
;
entry:
- %load = call <9 x double> @llvm.matrix.column.major.load.v9f64.i64(double* %in, i64 %stride, i1 false, i32 9, i32 1)
+ %load = call <9 x double> @llvm.matrix.column.major.load.v9f64.i64(ptr %in, i64 %stride, i1 false, i32 9, i32 1)
ret <9 x double> %load
}
-declare <8 x double> @llvm.matrix.column.major.load.v8f64.i64(double*, i64, i1, i32, i32)
+declare <8 x double> @llvm.matrix.column.major.load.v8f64.i64(ptr, i64, i1, i32, i32)
-define <8 x double> @strided_load_4x2(double* %in, i64 %stride) {
+define <8 x double> @strided_load_4x2(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_4x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[IN]], i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST3]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN]], i64 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x double> [[COL_LOAD]], <4 x double> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x double> [[TMP0]]
;
entry:
- %load = call <8 x double> @llvm.matrix.column.major.load.v8f64.i64(double* %in, i64 %stride, i1 false, i32 4, i32 2)
+ %load = call <8 x double> @llvm.matrix.column.major.load.v8f64.i64(ptr %in, i64 %stride, i1 false, i32 4, i32 2)
ret <8 x double> %load
}
-declare <8 x double> @llvm.matrix.column.major.load.v8f64.i32(double*, i32, i1, i32, i32)
+declare <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr, i32, i1, i32, i32)
-define <8 x double> @strided_load_4x2_stride_i32(double* %in, i32 %stride) {
+define <8 x double> @strided_load_4x2_stride_i32(ptr %in, i32 %stride) {
; CHECK-LABEL: @strided_load_4x2_stride_i32(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i32 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[IN:%.*]], i32 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN:%.*]], i32 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x double>, ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i32 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, double* [[IN]], i32 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast double* [[VEC_GEP2]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST3]], align 8
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN]], i32 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x double>, ptr [[VEC_GEP2]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x double> [[COL_LOAD]], <4 x double> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x double> [[TMP0]]
;
entry:
- %load = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(double* %in, i32 %stride, i1 false, i32 4, i32 2)
+ %load = call <8 x double> @llvm.matrix.column.major.load.v8f64.i32(ptr %in, i32 %stride, i1 false, i32 4, i32 2)
ret <8 x double> %load
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
index e9740bf7522ab..6e54684fbc2c9 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-float.ll
@@ -1,64 +1,58 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define <9 x float> @strided_load_3x3(float* %in, i64 %stride) {
+define <9 x float> @strided_load_3x3(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <3 x float>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x float>, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, float* [[IN]], i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast float* [[VEC_GEP2]] to <3 x float>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST3]], align 4
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, ptr [[IN]], i64 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x float>, ptr [[VEC_GEP2]], align 4
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr float, float* [[IN]], i64 [[VEC_START5]]
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast float* [[VEC_GEP6]] to <3 x float>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x float>, <3 x float>* [[VEC_CAST7]], align 4
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr float, ptr [[IN]], i64 [[VEC_START5]]
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x float>, ptr [[VEC_GEP6]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x float> [[COL_LOAD]], <3 x float> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x float> [[COL_LOAD8]], <3 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x float> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: ret <9 x float> [[TMP2]]
;
entry:
- %load = call <9 x float> @llvm.matrix.column.major.load(float* %in, i64 %stride, i1 false, i32 3, i32 3)
+ %load = call <9 x float> @llvm.matrix.column.major.load(ptr %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x float> %load
}
-declare <9 x float> @llvm.matrix.column.major.load(float*, i64, i1, i32, i32)
+declare <9 x float> @llvm.matrix.column.major.load(ptr, i64, i1, i32, i32)
-define <9 x float> @strided_load_9x1(float* %in, i64 %stride) {
+define <9 x float> @strided_load_9x1(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_9x1(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <9 x float>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x float>, <9 x float>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x float>, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: ret <9 x float> [[COL_LOAD]]
;
entry:
- %load = call <9 x float> @llvm.matrix.column.major.load(float* %in, i64 %stride, i1 false, i32 9, i32 1)
+ %load = call <9 x float> @llvm.matrix.column.major.load(ptr %in, i64 %stride, i1 false, i32 9, i32 1)
ret <9 x float> %load
}
-declare <8 x float> @llvm.matrix.column.major.load.v8f32(float*, i64, i1, i32, i32)
+declare <8 x float> @llvm.matrix.column.major.load.v8f32(ptr, i64, i1, i32, i32)
-define <8 x float> @strided_load_4x2(float* %in, i64 %stride) {
+define <8 x float> @strided_load_4x2(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_4x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <4 x float>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x float>, <4 x float>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x float>, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, float* [[IN]], i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast float* [[VEC_GEP2]] to <4 x float>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x float>, <4 x float>* [[VEC_CAST3]], align 4
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr float, ptr [[IN]], i64 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x float>, ptr [[VEC_GEP2]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x float> [[COL_LOAD]], <4 x float> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x float> [[TMP0]]
;
entry:
- %load = call <8 x float> @llvm.matrix.column.major.load.v8f32(float* %in, i64 %stride, i1 false, i32 4, i32 2)
+ %load = call <8 x float> @llvm.matrix.column.major.load.v8f32(ptr %in, i64 %stride, i1 false, i32 4, i32 2)
ret <8 x float> %load
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
index 5dd042f605fd3..baf6cb7853fa2 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-load-i32.ll
@@ -1,64 +1,58 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define <9 x i32> @strided_load_3x3(i32* %in, i64 %stride) {
+define <9 x i32> @strided_load_3x3(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_3x3(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x i32>, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, i32* [[IN]], i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP2]] to <3 x i32>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST3]], align 4
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[IN]], i64 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <3 x i32>, ptr [[VEC_GEP2]], align 4
; CHECK-NEXT: [[VEC_START5:%.*]] = mul i64 2, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i32, i32* [[IN]], i64 [[VEC_START5]]
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast i32* [[VEC_GEP6]] to <3 x i32>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x i32>, <3 x i32>* [[VEC_CAST7]], align 4
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i32, ptr [[IN]], i64 [[VEC_START5]]
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <3 x i32>, ptr [[VEC_GEP6]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x i32> [[COL_LOAD]], <3 x i32> [[COL_LOAD4]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x i32> [[COL_LOAD8]], <3 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <6 x i32> [[TMP0]], <6 x i32> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: ret <9 x i32> [[TMP2]]
;
entry:
- %load = call <9 x i32> @llvm.matrix.column.major.load(i32* %in, i64 %stride, i1 false, i32 3, i32 3)
+ %load = call <9 x i32> @llvm.matrix.column.major.load(ptr %in, i64 %stride, i1 false, i32 3, i32 3)
ret <9 x i32> %load
}
-declare <9 x i32> @llvm.matrix.column.major.load(i32*, i64, i1, i32, i32)
+declare <9 x i32> @llvm.matrix.column.major.load(ptr, i64, i1, i32, i32)
-define <9 x i32> @strided_load_9x1(i32* %in, i64 %stride) {
+define <9 x i32> @strided_load_9x1(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_9x1(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <9 x i32>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x i32>, <9 x i32>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <9 x i32>, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: ret <9 x i32> [[COL_LOAD]]
;
entry:
- %load = call <9 x i32> @llvm.matrix.column.major.load(i32* %in, i64 %stride, i1 false, i32 9, i32 1)
+ %load = call <9 x i32> @llvm.matrix.column.major.load(ptr %in, i64 %stride, i1 false, i32 9, i32 1)
ret <9 x i32> %load
}
-declare <8 x i32> @llvm.matrix.column.major.load.v8i32(i32*, i64, i1, i32, i32)
+declare <8 x i32> @llvm.matrix.column.major.load.v8i32(ptr, i64, i1, i32, i32)
-define <8 x i32> @strided_load_4x2(i32* %in, i64 %stride) {
+define <8 x i32> @strided_load_4x2(ptr %in, i64 %stride) {
; CHECK-LABEL: @strided_load_4x2(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[IN:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <4 x i32>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[IN:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <4 x i32>, ptr [[VEC_GEP]], align 4
; CHECK-NEXT: [[VEC_START1:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, i32* [[IN]], i64 [[VEC_START1]]
-; CHECK-NEXT: [[VEC_CAST3:%.*]] = bitcast i32* [[VEC_GEP2]] to <4 x i32>*
-; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x i32>, <4 x i32>* [[VEC_CAST3]], align 4
+; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[IN]], i64 [[VEC_START1]]
+; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <4 x i32>, ptr [[VEC_GEP2]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[COL_LOAD]], <4 x i32> [[COL_LOAD4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: ret <8 x i32> [[TMP0]]
;
entry:
- %load = call <8 x i32> @llvm.matrix.column.major.load.v8i32(i32* %in, i64 %stride, i1 false, i32 4, i32 2)
+ %load = call <8 x i32> @llvm.matrix.column.major.load.v8i32(ptr %in, i64 %stride, i1 false, i32 4, i32 2)
ret <8 x i32> %load
}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
index e5dec5d06eca3..80e6df557b81b 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-double.ll
@@ -1,88 +1,77 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define void @strided_store_3x2(<6 x double> %in, double* %out) {
+define void @strided_store_3x2(<6 x double> %in, ptr %out) {
; CHECK-LABEL: @strided_store_3x2(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[OUT:%.*]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT]], i64 5
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST2]], align 8
+; CHECK-NEXT: store <3 x double> [[SPLIT]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[OUT]], i64 5
+; CHECK-NEXT: store <3 x double> [[SPLIT1]], ptr [[VEC_GEP]], align 8
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store.v6f64.i64(<6 x double> %in, double* %out, i64 5, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store.v6f64.i64(<6 x double> %in, ptr %out, i64 5, i1 false, i32 3, i32 2)
ret void
}
-define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i64 %stride, double* %out) {
+define void @strided_store_3x2_nonconst_stride(<6 x double> %in, i64 %stride, ptr %out) {
; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[OUT:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: store <3 x double> [[SPLIT]], ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[OUT]], i64 [[VEC_START2]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[OUT]], i64 [[VEC_START2]]
+; CHECK-NEXT: store <3 x double> [[SPLIT1]], ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store.v6f64.i64(<6 x double> %in, double* %out, i64 %stride, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store.v6f64.i64(<6 x double> %in, ptr %out, i64 %stride, i1 false, i32 3, i32 2)
ret void
}
-define void @strided_store_3x2_nonconst_i32_stride(<6 x double> %in, i32 %stride, double* %out) {
+define void @strided_store_3x2_nonconst_i32_stride(<6 x double> %in, i32 %stride, ptr %out) {
; CHECK-LABEL: @strided_store_3x2_nonconst_i32_stride(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x double> [[IN:%.*]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x double> [[IN]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i32 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT:%.*]], i32 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[SPLIT]], <3 x double>* [[VEC_CAST]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[OUT:%.*]], i32 [[VEC_START]]
+; CHECK-NEXT: store <3 x double> [[SPLIT]], ptr [[VEC_GEP]], align 8
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i32 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[OUT]], i32 [[VEC_START2]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[SPLIT1]], <3 x double>* [[VEC_CAST4]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[OUT]], i32 [[VEC_START2]]
+; CHECK-NEXT: store <3 x double> [[SPLIT1]], ptr [[VEC_GEP3]], align 8
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store.v6f64.i32(<6 x double> %in, double* %out, i32 %stride, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store.v6f64.i32(<6 x double> %in, ptr %out, i32 %stride, i1 false, i32 3, i32 2)
ret void
}
-define void @strided_store_2x3(<10 x double> %in, double* %out) {
+define void @strided_store_2x3(<10 x double> %in, ptr %out) {
; CHECK-LABEL: @strided_store_2x3(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <10 x double> [[IN:%.*]], <10 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> poison, <2 x i32> <i32 6, i32 7>
; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <10 x double> [[IN]], <10 x double> poison, <2 x i32> <i32 8, i32 9>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[OUT:%.*]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[SPLIT]], <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[OUT]], i64 4
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[SPLIT1]], <2 x double>* [[VEC_CAST5]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[OUT]], i64 8
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[SPLIT2]], <2 x double>* [[VEC_CAST7]], align 8
-; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, double* [[OUT]], i64 12
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[SPLIT3]], <2 x double>* [[VEC_CAST9]], align 8
-; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr double, double* [[OUT]], i64 16
-; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast double* [[VEC_GEP10]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[SPLIT4]], <2 x double>* [[VEC_CAST11]], align 8
+; CHECK-NEXT: store <2 x double> [[SPLIT]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[OUT]], i64 4
+; CHECK-NEXT: store <2 x double> [[SPLIT1]], ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, ptr [[OUT]], i64 8
+; CHECK-NEXT: store <2 x double> [[SPLIT2]], ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, ptr [[OUT]], i64 12
+; CHECK-NEXT: store <2 x double> [[SPLIT3]], ptr [[VEC_GEP6]], align 8
+; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[OUT]], i64 16
+; CHECK-NEXT: store <2 x double> [[SPLIT4]], ptr [[VEC_GEP7]], align 8
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store.v10f64.i64(<10 x double> %in, double* %out, i64 4, i1 false, i32 2, i32 5)
+ call void @llvm.matrix.column.major.store.v10f64.i64(<10 x double> %in, ptr %out, i64 4, i1 false, i32 2, i32 5)
ret void
}
-declare void @llvm.matrix.column.major.store.v6f64.i64(<6 x double>, double*, i64, i1, i32, i32)
-declare void @llvm.matrix.column.major.store.v6f64.i32(<6 x double>, double*, i32, i1, i32, i32)
-declare void @llvm.matrix.column.major.store.v10f64.i64(<10 x double>, double*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store.v6f64.i64(<6 x double>, ptr, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store.v6f64.i32(<6 x double>, ptr, i32, i1, i32, i32)
+declare void @llvm.matrix.column.major.store.v10f64.i64(<10 x double>, ptr, i64, i1, i32, i32)
-; CHECK: declare void @llvm.matrix.column.major.store.v6f64.i64(<6 x double>, double* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
-; CHECK: declare void @llvm.matrix.column.major.store.v10f64.i64(<10 x double>, double* nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
+; CHECK: declare void @llvm.matrix.column.major.store.v6f64.i64(<6 x double>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
+; CHECK: declare void @llvm.matrix.column.major.store.v10f64.i64(<10 x double>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) #0
; CHECK: attributes #0 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
index 11211469e58ed..40ab8692501d2 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-float.ll
@@ -1,67 +1,58 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define void @strided_store_3x2(<6 x float> %in, float* %out) {
+define void @strided_store_3x2(<6 x float> %in, ptr %out) {
; CHECK-LABEL: @strided_store_3x2(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x float> [[IN:%.*]], <6 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x float> [[IN]], <6 x float> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[OUT:%.*]] to <3 x float>*
-; CHECK-NEXT: store <3 x float> [[SPLIT]], <3 x float>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[OUT]], i64 5
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast float* [[VEC_GEP]] to <3 x float>*
-; CHECK-NEXT: store <3 x float> [[SPLIT1]], <3 x float>* [[VEC_CAST2]], align 4
+; CHECK-NEXT: store <3 x float> [[SPLIT]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[OUT]], i64 5
+; CHECK-NEXT: store <3 x float> [[SPLIT1]], ptr [[VEC_GEP]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x float> %in, float* %out, i64 5, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x float> %in, ptr %out, i64 5, i1 false, i32 3, i32 2)
ret void
}
-define void @strided_store_3x2_nonconst_stride(<6 x float> %in, i64 %stride, float* %out) {
+define void @strided_store_3x2_nonconst_stride(<6 x float> %in, i64 %stride, ptr %out) {
; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x float> [[IN:%.*]], <6 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x float> [[IN]], <6 x float> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[OUT:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[VEC_GEP]] to <3 x float>*
-; CHECK-NEXT: store <3 x float> [[SPLIT]], <3 x float>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[OUT:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: store <3 x float> [[SPLIT]], ptr [[VEC_GEP]], align 4
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, float* [[OUT]], i64 [[VEC_START2]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast float* [[VEC_GEP3]] to <3 x float>*
-; CHECK-NEXT: store <3 x float> [[SPLIT1]], <3 x float>* [[VEC_CAST4]], align 4
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, ptr [[OUT]], i64 [[VEC_START2]]
+; CHECK-NEXT: store <3 x float> [[SPLIT1]], ptr [[VEC_GEP3]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x float> %in, float* %out, i64 %stride, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x float> %in, ptr %out, i64 %stride, i1 false, i32 3, i32 2)
ret void
}
-declare void @llvm.matrix.column.major.store(<6 x float>, float*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store(<6 x float>, ptr, i64, i1, i32, i32)
-define void @strided_store_2x3(<10 x float> %in, float* %out) {
+define void @strided_store_2x3(<10 x float> %in, ptr %out) {
; CHECK-LABEL: @strided_store_2x3(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <10 x float> [[IN:%.*]], <10 x float> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> poison, <2 x i32> <i32 6, i32 7>
; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <10 x float> [[IN]], <10 x float> poison, <2 x i32> <i32 8, i32 9>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast float* [[OUT:%.*]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[SPLIT]], <2 x float>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, float* [[OUT]], i64 4
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast float* [[VEC_GEP]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[SPLIT1]], <2 x float>* [[VEC_CAST5]], align 4
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr float, float* [[OUT]], i64 8
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast float* [[VEC_GEP6]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[SPLIT2]], <2 x float>* [[VEC_CAST7]], align 4
-; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr float, float* [[OUT]], i64 12
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast float* [[VEC_GEP8]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[SPLIT3]], <2 x float>* [[VEC_CAST9]], align 4
-; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr float, float* [[OUT]], i64 16
-; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast float* [[VEC_GEP10]] to <2 x float>*
-; CHECK-NEXT: store <2 x float> [[SPLIT4]], <2 x float>* [[VEC_CAST11]], align 4
+; CHECK-NEXT: store <2 x float> [[SPLIT]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[OUT]], i64 4
+; CHECK-NEXT: store <2 x float> [[SPLIT1]], ptr [[VEC_GEP]], align 4
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr float, ptr [[OUT]], i64 8
+; CHECK-NEXT: store <2 x float> [[SPLIT2]], ptr [[VEC_GEP6]], align 4
+; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr float, ptr [[OUT]], i64 12
+; CHECK-NEXT: store <2 x float> [[SPLIT3]], ptr [[VEC_GEP8]], align 4
+; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr float, ptr [[OUT]], i64 16
+; CHECK-NEXT: store <2 x float> [[SPLIT4]], ptr [[VEC_GEP10]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store.v10f32(<10 x float> %in, float* %out, i64 4, i1 false, i32 2, i32 5)
+ call void @llvm.matrix.column.major.store.v10f32(<10 x float> %in, ptr %out, i64 4, i1 false, i32 2, i32 5)
ret void
}
-declare void @llvm.matrix.column.major.store.v10f32(<10 x float>, float*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store.v10f32(<10 x float>, ptr, i64, i1, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
index ac9a956576c59..758237aa3405b 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/strided-store-i32.ll
@@ -1,67 +1,58 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
-define void @strided_store_3x2(<6 x i32> %in, i32* %out) {
+define void @strided_store_3x2(<6 x i32> %in, ptr %out) {
; CHECK-LABEL: @strided_store_3x2(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[OUT:%.*]] to <3 x i32>*
-; CHECK-NEXT: store <3 x i32> [[SPLIT]], <3 x i32>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT]], i64 5
-; CHECK-NEXT: [[VEC_CAST2:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
-; CHECK-NEXT: store <3 x i32> [[SPLIT1]], <3 x i32>* [[VEC_CAST2]], align 4
+; CHECK-NEXT: store <3 x i32> [[SPLIT]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT]], i64 5
+; CHECK-NEXT: store <3 x i32> [[SPLIT1]], ptr [[VEC_GEP]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x i32> %in, i32* %out, i64 5, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x i32> %in, ptr %out, i64 5, i1 false, i32 3, i32 2)
ret void
}
-define void @strided_store_3x2_nonconst_stride(<6 x i32> %in, i64 %stride, i32* %out) {
+define void @strided_store_3x2_nonconst_stride(<6 x i32> %in, i64 %stride, ptr %out) {
; CHECK-LABEL: @strided_store_3x2_nonconst_stride(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <6 x i32> [[IN:%.*]], <6 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <6 x i32> [[IN]], <6 x i32> poison, <3 x i32> <i32 3, i32 4, i32 5>
; CHECK-NEXT: [[VEC_START:%.*]] = mul i64 0, [[STRIDE:%.*]]
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT:%.*]], i64 [[VEC_START]]
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[VEC_GEP]] to <3 x i32>*
-; CHECK-NEXT: store <3 x i32> [[SPLIT]], <3 x i32>* [[VEC_CAST]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT:%.*]], i64 [[VEC_START]]
+; CHECK-NEXT: store <3 x i32> [[SPLIT]], ptr [[VEC_GEP]], align 4
; CHECK-NEXT: [[VEC_START2:%.*]] = mul i64 1, [[STRIDE]]
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, i32* [[OUT]], i64 [[VEC_START2]]
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast i32* [[VEC_GEP3]] to <3 x i32>*
-; CHECK-NEXT: store <3 x i32> [[SPLIT1]], <3 x i32>* [[VEC_CAST4]], align 4
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[OUT]], i64 [[VEC_START2]]
+; CHECK-NEXT: store <3 x i32> [[SPLIT1]], ptr [[VEC_GEP3]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store(<6 x i32> %in, i32* %out, i64 %stride, i1 false, i32 3, i32 2)
+ call void @llvm.matrix.column.major.store(<6 x i32> %in, ptr %out, i64 %stride, i1 false, i32 3, i32 2)
ret void
}
-declare void @llvm.matrix.column.major.store(<6 x i32>, i32*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store(<6 x i32>, ptr, i64, i1, i32, i32)
-define void @strided_store_2x3(<10 x i32> %in, i32* %out) {
+define void @strided_store_2x3(<10 x i32> %in, ptr %out) {
; CHECK-LABEL: @strided_store_2x3(
; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <10 x i32> [[IN:%.*]], <10 x i32> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> poison, <2 x i32> <i32 4, i32 5>
; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> poison, <2 x i32> <i32 6, i32 7>
; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <10 x i32> [[IN]], <10 x i32> poison, <2 x i32> <i32 8, i32 9>
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast i32* [[OUT:%.*]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT]], <2 x i32>* [[VEC_CAST]], align 4
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, i32* [[OUT]], i64 4
-; CHECK-NEXT: [[VEC_CAST5:%.*]] = bitcast i32* [[VEC_GEP]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT1]], <2 x i32>* [[VEC_CAST5]], align 4
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i32, i32* [[OUT]], i64 8
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast i32* [[VEC_GEP6]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT2]], <2 x i32>* [[VEC_CAST7]], align 4
-; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr i32, i32* [[OUT]], i64 12
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast i32* [[VEC_GEP8]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT3]], <2 x i32>* [[VEC_CAST9]], align 4
-; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr i32, i32* [[OUT]], i64 16
-; CHECK-NEXT: [[VEC_CAST11:%.*]] = bitcast i32* [[VEC_GEP10]] to <2 x i32>*
-; CHECK-NEXT: store <2 x i32> [[SPLIT4]], <2 x i32>* [[VEC_CAST11]], align 4
+; CHECK-NEXT: store <2 x i32> [[SPLIT]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[OUT]], i64 4
+; CHECK-NEXT: store <2 x i32> [[SPLIT1]], ptr [[VEC_GEP]], align 4
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr i32, ptr [[OUT]], i64 8
+; CHECK-NEXT: store <2 x i32> [[SPLIT2]], ptr [[VEC_GEP6]], align 4
+; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr i32, ptr [[OUT]], i64 12
+; CHECK-NEXT: store <2 x i32> [[SPLIT3]], ptr [[VEC_GEP8]], align 4
+; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr i32, ptr [[OUT]], i64 16
+; CHECK-NEXT: store <2 x i32> [[SPLIT4]], ptr [[VEC_GEP10]], align 4
; CHECK-NEXT: ret void
;
- call void @llvm.matrix.column.major.store.v10i32(<10 x i32> %in, i32* %out, i64 4, i1 false, i32 2, i32 5)
+ call void @llvm.matrix.column.major.store.v10i32(<10 x i32> %in, ptr %out, i64 4, i1 false, i32 2, i32 5)
ret void
}
-declare void @llvm.matrix.column.major.store.v10i32(<10 x i32>, i32*, i64, i1, i32, i32)
+declare void @llvm.matrix.column.major.store.v10i32(<10 x i32>, ptr, i64, i1, i32, i32)
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
index 4933a021a46d3..f2f9f2682ab4d 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/transpose-opts.ll
@@ -9,7 +9,7 @@
target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
target triple = "aarch64-apple-ios"
-define void @double_transpose(<9 x double>* %A, <9 x double>* %B) {
+define void @double_transpose(ptr %A, ptr %B) {
; REMARK: Pass: lower-matrix-intrinsics
; REMARK-NEXT: Name: matrix-lowered
; REMARK-NEXT: Function: double_transpose
@@ -29,35 +29,27 @@ define void @double_transpose(<9 x double>* %A, <9 x double>* %B) {
; REMARK-NEXT: addr %B)
; CHECK-LABEL: @double_transpose(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST4]], align 16
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <9 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[COL_LOAD]], <3 x double>* [[VEC_CAST6]], align 16
-; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, double* [[TMP1]], i64 3
-; CHECK-NEXT: [[VEC_CAST8:%.*]] = bitcast double* [[VEC_GEP7]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[COL_LOAD2]], <3 x double>* [[VEC_CAST8]], align 8
-; CHECK-NEXT: [[VEC_GEP9:%.*]] = getelementptr double, double* [[TMP1]], i64 6
-; CHECK-NEXT: [[VEC_CAST10:%.*]] = bitcast double* [[VEC_GEP9]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[COL_LOAD5]], <3 x double>* [[VEC_CAST10]], align 16
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <3 x double>, ptr [[VEC_GEP3]], align 16
+; CHECK-NEXT: store <3 x double> [[COL_LOAD]], ptr [[B:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B]], i64 3
+; CHECK-NEXT: store <3 x double> [[COL_LOAD2]], ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT: [[VEC_GEP9:%.*]] = getelementptr double, ptr [[B]], i64 6
+; CHECK-NEXT: store <3 x double> [[COL_LOAD5]], ptr [[VEC_GEP9]], align 16
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %A, align 16
+ %a = load <9 x double>, ptr %A, align 16
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
%att = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %at, i32 3, i32 3)
- store <9 x double> %att, <9 x double>* %B, align 16
+ store <9 x double> %att, ptr %B, align 16
ret void
}
-define void @multiply_ntt(<6 x double>* %A, <6 x double>* %B, <8 x double>* %C, <8 x double>* %R) {
+define void @multiply_ntt(ptr %A, ptr %B, ptr %C, ptr %R) {
; REMARK: Pass: lower-matrix-intrinsics
; REMARK-NEXT: Name: matrix-lowered
; REMARK-NEXT: Function: multiply_ntt
@@ -81,33 +73,22 @@ define void @multiply_ntt(<6 x double>* %A, <6 x double>* %B, <8 x double>* %C,
; REMARK-NEXT: addr %R)
; CHECK-LABEL: @multiply_ntt(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <6 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 2
-; CHECK-NEXT: [[VEC_CAST39:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST39]], align 16
-; CHECK-NEXT: [[VEC_GEP41:%.*]] = getelementptr double, double* [[TMP0]], i64 4
-; CHECK-NEXT: [[VEC_CAST42:%.*]] = bitcast double* [[VEC_GEP41]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD43:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST42]], align 16
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT: [[VEC_GEP41:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[VEC_GEP41]], align 16
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> [[COL_LOAD40]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD43]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <6 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST44:%.*]] = bitcast double* [[TMP4]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD45:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST44]], align 16
-; CHECK-NEXT: [[VEC_GEP46:%.*]] = getelementptr double, double* [[TMP4]], i64 2
-; CHECK-NEXT: [[VEC_CAST47:%.*]] = bitcast double* [[VEC_GEP46]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST47]], align 16
-; CHECK-NEXT: [[VEC_GEP49:%.*]] = getelementptr double, double* [[TMP4]], i64 4
-; CHECK-NEXT: [[VEC_CAST50:%.*]] = bitcast double* [[VEC_GEP49]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST50]], align 16
-; CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST52:%.*]] = bitcast double* [[TMP5]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST52]], align 16
-; CHECK-NEXT: [[VEC_GEP54:%.*]] = getelementptr double, double* [[TMP5]], i64 4
-; CHECK-NEXT: [[VEC_CAST55:%.*]] = bitcast double* [[VEC_GEP54]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD56:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST55]], align 16
+; CHECK-NEXT: [[COL_LOAD45:%.*]] = load <2 x double>, ptr [[B:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP46:%.*]] = getelementptr double, ptr [[B]], i64 2
+; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <2 x double>, ptr [[VEC_GEP46]], align 16
+; CHECK-NEXT: [[VEC_GEP49:%.*]] = getelementptr double, ptr [[B]], i64 4
+; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <2 x double>, ptr [[VEC_GEP49]], align 16
+; CHECK-NEXT: [[COL_LOAD53:%.*]] = load <4 x double>, ptr [[C:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP54:%.*]] = getelementptr double, ptr [[C]], i64 4
+; CHECK-NEXT: [[COL_LOAD56:%.*]] = load <4 x double>, ptr [[VEC_GEP54]], align 16
; CHECK-NEXT: [[BLOCK57:%.*]] = shufflevector <4 x double> [[COL_LOAD53]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD45]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT58:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i32 0
@@ -271,33 +252,28 @@ define void @multiply_ntt(<6 x double>* %A, <6 x double>* %B, <8 x double>* %C,
; CHECK-NEXT: [[TMP88:%.*]] = fadd <2 x double> [[TMP85]], [[TMP87]]
; CHECK-NEXT: [[TMP89:%.*]] = shufflevector <2 x double> [[TMP88]], <2 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[TMP90:%.*]] = shufflevector <2 x double> undef, <2 x double> [[TMP89]], <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[TMP91:%.*]] = bitcast <8 x double>* [[R:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST93:%.*]] = bitcast double* [[TMP91]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP60]], <2 x double>* [[VEC_CAST93]], align 16
-; CHECK-NEXT: [[VEC_GEP94:%.*]] = getelementptr double, double* [[TMP91]], i64 2
-; CHECK-NEXT: [[VEC_CAST95:%.*]] = bitcast double* [[VEC_GEP94]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP70]], <2 x double>* [[VEC_CAST95]], align 16
-; CHECK-NEXT: [[VEC_GEP96:%.*]] = getelementptr double, double* [[TMP91]], i64 4
-; CHECK-NEXT: [[VEC_CAST97:%.*]] = bitcast double* [[VEC_GEP96]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP80]], <2 x double>* [[VEC_CAST97]], align 16
-; CHECK-NEXT: [[VEC_GEP98:%.*]] = getelementptr double, double* [[TMP91]], i64 6
-; CHECK-NEXT: [[VEC_CAST99:%.*]] = bitcast double* [[VEC_GEP98]] to <2 x double>*
-; CHECK-NEXT: store <2 x double> [[TMP90]], <2 x double>* [[VEC_CAST99]], align 16
+; CHECK-NEXT: store <2 x double> [[TMP60]], ptr [[R:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP94:%.*]] = getelementptr double, ptr [[R]], i64 2
+; CHECK-NEXT: store <2 x double> [[TMP70]], ptr [[VEC_GEP94]], align 16
+; CHECK-NEXT: [[VEC_GEP96:%.*]] = getelementptr double, ptr [[R]], i64 4
+; CHECK-NEXT: store <2 x double> [[TMP80]], ptr [[VEC_GEP96]], align 16
+; CHECK-NEXT: [[VEC_GEP98:%.*]] = getelementptr double, ptr [[R]], i64 6
+; CHECK-NEXT: store <2 x double> [[TMP90]], ptr [[VEC_GEP98]], align 16
; CHECK-NEXT: ret void
;
entry:
- %a = load <6 x double>, <6 x double>* %A, align 16
- %b = load <6 x double>, <6 x double>* %B, align 16
- %c = load <8 x double>, <8 x double>* %C, align 16
+ %a = load <6 x double>, ptr %A, align 16
+ %b = load <6 x double>, ptr %B, align 16
+ %c = load <8 x double>, ptr %C, align 16
%b_t = call <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double> %b, i32 2, i32 3)
%c_t = call <8 x double> @llvm.matrix.transpose.v8f64.v8f64(<8 x double> %c, i32 4, i32 2)
%m1 = call <12 x double> @llvm.matrix.multiply.v12f64.v6f64.v8f64(<6 x double> %b_t, <8 x double> %c_t, i32 3, i32 2, i32 4)
%m2 = call <8 x double> @llvm.matrix.multiply.v8f64.v6f64.v12f64(<6 x double> %a, <12 x double> %m1, i32 2, i32 3, i32 4)
- store <8 x double> %m2, <8 x double>* %R, align 16
+ store <8 x double> %m2, ptr %R, align 16
ret void
}
-define void @multiply_tt_t(<9 x double>* %A, <9 x double>* %B, <9 x double>* %C) {
+define void @multiply_tt_t(ptr %A, ptr %B, ptr %C) {
; REMARK: Pass: lower-matrix-intrinsics
; REMARK-NEXT: Name: matrix-lowered
; REMARK-NEXT: Function: multiply_tt_t
@@ -319,24 +295,16 @@ define void @multiply_tt_t(<9 x double>* %A, <9 x double>* %B, <9 x double>* %C)
; REMARK-NEXT: addr %C)
; CHECK-LABEL: @multiply_tt_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST4]], align 16
-; CHECK-NEXT: [[TMP1:%.*]] = bitcast <9 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST6:%.*]] = bitcast double* [[TMP1]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST6]], align 16
-; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, double* [[TMP1]], i64 3
-; CHECK-NEXT: [[VEC_CAST9:%.*]] = bitcast double* [[VEC_GEP8]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST9]], align 8
-; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr double, double* [[TMP1]], i64 6
-; CHECK-NEXT: [[VEC_CAST12:%.*]] = bitcast double* [[VEC_GEP11]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST12]], align 16
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <3 x double>, ptr [[VEC_GEP3]], align 16
+; CHECK-NEXT: [[COL_LOAD7:%.*]] = load <3 x double>, ptr [[B:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP8:%.*]] = getelementptr double, ptr [[B]], i64 3
+; CHECK-NEXT: [[COL_LOAD10:%.*]] = load <3 x double>, ptr [[VEC_GEP8]], align 8
+; CHECK-NEXT: [[VEC_GEP11:%.*]] = getelementptr double, ptr [[B]], i64 6
+; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <3 x double>, ptr [[VEC_GEP11]], align 16
; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[COL_LOAD7]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
@@ -451,29 +419,25 @@ define void @multiply_tt_t(<9 x double>* %A, <9 x double>* %B, <9 x double>* %C)
; CHECK-NEXT: [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP60]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[TMP62:%.*]] = bitcast <9 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST65:%.*]] = bitcast double* [[TMP62]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP21]], <3 x double>* [[VEC_CAST65]], align 16
-; CHECK-NEXT: [[VEC_GEP66:%.*]] = getelementptr double, double* [[TMP62]], i64 3
-; CHECK-NEXT: [[VEC_CAST67:%.*]] = bitcast double* [[VEC_GEP66]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP41]], <3 x double>* [[VEC_CAST67]], align 8
-; CHECK-NEXT: [[VEC_GEP68:%.*]] = getelementptr double, double* [[TMP62]], i64 6
-; CHECK-NEXT: [[VEC_CAST69:%.*]] = bitcast double* [[VEC_GEP68]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP61]], <3 x double>* [[VEC_CAST69]], align 16
+; CHECK-NEXT: store <3 x double> [[TMP21]], ptr [[C:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP66:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT: store <3 x double> [[TMP41]], ptr [[VEC_GEP66]], align 8
+; CHECK-NEXT: [[VEC_GEP68:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT: store <3 x double> [[TMP61]], ptr [[VEC_GEP68]], align 16
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %A, align 16
+ %a = load <9 x double>, ptr %A, align 16
%at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)
- %b = load <9 x double>, <9 x double>* %B, align 16
+ %b = load <9 x double>, ptr %B, align 16
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
%c = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %at, <9 x double> %bt, i32 3, i32 3, i32 3)
%ct = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %c, i32 3, i32 3)
- store <9 x double> %ct, <9 x double>* %C, align 16
+ store <9 x double> %ct, ptr %C, align 16
ret void
}
-define void @multiply_nt_t(<6 x double>* %A, <12 x double>* %B, <8 x double>* %C) {
+define void @multiply_nt_t(ptr %A, ptr %B, ptr %C) {
; REMARK: Pass: lower-matrix-intrinsics
; REMARK-NEXT: Name: matrix-lowered
; REMARK-NEXT: Function: multiply_nt_t
@@ -495,27 +459,19 @@ define void @multiply_nt_t(<6 x double>* %A, <12 x double>* %B, <8 x double>* %C
; REMARK-NEXT: addr %C)
; CHECK-LABEL: @multiply_nt_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <6 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 2
-; CHECK-NEXT: [[VEC_CAST39:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST39]], align 16
-; CHECK-NEXT: [[VEC_GEP41:%.*]] = getelementptr double, double* [[TMP0]], i64 4
-; CHECK-NEXT: [[VEC_CAST42:%.*]] = bitcast double* [[VEC_GEP41]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD43:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST42]], align 16
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 2
+; CHECK-NEXT: [[COL_LOAD40:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16
+; CHECK-NEXT: [[VEC_GEP41:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD43:%.*]] = load <2 x double>, ptr [[VEC_GEP41]], align 16
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> [[COL_LOAD40]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD43]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <12 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST44:%.*]] = bitcast double* [[TMP4]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD45:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST44]], align 16
-; CHECK-NEXT: [[VEC_GEP46:%.*]] = getelementptr double, double* [[TMP4]], i64 4
-; CHECK-NEXT: [[VEC_CAST47:%.*]] = bitcast double* [[VEC_GEP46]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST47]], align 16
-; CHECK-NEXT: [[VEC_GEP49:%.*]] = getelementptr double, double* [[TMP4]], i64 8
-; CHECK-NEXT: [[VEC_CAST50:%.*]] = bitcast double* [[VEC_GEP49]] to <4 x double>*
-; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <4 x double>, <4 x double>* [[VEC_CAST50]], align 16
+; CHECK-NEXT: [[COL_LOAD45:%.*]] = load <4 x double>, ptr [[B:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP46:%.*]] = getelementptr double, ptr [[B]], i64 4
+; CHECK-NEXT: [[COL_LOAD48:%.*]] = load <4 x double>, ptr [[VEC_GEP46]], align 16
+; CHECK-NEXT: [[VEC_GEP49:%.*]] = getelementptr double, ptr [[B]], i64 8
+; CHECK-NEXT: [[COL_LOAD51:%.*]] = load <4 x double>, ptr [[VEC_GEP49]], align 16
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[COL_LOAD45]], <4 x double> [[COL_LOAD48]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[COL_LOAD51]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP6]], <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -601,25 +557,22 @@ define void @multiply_nt_t(<6 x double>* %A, <12 x double>* %B, <8 x double>* %C
; CHECK-NEXT: [[TMP45:%.*]] = fadd <2 x double> [[TMP42]], [[TMP44]]
; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x double> [[TMP45]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> [[TMP46]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT: [[TMP48:%.*]] = bitcast <8 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST52:%.*]] = bitcast double* [[TMP48]] to <4 x double>*
-; CHECK-NEXT: store <4 x double> [[TMP27]], <4 x double>* [[VEC_CAST52]], align 16
-; CHECK-NEXT: [[VEC_GEP53:%.*]] = getelementptr double, double* [[TMP48]], i64 4
-; CHECK-NEXT: [[VEC_CAST54:%.*]] = bitcast double* [[VEC_GEP53]] to <4 x double>*
-; CHECK-NEXT: store <4 x double> [[TMP47]], <4 x double>* [[VEC_CAST54]], align 16
+; CHECK-NEXT: store <4 x double> [[TMP27]], ptr [[C:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP53:%.*]] = getelementptr double, ptr [[C]], i64 4
+; CHECK-NEXT: store <4 x double> [[TMP47]], ptr [[VEC_GEP53]], align 16
; CHECK-NEXT: ret void
;
entry:
- %a = load <6 x double>, <6 x double>* %A, align 16
- %b = load <12 x double>, <12 x double>* %B, align 16
+ %a = load <6 x double>, ptr %A, align 16
+ %b = load <12 x double>, ptr %B, align 16
%bt = call <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double> %b, i32 4, i32 3)
%c = call <8 x double> @llvm.matrix.multiply.v8f64.v6f64.v12f64(<6 x double> %a, <12 x double> %bt, i32 2, i32 3, i32 4)
%ct = call <8 x double> @llvm.matrix.transpose.v8f64.v8f64(<8 x double> %c, i32 2, i32 4)
- store <8 x double> %ct, <8 x double>* %C, align 16
+ store <8 x double> %ct, ptr %C, align 16
ret void
}
-define void @multiply_ntt_t(<9 x double>* %A, <9 x double>* %B, <9 x double>* %C, <9 x double>* %R) {
+define void @multiply_ntt_t(ptr %A, ptr %B, ptr %C, ptr %R) {
; REMARK: Pass: lower-matrix-intrinsics
; REMARK-NEXT: Name: matrix-lowered
; REMARK-NEXT: Function: multiply_ntt_t
@@ -643,36 +596,24 @@ define void @multiply_ntt_t(<9 x double>* %A, <9 x double>* %B, <9 x double>* %C
; REMARK-NEXT: addr %R)
; CHECK-LABEL: @multiply_ntt_t(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <9 x double>* [[A:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[TMP0]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST]], align 16
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[TMP0]], i64 3
-; CHECK-NEXT: [[VEC_CAST58:%.*]] = bitcast double* [[VEC_GEP]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD59:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST58]], align 8
-; CHECK-NEXT: [[VEC_GEP60:%.*]] = getelementptr double, double* [[TMP0]], i64 6
-; CHECK-NEXT: [[VEC_CAST61:%.*]] = bitcast double* [[VEC_GEP60]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD62:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST61]], align 16
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT: [[COL_LOAD59:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP60:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT: [[COL_LOAD62:%.*]] = load <3 x double>, ptr [[VEC_GEP60]], align 16
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD]], <3 x double> [[COL_LOAD59]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x double> [[COL_LOAD62]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x double> [[TMP1]], <6 x double> [[TMP2]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast <9 x double>* [[B:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST63:%.*]] = bitcast double* [[TMP4]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD64:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST63]], align 16
-; CHECK-NEXT: [[VEC_GEP65:%.*]] = getelementptr double, double* [[TMP4]], i64 3
-; CHECK-NEXT: [[VEC_CAST66:%.*]] = bitcast double* [[VEC_GEP65]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD67:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST66]], align 8
-; CHECK-NEXT: [[VEC_GEP68:%.*]] = getelementptr double, double* [[TMP4]], i64 6
-; CHECK-NEXT: [[VEC_CAST69:%.*]] = bitcast double* [[VEC_GEP68]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD70:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST69]], align 16
-; CHECK-NEXT: [[TMP5:%.*]] = bitcast <9 x double>* [[C:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST71:%.*]] = bitcast double* [[TMP5]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD72:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST71]], align 16
-; CHECK-NEXT: [[VEC_GEP73:%.*]] = getelementptr double, double* [[TMP5]], i64 3
-; CHECK-NEXT: [[VEC_CAST74:%.*]] = bitcast double* [[VEC_GEP73]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD75:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST74]], align 8
-; CHECK-NEXT: [[VEC_GEP76:%.*]] = getelementptr double, double* [[TMP5]], i64 6
-; CHECK-NEXT: [[VEC_CAST77:%.*]] = bitcast double* [[VEC_GEP76]] to <3 x double>*
-; CHECK-NEXT: [[COL_LOAD78:%.*]] = load <3 x double>, <3 x double>* [[VEC_CAST77]], align 16
+; CHECK-NEXT: [[COL_LOAD64:%.*]] = load <3 x double>, ptr [[B:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP65:%.*]] = getelementptr double, ptr [[B]], i64 3
+; CHECK-NEXT: [[COL_LOAD67:%.*]] = load <3 x double>, ptr [[VEC_GEP65]], align 8
+; CHECK-NEXT: [[VEC_GEP68:%.*]] = getelementptr double, ptr [[B]], i64 6
+; CHECK-NEXT: [[COL_LOAD70:%.*]] = load <3 x double>, ptr [[VEC_GEP68]], align 16
+; CHECK-NEXT: [[COL_LOAD72:%.*]] = load <3 x double>, ptr [[C:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP73:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT: [[COL_LOAD75:%.*]] = load <3 x double>, ptr [[VEC_GEP73]], align 8
+; CHECK-NEXT: [[VEC_GEP76:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT: [[COL_LOAD78:%.*]] = load <3 x double>, ptr [[VEC_GEP76]], align 16
; CHECK-NEXT: [[BLOCK79:%.*]] = shufflevector <3 x double> [[COL_LOAD72]], <3 x double> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD64]], i64 0
; CHECK-NEXT: [[SPLAT_SPLATINSERT80:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i32 0
@@ -910,27 +851,23 @@ define void @multiply_ntt_t(<9 x double>* %A, <9 x double>* %B, <9 x double>* %C
; CHECK-NEXT: [[TMP126:%.*]] = fadd <1 x double> [[TMP123]], [[TMP125]]
; CHECK-NEXT: [[TMP127:%.*]] = shufflevector <1 x double> [[TMP126]], <1 x double> poison, <3 x i32> <i32 0, i32 undef, i32 undef>
; CHECK-NEXT: [[TMP128:%.*]] = shufflevector <3 x double> [[TMP118]], <3 x double> [[TMP127]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT: [[TMP129:%.*]] = bitcast <9 x double>* [[R:%.*]] to double*
-; CHECK-NEXT: [[VEC_CAST133:%.*]] = bitcast double* [[TMP129]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP88]], <3 x double>* [[VEC_CAST133]], align 16
-; CHECK-NEXT: [[VEC_GEP134:%.*]] = getelementptr double, double* [[TMP129]], i64 3
-; CHECK-NEXT: [[VEC_CAST135:%.*]] = bitcast double* [[VEC_GEP134]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP108]], <3 x double>* [[VEC_CAST135]], align 8
-; CHECK-NEXT: [[VEC_GEP136:%.*]] = getelementptr double, double* [[TMP129]], i64 6
-; CHECK-NEXT: [[VEC_CAST137:%.*]] = bitcast double* [[VEC_GEP136]] to <3 x double>*
-; CHECK-NEXT: store <3 x double> [[TMP128]], <3 x double>* [[VEC_CAST137]], align 16
+; CHECK-NEXT: store <3 x double> [[TMP88]], ptr [[R:%.*]], align 16
+; CHECK-NEXT: [[VEC_GEP134:%.*]] = getelementptr double, ptr [[R]], i64 3
+; CHECK-NEXT: store <3 x double> [[TMP108]], ptr [[VEC_GEP134]], align 8
+; CHECK-NEXT: [[VEC_GEP136:%.*]] = getelementptr double, ptr [[R]], i64 6
+; CHECK-NEXT: store <3 x double> [[TMP128]], ptr [[VEC_GEP136]], align 16
; CHECK-NEXT: ret void
;
entry:
- %a = load <9 x double>, <9 x double>* %A, align 16
- %b = load <9 x double>, <9 x double>* %B, align 16
+ %a = load <9 x double>, ptr %A, align 16
+ %b = load <9 x double>, ptr %B, align 16
%bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)
- %c = load <9 x double>, <9 x double>* %C, align 16
+ %c = load <9 x double>, ptr %C, align 16
%ct = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %c, i32 3, i32 3)
%btct = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %bt, <9 x double> %ct, i32 3, i32 3, i32 3)
%abtct= call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a, <9 x double> %btct, i32 3, i32 3, i32 3)
%abtct_t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %abtct, i32 3, i32 3)
- store <9 x double> %abtct_t, <9 x double>* %R, align 16
+ store <9 x double> %abtct_t, ptr %R, align 16
ret void
}
@@ -984,26 +921,22 @@ entry:
ret <4 x float> %m
}
-define <6 x double> @transpose_of_transpose_of_non_matrix_op(double* %a) {
+define <6 x double> @transpose_of_transpose_of_non_matrix_op(ptr %a) {
; CHECK-LABEL: @transpose_of_transpose_of_non_matrix_op(
-; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST]], align 8
-; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, double* [[A]], i64 4
-; CHECK-NEXT: [[VEC_CAST1:%.*]] = bitcast double* [[VEC_GEP]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST1]], align 8
-; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, double* [[A]], i64 8
-; CHECK-NEXT: [[VEC_CAST4:%.*]] = bitcast double* [[VEC_GEP3]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST4]], align 8
-; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, double* [[A]], i64 12
-; CHECK-NEXT: [[VEC_CAST7:%.*]] = bitcast double* [[VEC_GEP6]] to <2 x double>*
-; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, <2 x double>* [[VEC_CAST7]], align 8
+; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 4
+; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[A]], i64 8
+; CHECK-NEXT: [[COL_LOAD5:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT: [[VEC_GEP6:%.*]] = getelementptr double, ptr [[A]], i64 12
+; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP6]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> [[COL_LOAD2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[COL_LOAD5]], <2 x double> [[COL_LOAD8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <8 x double> [[TMP3]], <8 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6>
; CHECK-NEXT: ret <6 x double> [[SHUF]]
;
- %load = call <8 x double> @llvm.matrix.column.major.load.v8f64(double* %a, i64 4, i1 false, i32 2, i32 4)
+ %load = call <8 x double> @llvm.matrix.column.major.load.v8f64(ptr %a, i64 4, i1 false, i32 2, i32 4)
%shuf = shufflevector <8 x double> %load, <8 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 4, i32 5, i32 6>
%t = call <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double> %shuf, i32 3, i32 2)
%tt = call <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double> %t, i32 2, i32 3)
@@ -1139,4 +1072,4 @@ declare <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double>, i32, i32)
declare <8 x double> @llvm.matrix.transpose.v8f64.v8f64(<8 x double>, i32, i32)
declare <12 x double> @llvm.matrix.transpose.v12f64.v12f64(<12 x double>, i32, i32)
declare <4 x float> @llvm.matrix.transpose.v4f32(<4 x float>, i32, i32)
-declare <8 x double> @llvm.matrix.column.major.load.v8f64(double*, i64, i1, i32, i32)
+declare <8 x double> @llvm.matrix.column.major.load.v8f64(ptr, i64, i1, i32, i32)
More information about the llvm-commits
mailing list