[clang] [Matrix] Fix cbuffers support for matrix element expr (PR #185471)
Farzon Lotfi via cfe-commits
cfe-commits at lists.llvm.org
Mon Mar 9 10:38:56 PDT 2026
https://github.com/farzonl created https://github.com/llvm/llvm-project/pull/185471
fixes #184877
This change was threefold.
1. copy the padded cbuffer from memory to a local alloca
2. switch to using the new `getFlattenedIndex` helpers for index generation
3. convert row major to column major indicies in codegen depending on LangOptions
>From f4864036f96f2e23935e30ae6805c300ba90ef53 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi at microsoft.com>
Date: Mon, 9 Mar 2026 13:18:43 -0400
Subject: [PATCH] [Matrix] Fix cbuffers support for matrix element expr
fixes #184877
This change was threefold.
1. copy the padded cbuffer from memory to a local alloca
2. switch to using the new `getFlattenedIndex` helpers for index
generation
3. convert row major to column major indicies in codegen depending on LangOptions
---
clang/lib/AST/Expr.cpp | 24 ++---
clang/lib/CodeGen/CGExpr.cpp | 31 +++++-
.../MatrixElementRowColFlags.hlsl | 37 +++++++
...member-one-based-accessor-scalar-load.hlsl | 24 ++---
...ember-one-based-accessor-scalar-store.hlsl | 24 ++---
.../matrix-member-one-based-swizzle-load.hlsl | 16 ++--
...matrix-member-one-based-swizzle-store.hlsl | 48 +++++-----
...ember-zero-based-accessor-scalar-load.hlsl | 24 ++---
...mber-zero-based-accessor-scalar-store.hlsl | 24 ++---
...matrix-member-zero-based-swizzle-load.hlsl | 16 ++--
...atrix-member-zero-based-swizzle-store.hlsl | 48 +++++-----
.../resources/MatrixElement_cbuffer.hlsl | 96 +++++++++++++++++++
12 files changed, 285 insertions(+), 127 deletions(-)
create mode 100644 clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl
create mode 100644 clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 9632d88fae4e4..185e887fb05c3 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -4478,8 +4478,8 @@ static MatrixAccessorFormat GetHLSLMatrixAccessorFormat(StringRef Comp) {
}
template <typename Fn>
-static bool ForEachMatrixAccessorIndex(StringRef Comp, unsigned Rows,
- unsigned Cols, Fn &&F) {
+static bool ForEachMatrixAccessorIndex(StringRef Comp,
+ const ConstantMatrixType *MT, Fn &&F) {
auto Format = GetHLSLMatrixAccessorFormat(Comp);
for (unsigned I = 0, E = Comp.size(); I < E; I += Format.ChunkLen) {
@@ -4491,8 +4491,13 @@ static bool ForEachMatrixAccessorIndex(StringRef Comp, unsigned Rows,
Col = static_cast<unsigned>(Comp[I + ZeroIndexOffset + 2] - '0') -
OneIndexOffset;
- assert(Row < Rows && Col < Cols && "matrix swizzle index out of bounds");
- const unsigned Index = Row * Cols + Col;
+ assert(Row < MT->getNumRows() && Col < MT->getNumColumns() &&
+ "matrix swizzle index out of bounds");
+ // NOTE: AST layer has no access to LangOptions so we will default to row
+ // major b\c all other AST matrix representations are row major.
+ // However in codegen we need to convert to column major if the flag
+ // requires it.
+ const unsigned Index = MT->getFlattenedIndex(Row, Col, /*IsRowMajor*/ true);
// Callback returns true to continue, false to stop early.
if (!F(Index))
return false;
@@ -4507,13 +4512,10 @@ static bool ForEachMatrixAccessorIndex(StringRef Comp, unsigned Rows,
bool MatrixElementExpr::containsDuplicateElements() const {
StringRef Comp = Accessor->getName();
const auto *MT = getBase()->getType()->castAs<ConstantMatrixType>();
- const unsigned Rows = MT->getNumRows();
- const unsigned Cols = MT->getNumColumns();
- const unsigned Max = Rows * Cols;
- llvm::BitVector Seen(Max, /*t=*/false);
+ llvm::BitVector Seen(MT->getNumElementsFlattened(), /*t=*/false);
bool HasDup = false;
- ForEachMatrixAccessorIndex(Comp, Rows, Cols, [&](unsigned Index) -> bool {
+ ForEachMatrixAccessorIndex(Comp, MT, [&](unsigned Index) -> bool {
if (Seen[Index]) {
HasDup = true;
return false; // exit early
@@ -4562,9 +4564,7 @@ void MatrixElementExpr::getEncodedElementAccess(
SmallVectorImpl<uint32_t> &Elts) const {
StringRef Comp = Accessor->getName();
const auto *MT = getBase()->getType()->castAs<ConstantMatrixType>();
- const unsigned Rows = MT->getNumRows();
- const unsigned Cols = MT->getNumColumns();
- ForEachMatrixAccessorIndex(Comp, Rows, Cols, [&](unsigned Index) -> bool {
+ ForEachMatrixAccessorIndex(Comp, MT, [&](unsigned Index) -> bool {
Elts.push_back(Index);
return true;
});
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index eebb36276e0eb..fe5de9f8df09d 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -2329,15 +2329,39 @@ LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
// Encode the element access list into a vector of unsigned indices.
+ // getEncodedElementAccess returns row-major linearized indices.
SmallVector<uint32_t, 4> Indices;
E->getEncodedElementAccess(Indices);
+ // getEncodedElementAccess returns row-major linearized indices
+ // If the matrix memory layout is column-major, convert indices
+ // to column-major indices.
+ bool IsColMajor = getLangOpts().getDefaultMatrixMemoryLayout() ==
+ LangOptions::MatrixMemoryLayout::MatrixColMajor;
+ if (IsColMajor) {
+ const auto *MT = E->getBase()->getType()->castAs<ConstantMatrixType>();
+ unsigned NumCols = MT->getNumColumns();
+ for (uint32_t &Idx : Indices) {
+ // Decompose row-major index: Row = Idx / NumCols, Col = Idx % NumCols
+ unsigned Row = Idx / NumCols;
+ unsigned Col = Idx % NumCols;
+ // Re-linearize as column-major
+ Idx = MT->getColumnMajorFlattenedIndex(Row, Col);
+ }
+ }
+
if (Base.isSimple()) {
+ RawAddress MatAddr = Base.getAddress();
+ if (getLangOpts().HLSL &&
+ E->getBase()->getType().getAddressSpace() == LangAS::hlsl_constant)
+ MatAddr = CGM.getHLSLRuntime().createBufferMatrixTempAddress(
+ Base, E->getExprLoc(), *this);
+
llvm::Constant *CV =
llvm::ConstantDataVector::get(getLLVMContext(), Indices);
- return LValue::MakeExtVectorElt(
- MaybeConvertMatrixAddress(Base.getAddress(), *this), CV, ResultType,
- Base.getBaseInfo(), TBAAAccessInfo());
+ return LValue::MakeExtVectorElt(MaybeConvertMatrixAddress(MatAddr, *this),
+ CV, ResultType, Base.getBaseInfo(),
+ TBAAAccessInfo());
}
assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
@@ -2347,6 +2371,7 @@ LValue CodeGenFunction::EmitMatrixElementExpr(const MatrixElementExpr *E) {
for (unsigned Index : Indices)
CElts.push_back(BaseElts->getAggregateElement(Index));
llvm::Constant *CV = llvm::ConstantVector::get(CElts);
+
return LValue::MakeExtVectorElt(
MaybeConvertMatrixAddress(Base.getExtVectorAddress(), *this), CV,
ResultType, Base.getBaseInfo(), TBAAAccessInfo());
diff --git a/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl
new file mode 100644
index 0000000000000..fe9e69c6a6898
--- /dev/null
+++ b/clang/test/CodeGenHLSL/BasicFeatures/MatrixElementRowColFlags.hlsl
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes \
+// RUN: -emit-llvm -finclude-default-header -fmatrix-memory-layout=column-major \
+// RUN: -o - %s | FileCheck %s --check-prefixes=CHECK,COL
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes \
+// RUN: -emit-llvm -finclude-default-header -fmatrix-memory-layout=row-major \
+// RUN: -o - %s | FileCheck %s --check-prefixes=CHECK,ROW
+
+// For a float3x2 matrix (3 rows, 2 columns):
+// Column-major flat vector: [_11, _21, _31, _12, _22, _32]
+// idx: 0 1 2 3 4 5
+// Row-major flat vector: [_11, _12, _21, _22, _31, _32]
+// idx: 0 1 2 3 4 5
+
+
+// CHECK-LABEL: define {{.*}} @_Z16getScalarElementu11matrix_typeILm3ELm2EfE
+// CHECK: load <6 x float>, ptr
+// COL-NEXT: extractelement <6 x float> {{.*}}, i32 4
+// ROW-NEXT: extractelement <6 x float> {{.*}}, i32 3
+export float getScalarElement(float3x2 M) {
+ return M._22;
+}
+
+// CHECK-LABEL: define {{.*}} @_Z18getSwizzleElementsu11matrix_typeILm3ELm2EfE
+// CHECK: load <6 x float>, ptr
+// COL-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 4>
+// ROW-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+export float4 getSwizzleElements(float3x2 M) {
+ return M._11_12_21_22;
+}
+
+// CHECK-LABEL: define {{.*}} @_Z22getZeroBasedSwizzleEltu11matrix_typeILm3ELm2EfE
+// CHECK: load <6 x float>, ptr
+// COL-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <2 x i32> <i32 1, i32 3>
+// ROW-NEXT: shufflevector <6 x float> {{.*}}, <6 x float> poison, <2 x i32> <i32 2, i32 1>
+export float2 getZeroBasedSwizzleElt(float3x2 M) {
+ return M._m10_m01;
+}
\ No newline at end of file
diff --git a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl
index def8aa5440568..6d8a3ce6ecbb6 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-load.hlsl
@@ -23,7 +23,7 @@ int Return11(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 1
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 4
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return12(int4x4 A) {
@@ -36,7 +36,7 @@ int Return12(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 2
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 8
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return13(int4x4 A) {
@@ -49,7 +49,7 @@ int Return13(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 3
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 12
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return14(int4x4 A) {
@@ -62,7 +62,7 @@ int Return14(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 4
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 1
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return21(int4x4 A) {
@@ -88,7 +88,7 @@ int Return22(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 6
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 9
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return23(int4x4 A) {
@@ -101,7 +101,7 @@ int Return23(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 7
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 13
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return24(int4x4 A) {
@@ -114,7 +114,7 @@ int Return24(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 8
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 2
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return31(int4x4 A) {
@@ -127,7 +127,7 @@ int Return31(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 9
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 6
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return32(int4x4 A) {
@@ -153,7 +153,7 @@ int Return33(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 11
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 14
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return34(int4x4 A) {
@@ -166,7 +166,7 @@ int Return34(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 12
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 3
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return41(int4x4 A) {
@@ -179,7 +179,7 @@ int Return41(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 13
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 7
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return42(int4x4 A) {
@@ -192,7 +192,7 @@ int Return42(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 14
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i32> [[TMP0]], i32 11
// CHECK-NEXT: ret i32 [[TMP1]]
//
int Return43(int4x4 A) {
diff --git a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl
index fb3a46170ebe0..9ea292ecea007 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-accessor-scalar-store.hlsl
@@ -29,7 +29,7 @@ void StoreScalarAtMat11(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 1
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 4
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -46,7 +46,7 @@ void StoreScalarAtMat12(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 2
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 8
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -63,7 +63,7 @@ void StoreScalarAtMat13(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 3
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 12
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -80,7 +80,7 @@ void StoreScalarAtMat14(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 4
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 1
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -114,7 +114,7 @@ void StoreScalarAtMat22(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 6
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 9
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -131,7 +131,7 @@ void StoreScalarAtMat23(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 7
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 13
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -148,7 +148,7 @@ void StoreScalarAtMat24(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 8
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 2
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -165,7 +165,7 @@ void StoreScalarAtMat31(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 9
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 6
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -199,7 +199,7 @@ void StoreScalarAtMat33(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 11
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 14
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -216,7 +216,7 @@ void StoreScalarAtMat34(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 12
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 3
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -233,7 +233,7 @@ void StoreScalarAtMat41(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 13
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 7
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -250,7 +250,7 @@ void StoreScalarAtMat42(out int4x4 A, int I) {
// CHECK-NEXT: store i32 [[I]], ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 14
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 11
// CHECK-NEXT: store i32 [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
diff --git a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl
index 47737aaab0390..6567d49f04dcf 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-load.hlsl
@@ -9,7 +9,7 @@
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnOnesSwizzle(int4x4 A) {
@@ -22,7 +22,7 @@ int4 ReturnOnesSwizzle(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnOnesSwizzle2(int4x4 A) {
@@ -35,7 +35,7 @@ int4 ReturnOnesSwizzle2(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnTwosSwizzle(int4x4 A) {
@@ -48,7 +48,7 @@ int4 ReturnTwosSwizzle(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnTwosSwizzle2(int4x4 A) {
@@ -61,7 +61,7 @@ int4 ReturnTwosSwizzle2(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnThreesSwizzle(int4x4 A) {
@@ -74,7 +74,7 @@ int4 ReturnThreesSwizzle(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnThreesSwizzle2(int4x4 A) {
@@ -87,7 +87,7 @@ int4 ReturnThreesSwizzle2(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnFoursSwizzle(int4x4 A) {
@@ -100,7 +100,7 @@ int4 ReturnFoursSwizzle(int4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x i32>], align 4
// CHECK-NEXT: store <16 x i32> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
// CHECK-NEXT: ret <4 x i32> [[TMP1]]
//
int4 ReturnFoursSwizzle2(int4x4 A) {
diff --git a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl
index ff7fab662a012..26b16d02a0d4a 100644
--- a/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-one-based-swizzle-store.hlsl
@@ -17,13 +17,13 @@
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
// CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP2]], align 4
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 1
+// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 4
// CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP5]], align 4
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 2
+// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 8
// CHECK-NEXT: store i32 [[TMP6]], ptr [[TMP7]], align 4
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 3
+// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 12
// CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP9]], align 4
// CHECK-NEXT: ret void
//
@@ -43,13 +43,13 @@ void OnesSwizzleToScalar(out int4x4 A, int I) {
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
// CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP1]], align 4
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 4
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 1
// CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP4]], align 4
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 8
+// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 2
// CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP6]], align 4
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
-// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 12
+// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 3
// CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 4
// CHECK-NEXT: ret void
//
@@ -69,16 +69,16 @@ void OnesSwizzleToVector(out int4x4 A, int4 V) {
// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x i32> [[CAST_SPLAT]], <1 x i32> poison, <4 x i32> zeroinitializer
// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 4
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 1
// CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP4]], align 4
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 5
// CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP6]], align 4
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 6
+// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 9
// CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 4
// CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 7
+// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 13
// CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP10]], align 4
// CHECK-NEXT: ret void
//
@@ -96,16 +96,16 @@ void TwosSwizzleToScalar(out int4x4 A, int I) {
// CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[V_ADDR]], align 16
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 1
+// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 4
// CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP3]], align 4
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 5
// CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP5]], align 4
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 9
+// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 6
// CHECK-NEXT: store i32 [[TMP6]], ptr [[TMP7]], align 4
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
-// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 13
+// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 7
// CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP9]], align 4
// CHECK-NEXT: ret void
//
@@ -125,16 +125,16 @@ void TwosSwizzleToVector(out int4x4 A, int4 V) {
// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x i32> [[CAST_SPLAT]], <1 x i32> poison, <4 x i32> zeroinitializer
// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 8
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 2
// CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP4]], align 4
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 9
+// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 6
// CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP6]], align 4
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 10
// CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 4
// CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 11
+// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 14
// CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP10]], align 4
// CHECK-NEXT: ret void
//
@@ -152,16 +152,16 @@ void ThreesSwizzleToScalar(out int4x4 A, int I) {
// CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[V_ADDR]], align 16
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 2
+// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 8
// CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP3]], align 4
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 6
+// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 9
// CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP5]], align 4
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 10
// CHECK-NEXT: store i32 [[TMP6]], ptr [[TMP7]], align 4
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
-// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 14
+// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 11
// CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP9]], align 4
// CHECK-NEXT: ret void
//
@@ -181,13 +181,13 @@ void ThreesSwizzleToVector(out int4x4 A, int4 V) {
// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x i32> [[CAST_SPLAT]], <1 x i32> poison, <4 x i32> zeroinitializer
// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 12
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 3
// CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP4]], align 4
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 13
+// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 7
// CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP6]], align 4
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 14
+// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 11
// CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 4
// CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x i32>, ptr [[TMP2]], i32 0, i32 15
@@ -208,13 +208,13 @@ void FoursSwizzleToScalar(out int4x4 A, int I) {
// CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[V_ADDR]], align 16
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 12
+// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 3
// CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP3]], align 4
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 13
+// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 7
// CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP5]], align 4
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 14
+// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 11
// CHECK-NEXT: store i32 [[TMP6]], ptr [[TMP7]], align 4
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x i32>, ptr [[TMP1]], i32 0, i32 15
diff --git a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl
index 5d542b8647eac..def0b884290da 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-load.hlsl
@@ -23,7 +23,7 @@ float Return00(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 1
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 4
// CHECK-NEXT: ret float [[TMP1]]
//
float Return01(float4x4 A) {
@@ -36,7 +36,7 @@ float Return01(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 2
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 8
// CHECK-NEXT: ret float [[TMP1]]
//
float Return02(float4x4 A) {
@@ -49,7 +49,7 @@ float Return02(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 3
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 12
// CHECK-NEXT: ret float [[TMP1]]
//
float Return03(float4x4 A) {
@@ -62,7 +62,7 @@ float Return03(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 4
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 1
// CHECK-NEXT: ret float [[TMP1]]
//
float Return10(float4x4 A) {
@@ -88,7 +88,7 @@ float Return11(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 6
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 9
// CHECK-NEXT: ret float [[TMP1]]
//
float Return12(float4x4 A) {
@@ -101,7 +101,7 @@ float Return12(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 7
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 13
// CHECK-NEXT: ret float [[TMP1]]
//
float Return13(float4x4 A) {
@@ -114,7 +114,7 @@ float Return13(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 8
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 2
// CHECK-NEXT: ret float [[TMP1]]
//
float Return20(float4x4 A) {
@@ -127,7 +127,7 @@ float Return20(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 9
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 6
// CHECK-NEXT: ret float [[TMP1]]
//
float Return21(float4x4 A) {
@@ -153,7 +153,7 @@ float Return22(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 11
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 14
// CHECK-NEXT: ret float [[TMP1]]
//
float Return23(float4x4 A) {
@@ -166,7 +166,7 @@ float Return23(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 12
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 3
// CHECK-NEXT: ret float [[TMP1]]
//
float Return30(float4x4 A) {
@@ -179,7 +179,7 @@ float Return30(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 13
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 7
// CHECK-NEXT: ret float [[TMP1]]
//
float Return31(float4x4 A) {
@@ -192,7 +192,7 @@ float Return31(float4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x float>], align 4
// CHECK-NEXT: store <16 x float> [[A]], ptr [[A_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[A_ADDR]], align 4
-// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 14
+// CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x float> [[TMP0]], i32 11
// CHECK-NEXT: ret float [[TMP1]]
//
float Return32(float4x4 A) {
diff --git a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl
index 97dc7e7c2bd37..ab5b69cb794b6 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-accessor-scalar-store.hlsl
@@ -29,7 +29,7 @@ void StoreScalarAtMat00(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 1
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 4
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -46,7 +46,7 @@ void StoreScalarAtMat01(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 2
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 8
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -63,7 +63,7 @@ void StoreScalarAtMat02(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 3
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 12
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -80,7 +80,7 @@ void StoreScalarAtMat03(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 4
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 1
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -114,7 +114,7 @@ void StoreScalarAtMat11(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 6
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 9
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -131,7 +131,7 @@ void StoreScalarAtMat12(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 7
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 13
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -148,7 +148,7 @@ void StoreScalarAtMat13(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 8
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 2
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -165,7 +165,7 @@ void StoreScalarAtMat20(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 9
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 6
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -199,7 +199,7 @@ void StoreScalarAtMat22(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 11
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 14
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -216,7 +216,7 @@ void StoreScalarAtMat23(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 12
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 3
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -233,7 +233,7 @@ void StoreScalarAtMat30(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 13
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 7
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
@@ -250,7 +250,7 @@ void StoreScalarAtMat31(out float4x4 A, float F) {
// CHECK-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[F_ADDR]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
-// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 14
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr <16 x float>, ptr [[TMP1]], i32 0, i32 11
// CHECK-NEXT: store float [[TMP0]], ptr [[TMP2]], align 4
// CHECK-NEXT: ret void
//
diff --git a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl
index dca2e6132de60..d9daf6ed7305b 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-load.hlsl
@@ -9,7 +9,7 @@
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnZerosSwizzle(half4x4 A) {
@@ -22,7 +22,7 @@ half4 ReturnZerosSwizzle(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnZerosSwizzle2(half4x4 A) {
@@ -35,7 +35,7 @@ half4 ReturnZerosSwizzle2(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnOnesSwizzle(half4x4 A) {
@@ -48,7 +48,7 @@ half4 ReturnOnesSwizzle(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnOnesSwizzle2(half4x4 A) {
@@ -61,7 +61,7 @@ half4 ReturnOnesSwizzle2(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnTwosSwizzle(half4x4 A) {
@@ -74,7 +74,7 @@ half4 ReturnTwosSwizzle(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnTwosSwizzle2(half4x4 A) {
@@ -87,7 +87,7 @@ half4 ReturnTwosSwizzle2(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnThreesSwizzle(half4x4 A) {
@@ -100,7 +100,7 @@ half4 ReturnThreesSwizzle(half4x4 A) {
// CHECK-NEXT: [[A_ADDR:%.*]] = alloca [4 x <4 x half>], align 2
// CHECK-NEXT: store <16 x half> [[A]], ptr [[A_ADDR]], align 2
// CHECK-NEXT: [[TMP0:%.*]] = load <16 x half>, ptr [[A_ADDR]], align 2
-// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
// CHECK-NEXT: ret <4 x half> [[TMP1]]
//
half4 ReturnThreesSwizzle2(half4x4 A) {
diff --git a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl
index 72fbc015d5934..986cdf136631b 100644
--- a/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl
+++ b/clang/test/CodeGenHLSL/matrix-member-zero-based-swizzle-store.hlsl
@@ -17,13 +17,13 @@
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
// CHECK-NEXT: store double [[TMP3]], ptr [[TMP2]], align 8
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 1
+// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 4
// CHECK-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 2
+// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 8
// CHECK-NEXT: store double [[TMP6]], ptr [[TMP7]], align 8
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 3
+// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 12
// CHECK-NEXT: store double [[TMP8]], ptr [[TMP9]], align 8
// CHECK-NEXT: ret void
//
@@ -43,13 +43,13 @@ void ZerosSwizzleToScalar(out double4x4 A, double D) {
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
// CHECK-NEXT: store double [[TMP2]], ptr [[TMP1]], align 8
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 4
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 1
// CHECK-NEXT: store double [[TMP3]], ptr [[TMP4]], align 8
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
-// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 8
+// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 2
// CHECK-NEXT: store double [[TMP5]], ptr [[TMP6]], align 8
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
-// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 12
+// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 3
// CHECK-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8
// CHECK-NEXT: ret void
//
@@ -69,16 +69,16 @@ void ZerosSwizzleToVector(out double4x4 A, double4 V) {
// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x double> [[CAST_SPLAT]], <1 x double> poison, <4 x i32> zeroinitializer
// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 4
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 1
// CHECK-NEXT: store double [[TMP3]], ptr [[TMP4]], align 8
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 5
// CHECK-NEXT: store double [[TMP5]], ptr [[TMP6]], align 8
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 6
+// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 9
// CHECK-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8
// CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 7
+// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 13
// CHECK-NEXT: store double [[TMP9]], ptr [[TMP10]], align 8
// CHECK-NEXT: ret void
//
@@ -96,16 +96,16 @@ void OnesSwizzleToScalar(out double4x4 A, double D) {
// CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[V_ADDR]], align 32
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
-// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 1
+// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 4
// CHECK-NEXT: store double [[TMP2]], ptr [[TMP3]], align 8
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 5
// CHECK-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
-// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 9
+// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 6
// CHECK-NEXT: store double [[TMP6]], ptr [[TMP7]], align 8
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
-// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 13
+// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 7
// CHECK-NEXT: store double [[TMP8]], ptr [[TMP9]], align 8
// CHECK-NEXT: ret void
//
@@ -125,16 +125,16 @@ void OnesSwizzleToVector(out double4x4 A, double4 V) {
// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x double> [[CAST_SPLAT]], <1 x double> poison, <4 x i32> zeroinitializer
// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 8
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 2
// CHECK-NEXT: store double [[TMP3]], ptr [[TMP4]], align 8
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 9
+// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 6
// CHECK-NEXT: store double [[TMP5]], ptr [[TMP6]], align 8
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 10
// CHECK-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8
// CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 11
+// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 14
// CHECK-NEXT: store double [[TMP9]], ptr [[TMP10]], align 8
// CHECK-NEXT: ret void
//
@@ -152,16 +152,16 @@ void TwosSwizzleToScalar(out double4x4 A, double D) {
// CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[V_ADDR]], align 32
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
-// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 2
+// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 8
// CHECK-NEXT: store double [[TMP2]], ptr [[TMP3]], align 8
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
-// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 6
+// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 9
// CHECK-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 10
// CHECK-NEXT: store double [[TMP6]], ptr [[TMP7]], align 8
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
-// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 14
+// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 11
// CHECK-NEXT: store double [[TMP8]], ptr [[TMP9]], align 8
// CHECK-NEXT: ret void
//
@@ -181,13 +181,13 @@ void TwosSwizzleToVector(out double4x4 A, double4 V) {
// CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x double> [[CAST_SPLAT]], <1 x double> poison, <4 x i32> zeroinitializer
// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 12
+// CHECK-NEXT: [[TMP4:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 3
// CHECK-NEXT: store double [[TMP3]], ptr [[TMP4]], align 8
// CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 13
+// CHECK-NEXT: [[TMP6:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 7
// CHECK-NEXT: store double [[TMP5]], ptr [[TMP6]], align 8
// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 14
+// CHECK-NEXT: [[TMP8:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 11
// CHECK-NEXT: store double [[TMP7]], ptr [[TMP8]], align 8
// CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
// CHECK-NEXT: [[TMP10:%.*]] = getelementptr <16 x double>, ptr [[TMP2]], i32 0, i32 15
@@ -208,13 +208,13 @@ void ThreesSwizzleToScalar(out double4x4 A, double D) {
// CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[V_ADDR]], align 32
// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[A_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP0]], i32 0
-// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 3
+// CHECK-NEXT: [[TMP3:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 12
// CHECK-NEXT: store double [[TMP2]], ptr [[TMP3]], align 8
// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP0]], i32 1
-// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 7
+// CHECK-NEXT: [[TMP5:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 13
// CHECK-NEXT: store double [[TMP4]], ptr [[TMP5]], align 8
// CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP0]], i32 2
-// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 11
+// CHECK-NEXT: [[TMP7:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 14
// CHECK-NEXT: store double [[TMP6]], ptr [[TMP7]], align 8
// CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[TMP0]], i32 3
// CHECK-NEXT: [[TMP9:%.*]] = getelementptr <16 x double>, ptr [[TMP1]], i32 0, i32 15
diff --git a/clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl b/clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl
new file mode 100644
index 0000000000000..e0ef9ce92fa81
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/MatrixElement_cbuffer.hlsl
@@ -0,0 +1,96 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes \
+// RUN: -emit-llvm -finclude-default-header -fmatrix-memory-layout=column-major \
+// RUN: -o - %s | FileCheck %s --check-prefix=CHECK,COL-CHECK
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes \
+// RUN: -emit-llvm -finclude-default-header -fmatrix-memory-layout=row-major \
+// RUN: -o - %s | FileCheck %s --check-prefixes=CHECK,ROW-CHECK
+
+
+cbuffer CB {
+ float3x2 Mat;
+};
+
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) float @_Z23getCBufferScalarElementv(
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// COL-CHECK-NEXT: [[MATRIX_BUF_COPY:%.*]] = alloca [2 x <3 x float>], align 4
+// COL-CHECK-NEXT: [[CBUF_DEST:%.*]] = getelementptr inbounds [2 x <3 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// COL-CHECK-NEXT: [[CBUF_LOAD:%.*]] = load <3 x float>, ptr addrspace(2) @Mat, align 4
+// COL-CHECK-NEXT: store <3 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], align 4
+// COL-CHECK-NEXT: [[CBUF_DEST1:%.*]] = getelementptr inbounds [2 x <3 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// COL-CHECK-NEXT: [[CBUF_LOAD2:%.*]] = load <3 x float>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 4
+// COL-CHECK-NEXT: store <3 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], align 4
+// COL-CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[MATRIX_BUF_COPY]], align 4
+// COL-CHECK-NEXT: [[TMP1:%.*]] = extractelement <6 x float> [[TMP0]], i32 4
+// COL-CHECK-NEXT: ret float [[TMP1]]
+//
+// ROW-CHECK-NEXT: [[MATRIX_BUF_COPY:%.*]] = alloca [3 x <2 x float>], align 8
+// ROW-CHECK-NEXT: [[CBUF_DEST:%.*]] = getelementptr inbounds [3 x <2 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// ROW-CHECK-NEXT: [[CBUF_LOAD:%.*]] = load <2 x float>, ptr addrspace(2) @Mat, align 8
+// ROW-CHECK-NEXT: store <2 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], align 8
+// ROW-CHECK-NEXT: [[CBUF_DEST1:%.*]] = getelementptr inbounds [3 x <2 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// ROW-CHECK-NEXT: [[CBUF_LOAD2:%.*]] = load <2 x float>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 8
+// ROW-CHECK-NEXT: store <2 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], align 8
+// ROW-CHECK-NEXT: [[CBUF_DEST3:%.*]] = getelementptr inbounds [3 x <2 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 2
+// ROW-CHECK-NEXT: [[CBUF_LOAD4:%.*]] = load <2 x float>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 32), align 8
+// ROW-CHECK-NEXT: store <2 x float> [[CBUF_LOAD4]], ptr [[CBUF_DEST3]], align 8
+// ROW-CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[MATRIX_BUF_COPY]], align 8
+// ROW-CHECK-NEXT: [[TMP1:%.*]] = extractelement <6 x float> [[TMP0]], i32 3
+// ROW-CHECK-NEXT: ret float [[TMP1]]
+//
+float getCBufferScalarElement() {
+ return Mat._22;
+}
+
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) <4 x float> @_Z23getCBufferSwizzleAccessv(
+// CHECK-SAME: ) #[[ATTR2]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// COL-CHECK-NEXT: [[MATRIX_BUF_COPY:%.*]] = alloca [2 x <3 x float>], align 4
+// COL-CHECK-NEXT: [[CBUF_DEST:%.*]] = getelementptr inbounds [2 x <3 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// COL-CHECK-NEXT: [[CBUF_LOAD:%.*]] = load <3 x float>, ptr addrspace(2) @Mat, align 4
+// COL-CHECK-NEXT: store <3 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], align 4
+// COL-CHECK-NEXT: [[CBUF_DEST1:%.*]] = getelementptr inbounds [2 x <3 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// COL-CHECK-NEXT: [[CBUF_LOAD2:%.*]] = load <3 x float>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 4
+// COL-CHECK-NEXT: store <3 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], align 4
+// COL-CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[MATRIX_BUF_COPY]], align 4
+// COL-CHECK-NEXT: [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x float> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 4>
+// COL-CHECK-NEXT: ret <4 x float> [[TMP1]]
+//
+// ROW-CHECK-NEXT: [[MATRIX_BUF_COPY:%.*]] = alloca [3 x <2 x float>], align 8
+// ROW-CHECK-NEXT: [[CBUF_DEST:%.*]] = getelementptr inbounds [3 x <2 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 0
+// ROW-CHECK-NEXT: [[CBUF_LOAD:%.*]] = load <2 x float>, ptr addrspace(2) @Mat, align 8
+// ROW-CHECK-NEXT: store <2 x float> [[CBUF_LOAD]], ptr [[CBUF_DEST]], align 8
+// ROW-CHECK-NEXT: [[CBUF_DEST1:%.*]] = getelementptr inbounds [3 x <2 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 1
+// ROW-CHECK-NEXT: [[CBUF_LOAD2:%.*]] = load <2 x float>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 16), align 8
+// ROW-CHECK-NEXT: store <2 x float> [[CBUF_LOAD2]], ptr [[CBUF_DEST1]], align 8
+// ROW-CHECK-NEXT: [[CBUF_DEST3:%.*]] = getelementptr inbounds [3 x <2 x float>], ptr [[MATRIX_BUF_COPY]], i32 0, i32 2
+// ROW-CHECK-NEXT: [[CBUF_LOAD4:%.*]] = load <2 x float>, ptr addrspace(2) getelementptr inbounds nuw (i8, ptr addrspace(2) @Mat, i32 32), align 8
+// ROW-CHECK-NEXT: store <2 x float> [[CBUF_LOAD4]], ptr [[CBUF_DEST3]], align 8
+// ROW-CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[MATRIX_BUF_COPY]], align 8
+// ROW-CHECK-NEXT: [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// ROW-CHECK-NEXT: ret <4 x float> [[TMP1]]
+//
+float4 getCBufferSwizzleAccess() {
+ return Mat._11_12_21_22;
+}
+
+// CHECK-LABEL: define hidden noundef nofpclass(nan inf) <2 x float> @_Z22getZeroBasedSwizzleEltu11matrix_typeILm3ELm2EfE(
+// CHECK-SAME: <6 x float> noundef nofpclass(nan inf) [[M:%.*]]) #[[ATTR2]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// COL-CHECK-NEXT: [[M_ADDR:%.*]] = alloca [2 x <3 x float>], align 4
+// COL-CHECK-NEXT: store <6 x float> [[M]], ptr [[M_ADDR]], align 4
+// COL-CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[M_ADDR]], align 4
+// COL-CHECK-NEXT: [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x float> poison, <2 x i32> <i32 1, i32 3>
+// COL-CHECK-NEXT: ret <2 x float> [[TMP1]]
+//
+// ROW-CHECK-NEXT: [[M_ADDR:%.*]] = alloca [3 x <2 x float>], align 4
+// ROW-CHECK-NEXT: store <6 x float> [[M]], ptr [[M_ADDR]], align 4
+// ROW-CHECK-NEXT: [[TMP0:%.*]] = load <6 x float>, ptr [[M_ADDR]], align 4
+// ROW-CHECK-NEXT: [[TMP1:%.*]] = shufflevector <6 x float> [[TMP0]], <6 x float> poison, <2 x i32> <i32 2, i32 1>
+// ROW-CHECK-NEXT: ret <2 x float> [[TMP1]]
+//
+float2 getZeroBasedSwizzleElt(float3x2 M) {
+ return M._m10_m01;
+}
More information about the cfe-commits
mailing list