[llvm-branch-commits] [clang] [clang-tools-extra] [compiler-rt] [flang] [libcxx] [lldb] [llvm] [mlir] [openmp] [polly] [CIR][CodeGen][NFC] Share canUseSingleInheritance (PR #227257)
Henrich Lauko via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 7 05:06:06 PDT 2026
Vedran =?utf-8?q?Miletić?= <vedran at miletic.net>,Daniel Petrovic
<daniel-dev at hotmail.de>,Zahira Ammarguellat <zahira.ammarguellat at intel.com>,Henrich
Lauko <xlauko at mail.muni.cz>,Henrich Lauko <hlauko at nvidia.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/227257 at github.com>
https://github.com/xlauko updated https://github.com/llvm/llvm-project/pull/227257
>From 770a601a9c4fbfd9506335d25d9ecaa3f8661f4c Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 7 Oct 2026 08:10:22 +0100
Subject: [PATCH 01/52] [ARM] Add vtrn/vzip/vuzp shuffle costs. (#223611)
This is an extension to #223310 to add other shuffle cost kinds for the
Neon Arm cost model.
Fixes the cost model of #222833
---
llvm/lib/Target/ARM/ARMISelLowering.cpp | 226 -----------------
.../lib/Target/ARM/ARMTargetTransformInfo.cpp | 9 +-
llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 230 ++++++++++++++++++
llvm/test/Analysis/CostModel/ARM/shuffle.ll | 104 ++++----
4 files changed, 290 insertions(+), 279 deletions(-)
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 0b5f44580d7de..fc1bbb92406e5 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -7155,232 +7155,6 @@ static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
return VT == MVT::v8i8 && M.size() == 8;
}
-static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask,
- unsigned Index) {
- if (Mask.size() == Elements * 2)
- return Index / Elements;
- return Mask[Index] == 0 ? 0 : 1;
-}
-
-// Checks whether the shuffle mask represents a vector transpose (VTRN) by
-// checking that pairs of elements in the shuffle mask represent the same index
-// in each vector, incrementing the expected index by 2 at each step.
-// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
-// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
-// v2={e,f,g,h}
-// WhichResult gives the offset for each element in the mask based on which
-// of the two results it belongs to.
-//
-// The transpose can be represented either as:
-// result1 = shufflevector v1, v2, result1_shuffle_mask
-// result2 = shufflevector v1, v2, result2_shuffle_mask
-// where v1/v2 and the shuffle masks have the same number of elements
-// (here WhichResult (see below) indicates which result is being checked)
-//
-// or as:
-// results = shufflevector v1, v2, shuffle_mask
-// where both results are returned in one vector and the shuffle mask has twice
-// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
-// want to check the low half and high half of the shuffle mask as if it were
-// the other case
-static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
- unsigned EltSz = VT.getScalarSizeInBits();
- if (EltSz == 64)
- return false;
-
- unsigned NumElts = VT.getVectorNumElements();
- if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
- return false;
-
- // If the mask is twice as long as the input vector then we need to check the
- // upper and lower parts of the mask with a matching value for WhichResult
- // FIXME: A mask with only even values will be rejected in case the first
- // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
- // M[0] is used to determine WhichResult
- for (unsigned i = 0; i < M.size(); i += NumElts) {
- WhichResult = SelectPairHalf(NumElts, M, i);
- for (unsigned j = 0; j < NumElts; j += 2) {
- if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
- (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
- return false;
- }
- }
-
- if (M.size() == NumElts*2)
- WhichResult = 0;
-
- return true;
-}
-
-/// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
-/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
-/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
-static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
- unsigned EltSz = VT.getScalarSizeInBits();
- if (EltSz == 64)
- return false;
-
- unsigned NumElts = VT.getVectorNumElements();
- if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
- return false;
-
- for (unsigned i = 0; i < M.size(); i += NumElts) {
- WhichResult = SelectPairHalf(NumElts, M, i);
- for (unsigned j = 0; j < NumElts; j += 2) {
- if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
- (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
- return false;
- }
- }
-
- if (M.size() == NumElts*2)
- WhichResult = 0;
-
- return true;
-}
-
-// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
-// that the mask elements are either all even and in steps of size 2 or all odd
-// and in steps of size 2.
-// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
-// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
-// v2={e,f,g,h}
-// Requires similar checks to that of isVTRNMask with
-// respect the how results are returned.
-static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
- unsigned EltSz = VT.getScalarSizeInBits();
- if (EltSz == 64)
- return false;
-
- unsigned NumElts = VT.getVectorNumElements();
- if (M.size() != NumElts && M.size() != NumElts*2)
- return false;
-
- for (unsigned i = 0; i < M.size(); i += NumElts) {
- WhichResult = SelectPairHalf(NumElts, M, i);
- for (unsigned j = 0; j < NumElts; ++j) {
- if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
- return false;
- }
- }
-
- if (M.size() == NumElts*2)
- WhichResult = 0;
-
- // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
- if (VT.is64BitVector() && EltSz == 32)
- return false;
-
- return true;
-}
-
-/// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
-/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
-/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
-static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
- unsigned EltSz = VT.getScalarSizeInBits();
- if (EltSz == 64)
- return false;
-
- unsigned NumElts = VT.getVectorNumElements();
- if (M.size() != NumElts && M.size() != NumElts*2)
- return false;
-
- unsigned Half = NumElts / 2;
- for (unsigned i = 0; i < M.size(); i += NumElts) {
- WhichResult = SelectPairHalf(NumElts, M, i);
- for (unsigned j = 0; j < NumElts; j += Half) {
- unsigned Idx = WhichResult;
- for (unsigned k = 0; k < Half; ++k) {
- int MIdx = M[i + j + k];
- if (MIdx >= 0 && (unsigned) MIdx != Idx)
- return false;
- Idx += 2;
- }
- }
- }
-
- if (M.size() == NumElts*2)
- WhichResult = 0;
-
- // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
- if (VT.is64BitVector() && EltSz == 32)
- return false;
-
- return true;
-}
-
-// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
-// that pairs of elements of the shufflemask represent the same index in each
-// vector incrementing sequentially through the vectors.
-// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
-// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
-// v2={e,f,g,h}
-// Requires similar checks to that of isVTRNMask with respect the how results
-// are returned.
-static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
- unsigned EltSz = VT.getScalarSizeInBits();
- if (EltSz == 64)
- return false;
-
- unsigned NumElts = VT.getVectorNumElements();
- if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
- return false;
-
- for (unsigned i = 0; i < M.size(); i += NumElts) {
- WhichResult = SelectPairHalf(NumElts, M, i);
- unsigned Idx = WhichResult * NumElts / 2;
- for (unsigned j = 0; j < NumElts; j += 2) {
- if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
- (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
- return false;
- Idx += 1;
- }
- }
-
- if (M.size() == NumElts*2)
- WhichResult = 0;
-
- // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
- if (VT.is64BitVector() && EltSz == 32)
- return false;
-
- return true;
-}
-
-/// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
-/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
-/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
-static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
- unsigned EltSz = VT.getScalarSizeInBits();
- if (EltSz == 64)
- return false;
-
- unsigned NumElts = VT.getVectorNumElements();
- if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
- return false;
-
- for (unsigned i = 0; i < M.size(); i += NumElts) {
- WhichResult = SelectPairHalf(NumElts, M, i);
- unsigned Idx = WhichResult * NumElts / 2;
- for (unsigned j = 0; j < NumElts; j += 2) {
- if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
- (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
- return false;
- Idx += 1;
- }
- }
-
- if (M.size() == NumElts*2)
- WhichResult = 0;
-
- // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
- if (VT.is64BitVector() && EltSz == 32)
- return false;
-
- return true;
-}
-
/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 3ea4ddd10a00a..1599d9d751b22 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1311,10 +1311,17 @@ InstructionCost ARMTTIImpl::getShuffleCost(
// instructions for, for example REV.
if (!Mask.empty()) {
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);
+ unsigned Unused;
if (LT.second.isVector() &&
Mask.size() <= LT.second.getVectorNumElements() &&
(isVREVMask(Mask, LT.second, 16) || isVREVMask(Mask, LT.second, 32) ||
- isVREVMask(Mask, LT.second, 64)))
+ isVREVMask(Mask, LT.second, 64) ||
+ isVTRNMask(Mask, LT.second, Unused) ||
+ isVTRN_v_undef_Mask(Mask, LT.second, Unused) ||
+ isVZIPMask(Mask, LT.second, Unused) ||
+ isVZIP_v_undef_Mask(Mask, LT.second, Unused) ||
+ isVUZPMask(Mask, LT.second, Unused) ||
+ isVUZP_v_undef_Mask(Mask, LT.second, Unused)))
return LT.first;
}
}
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 7dacff87fae36..9c74d1b91a1fe 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -366,6 +366,236 @@ inline bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
return true;
}
+inline unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask,
+ unsigned Index) {
+ if (Mask.size() == Elements * 2)
+ return Index / Elements;
+ return Mask[Index] == 0 ? 0 : 1;
+}
+
+// Checks whether the shuffle mask represents a vector transpose (VTRN) by
+// checking that pairs of elements in the shuffle mask represent the same index
+// in each vector, incrementing the expected index by 2 at each step.
+// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
+// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
+// v2={e,f,g,h}
+// WhichResult gives the offset for each element in the mask based on which
+// of the two results it belongs to.
+//
+// The transpose can be represented either as:
+// result1 = shufflevector v1, v2, result1_shuffle_mask
+// result2 = shufflevector v1, v2, result2_shuffle_mask
+// where v1/v2 and the shuffle masks have the same number of elements
+// (here WhichResult (see below) indicates which result is being checked)
+//
+// or as:
+// results = shufflevector v1, v2, shuffle_mask
+// where both results are returned in one vector and the shuffle mask has twice
+// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
+// want to check the low half and high half of the shuffle mask as if it were
+// the other case
+inline bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
+ unsigned EltSz = VT.getScalarSizeInBits();
+ if (EltSz == 64)
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+ if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
+ return false;
+
+ // If the mask is twice as long as the input vector then we need to check the
+ // upper and lower parts of the mask with a matching value for WhichResult
+ // FIXME: A mask with only even values will be rejected in case the first
+ // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
+ // M[0] is used to determine WhichResult
+ for (unsigned i = 0; i < M.size(); i += NumElts) {
+ WhichResult = SelectPairHalf(NumElts, M, i);
+ for (unsigned j = 0; j < NumElts; j += 2) {
+ if ((M[i + j] >= 0 && (unsigned)M[i + j] != j + WhichResult) ||
+ (M[i + j + 1] >= 0 &&
+ (unsigned)M[i + j + 1] != j + NumElts + WhichResult))
+ return false;
+ }
+ }
+
+ if (M.size() == NumElts * 2)
+ WhichResult = 0;
+
+ return true;
+}
+
+/// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
+/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
+/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
+inline bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT,
+ unsigned &WhichResult) {
+ unsigned EltSz = VT.getScalarSizeInBits();
+ if (EltSz == 64)
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+ if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
+ return false;
+
+ for (unsigned i = 0; i < M.size(); i += NumElts) {
+ WhichResult = SelectPairHalf(NumElts, M, i);
+ for (unsigned j = 0; j < NumElts; j += 2) {
+ if ((M[i + j] >= 0 && (unsigned)M[i + j] != j + WhichResult) ||
+ (M[i + j + 1] >= 0 && (unsigned)M[i + j + 1] != j + WhichResult))
+ return false;
+ }
+ }
+
+ if (M.size() == NumElts * 2)
+ WhichResult = 0;
+
+ return true;
+}
+
+// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
+// that the mask elements are either all even and in steps of size 2 or all odd
+// and in steps of size 2.
+// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
+// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
+// v2={e,f,g,h}
+// Requires similar checks to that of isVTRNMask with
+// respect the how results are returned.
+inline bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
+ unsigned EltSz = VT.getScalarSizeInBits();
+ if (EltSz == 64)
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+ if (M.size() != NumElts && M.size() != NumElts * 2)
+ return false;
+
+ for (unsigned i = 0; i < M.size(); i += NumElts) {
+ WhichResult = SelectPairHalf(NumElts, M, i);
+ for (unsigned j = 0; j < NumElts; ++j) {
+ if (M[i + j] >= 0 && (unsigned)M[i + j] != 2 * j + WhichResult)
+ return false;
+ }
+ }
+
+ if (M.size() == NumElts * 2)
+ WhichResult = 0;
+
+ // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
+ if (VT.is64BitVector() && EltSz == 32)
+ return false;
+
+ return true;
+}
+
+/// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
+/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
+/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
+inline bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT,
+ unsigned &WhichResult) {
+ unsigned EltSz = VT.getScalarSizeInBits();
+ if (EltSz == 64)
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+ if (M.size() != NumElts && M.size() != NumElts * 2)
+ return false;
+
+ unsigned Half = NumElts / 2;
+ for (unsigned i = 0; i < M.size(); i += NumElts) {
+ WhichResult = SelectPairHalf(NumElts, M, i);
+ for (unsigned j = 0; j < NumElts; j += Half) {
+ unsigned Idx = WhichResult;
+ for (unsigned k = 0; k < Half; ++k) {
+ int MIdx = M[i + j + k];
+ if (MIdx >= 0 && (unsigned)MIdx != Idx)
+ return false;
+ Idx += 2;
+ }
+ }
+ }
+
+ if (M.size() == NumElts * 2)
+ WhichResult = 0;
+
+ // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
+ if (VT.is64BitVector() && EltSz == 32)
+ return false;
+
+ return true;
+}
+
+// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
+// that pairs of elements of the shufflemask represent the same index in each
+// vector incrementing sequentially through the vectors.
+// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
+// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
+// v2={e,f,g,h}
+// Requires similar checks to that of isVTRNMask with respect the how results
+// are returned.
+inline bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
+ unsigned EltSz = VT.getScalarSizeInBits();
+ if (EltSz == 64)
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+ if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
+ return false;
+
+ for (unsigned i = 0; i < M.size(); i += NumElts) {
+ WhichResult = SelectPairHalf(NumElts, M, i);
+ unsigned Idx = WhichResult * NumElts / 2;
+ for (unsigned j = 0; j < NumElts; j += 2) {
+ if ((M[i + j] >= 0 && (unsigned)M[i + j] != Idx) ||
+ (M[i + j + 1] >= 0 && (unsigned)M[i + j + 1] != Idx + NumElts))
+ return false;
+ Idx += 1;
+ }
+ }
+
+ if (M.size() == NumElts * 2)
+ WhichResult = 0;
+
+ // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
+ if (VT.is64BitVector() && EltSz == 32)
+ return false;
+
+ return true;
+}
+
+/// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
+/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
+/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
+inline bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT,
+ unsigned &WhichResult) {
+ unsigned EltSz = VT.getScalarSizeInBits();
+ if (EltSz == 64)
+ return false;
+
+ unsigned NumElts = VT.getVectorNumElements();
+ if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
+ return false;
+
+ for (unsigned i = 0; i < M.size(); i += NumElts) {
+ WhichResult = SelectPairHalf(NumElts, M, i);
+ unsigned Idx = WhichResult * NumElts / 2;
+ for (unsigned j = 0; j < NumElts; j += 2) {
+ if ((M[i + j] >= 0 && (unsigned)M[i + j] != Idx) ||
+ (M[i + j + 1] >= 0 && (unsigned)M[i + j + 1] != Idx))
+ return false;
+ Idx += 1;
+ }
+ }
+
+ if (M.size() == NumElts * 2)
+ WhichResult = 0;
+
+ // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
+ if (VT.is64BitVector() && EltSz == 32)
+ return false;
+
+ return true;
+}
+
} // end namespace llvm
#endif // LLVM_LIB_TARGET_ARM_ARMTARGETTRANSFORMINFO_H
diff --git a/llvm/test/Analysis/CostModel/ARM/shuffle.ll b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
index c172a0edd0e91..b051def3e0c47 100644
--- a/llvm/test/Analysis/CostModel/ARM/shuffle.ll
+++ b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
@@ -476,35 +476,35 @@ define void @zip() {
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-NEON-LABEL: 'zip'
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %zip1v2i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %zip2v2i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zipv2i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zip1v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zip2v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zipv4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zip1v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zip2v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %zipv8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %zip1v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %zip2v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v2i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v2i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zipv2i8 = shufflevector <2 x i8> poison, <2 x i8> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zipv4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zipv8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
; CHECK-NEON-NEXT: Cost Model: Found costs of 192 for: %zipv16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %zip1v2i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %zip2v2i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zipv2i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zip1v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zip2v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zipv4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zip1v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zip2v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v2i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v2i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zipv2i16 = shufflevector <2 x i16> poison, <2 x i16> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zipv4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %zipv8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %zip1v16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %zip2v16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
; CHECK-NEON-NEXT: Cost Model: Found costs of 192 for: %zipv16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %zip1v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 0, i32 2>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %zip2v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 1, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zipv2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zip1v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %zip2v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zipv2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip1v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %zip2v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zipv4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zip1v8i32 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %zip2v8i32 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
@@ -622,26 +622,26 @@ define void @uzp() {
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-NEON-LABEL: 'uzp'
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %uzp1v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %uzp2v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp1v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp2v4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzpv4i8 = shufflevector <4 x i8> poison, <4 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzp1v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzp2v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp1v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp2v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %uzpv8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %uzp1v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %uzp2v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp1v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp2v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
; CHECK-NEON-NEXT: Cost Model: Found costs of 192 for: %uzpv16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %uzp1v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %uzp2v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp1v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp2v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzpv4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzp1v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzp2v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp1v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp2v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %uzpv8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %uzp1v16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %uzp2v16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
; CHECK-NEON-NEXT: Cost Model: Found costs of 192 for: %uzpv16i16 = shufflevector <16 x i16> poison, <16 x i16> poison, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %uzp1v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %uzp2v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp1v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %uzp2v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzpv4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzp1v8i32 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %uzp2v8i32 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -756,41 +756,41 @@ define void @trn() {
; CHECK-MVE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; CHECK-NEON-LABEL: 'trn'
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn1v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn2v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 8, i32 0, i32 10, i32 2, i32 12, i32 4, i32 14, i32 6>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn3v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn4v8i8 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 9, i32 1, i32 11, i32 3, i32 13, i32 5, i32 15, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %trn1v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %trn2v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 16, i32 0, i32 18, i32 2, i32 20, i32 4, i32 22, i32 6, i32 24, i32 8, i32 26, i32 10, i32 28, i32 12, i32 30, i32 14>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %trn3v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>
; CHECK-NEON-NEXT: Cost Model: Found costs of 96 for: %trn4v16i8 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 17, i32 1, i32 19, i32 3, i32 21, i32 5, i32 23, i32 7, i32 25, i32 9, i32 27, i32 11, i32 29, i32 13, i32 31, i32 15>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn1v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn2v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 4, i32 0, i32 6, i32 2>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn3v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn4v4i16 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 5, i32 1, i32 7, i32 3>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn1v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn2v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 8, i32 0, i32 10, i32 2, i32 12, i32 4, i32 14, i32 6>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn3v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
; CHECK-NEON-NEXT: Cost Model: Found costs of 48 for: %trn4v8i16 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 9, i32 1, i32 11, i32 3, i32 13, i32 5, i32 15, i32 7>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %trn1v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 0, i32 2>
; CHECK-NEON-NEXT: Cost Model: Found costs of 6 for: %trn2v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 2, i32 0>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %trn3v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 1, i32 3>
; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %trn4v2i32 = shufflevector <2 x i32> poison, <2 x i32> poison, <2 x i32> <i32 3, i32 1>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn1v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn2v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 4, i32 0, i32 6, i32 2>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn3v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 24 for: %trn4v4i32 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 5, i32 1, i32 7, i32 3>
; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %trn1v2i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 0, i32 2>
; CHECK-NEON-NEXT: Cost Model: Found costs of 6 for: %trn2v2i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 2, i32 0>
; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %trn3v2i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 1, i32 3>
; CHECK-NEON-NEXT: Cost Model: Found costs of 12 for: %trn4v2i64 = shufflevector <2 x i64> poison, <2 x i64> poison, <2 x i32> <i32 3, i32 1>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 10 for: %trn1v2f32 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v2f32 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 0, i32 2>
; CHECK-NEON-NEXT: Cost Model: Found costs of 5 for: %trn2v2f32 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 2, i32 0>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 10 for: %trn3v2f32 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v2f32 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 1, i32 3>
; CHECK-NEON-NEXT: Cost Model: Found costs of 10 for: %trn4v2f32 = shufflevector <2 x float> poison, <2 x float> poison, <2 x i32> <i32 3, i32 1>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 20 for: %trn1v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn1v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
; CHECK-NEON-NEXT: Cost Model: Found costs of 20 for: %trn2v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 4, i32 0, i32 6, i32 2>
-; CHECK-NEON-NEXT: Cost Model: Found costs of 20 for: %trn3v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: %trn3v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
; CHECK-NEON-NEXT: Cost Model: Found costs of 20 for: %trn4v4f32 = shufflevector <4 x float> poison, <4 x float> poison, <4 x i32> <i32 5, i32 1, i32 7, i32 3>
; CHECK-NEON-NEXT: Cost Model: Found costs of 4 for: %trn1v2f64 = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> <i32 0, i32 2>
; CHECK-NEON-NEXT: Cost Model: Found costs of 2 for: %trn2v2f64 = shufflevector <2 x double> poison, <2 x double> poison, <2 x i32> <i32 2, i32 0>
>From bd1c9e3efed65cfcc137a183450e56dac3eb9ff4 Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier at nvidia.com>
Date: Wed, 7 Oct 2026 09:24:45 +0200
Subject: [PATCH 02/52] [flang][OpenACC] Preserve compute bindings for CUDA
DEVICE arguments (#229420)
Only use alternate bindings from enclosing structured data constructs
for calls outside OpenACC compute constructs. Inside compute constructs,
use the ordinary symbol binding so data legalization can find the
mapping and explicit mappings or privatization are respected.
---
flang/lib/Lower/ConvertCall.cpp | 6 +
.../OpenACC/acc-data-cuda-device-compute.f90 | 135 ++++++++++++++++++
2 files changed, 141 insertions(+)
create mode 100644 flang/test/Lower/OpenACC/acc-data-cuda-device-compute.f90
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 6cb12b8208203..b6fc18dbe41d0 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -19,6 +19,7 @@
#include "flang/Lower/ConvertVariable.h"
#include "flang/Lower/CustomIntrinsicCall.h"
#include "flang/Lower/HlfirIntrinsics.h"
+#include "flang/Lower/OpenACC.h"
#include "flang/Lower/PFTBuilder.h"
#include "flang/Lower/StatementContext.h"
#include "flang/Lower/SymbolMap.h"
@@ -3337,8 +3338,13 @@ genProcedureRef(CallContext &callContext) {
// binding must be in place for this lowering, which is the only one of
// the actual argument: lowering it again would duplicate any side
// effect of its subscripts.
+ // Inside OpenACC compute constructs, keep the ordinary binding so that
+ // calls use the same mapping as other references, including any mapping
+ // or privatization on the compute construct itself. The OpenACC data
+ // legalization handles references to enclosing data constructs.
std::optional<Fortran::lower::SymMapScope> deviceScope;
if (!isKernelLaunch && isCUDADeviceDummy(arg.characteristics) &&
+ !Fortran::lower::isInsideOpenACCComputeConstruct(builder) &&
Fortran::evaluate::IsVariable(*expr)) {
deviceScope.emplace(callContext.symMap);
if (!mapOpenACCDeviceBindings(*expr, callContext.symMap))
diff --git a/flang/test/Lower/OpenACC/acc-data-cuda-device-compute.f90 b/flang/test/Lower/OpenACC/acc-data-cuda-device-compute.f90
new file mode 100644
index 0000000000000..97ab6569e7455
--- /dev/null
+++ b/flang/test/Lower/OpenACC/acc-data-cuda-device-compute.f90
@@ -0,0 +1,135 @@
+! RUN: bbc -fopenacc -fcuda -emit-hlfir %s -o - | FileCheck %s
+
+! Calls inside compute constructs must use the ordinary symbol binding, like
+! other references in the construct. A DEVICE dummy must not override it with
+! the alternate binding of an enclosing data construct.
+module compute_calls
+ interface
+ attributes(device) subroutine device_sub(a)
+ real :: a(*)
+ end subroutine
+ subroutine host_sub(a)
+ real, device :: a(*)
+ end subroutine
+ attributes(device) subroutine device_c(a) bind(c)
+ real :: a(*)
+ end subroutine
+ end interface
+contains
+ subroutine implicit_mapping(a, n)
+ real :: a(100)
+ integer :: n, i
+ !$acc data copy(a)
+ call host_sub(a)
+ !$acc parallel
+ if (n > 0) call device_sub(a)
+ !$acc end parallel
+ !$acc serial
+ call device_sub(a)
+ !$acc end serial
+ !$acc kernels
+ call device_sub(a)
+ !$acc end kernels
+ !$acc parallel loop
+ do i = 1, n
+ call device_sub(a)
+ end do
+ !$acc end parallel loop
+ !$acc serial loop
+ do i = 1, n
+ call device_sub(a)
+ end do
+ !$acc end serial loop
+ !$acc kernels loop
+ do i = 1, n
+ call device_sub(a)
+ end do
+ !$acc end kernels loop
+ call host_sub(a)
+ !$acc end data
+ end subroutine
+
+! CHECK-LABEL: func.func @_QMcompute_callsPimplicit_mapping
+! CHECK: %[[HOST:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QMcompute_callsFimplicit_mappingEa")
+! CHECK: %[[COPY:.*]] = acc.copyin varPtr(%[[HOST]]#0
+! CHECK: acc.data dataOperands(%[[COPY]]
+! CHECK: %[[DEVICE:.*]]:2 = hlfir.declare %[[COPY]]
+! CHECK: %[[ARG:.*]] = fir.convert %[[DEVICE]]#0
+! CHECK: fir.call @_QPhost_sub(%[[ARG]])
+! CHECK: acc.parallel
+! CHECK: fir.if
+! CHECK: %[[ARG:.*]] = fir.convert %[[HOST]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: acc.serial
+! CHECK: %[[ARG:.*]] = fir.convert %[[HOST]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: acc.kernels
+! CHECK: %[[ARG:.*]] = fir.convert %[[HOST]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: acc.parallel combined(loop)
+! CHECK: acc.loop
+! CHECK: %[[ARG:.*]] = fir.convert %[[HOST]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: acc.serial combined(loop)
+! CHECK: acc.loop
+! CHECK: %[[ARG:.*]] = fir.convert %[[HOST]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: acc.kernels combined(loop)
+! CHECK: acc.loop
+! CHECK: %[[ARG:.*]] = fir.convert %[[HOST]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: %[[ARG:.*]] = fir.convert %[[DEVICE]]#0
+! CHECK: fir.call @_QPhost_sub(%[[ARG]])
+
+ subroutine explicit_mapping(a)
+ real :: a(100)
+ !$acc data copy(a)
+ !$acc parallel present(a)
+ call device_sub(a)
+ !$acc end parallel
+ !$acc serial private(a)
+ call device_sub(a)
+ !$acc end serial
+ !$acc kernels present(a)
+ call device_sub(a)
+ !$acc end kernels
+ !$acc end data
+ end subroutine
+
+! CHECK-LABEL: func.func @_QMcompute_callsPexplicit_mapping
+! CHECK: %[[HOST:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QMcompute_callsFexplicit_mappingEa")
+! CHECK: acc.data
+! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[HOST]]#0
+! CHECK: acc.parallel dataOperands(%[[PRESENT]]
+! CHECK: %[[LOCAL:.*]]:2 = hlfir.declare %[[PRESENT]]
+! CHECK: %[[ARG:.*]] = fir.convert %[[LOCAL]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: %[[PRIVATE:.*]] = acc.private varPtr(%[[HOST]]#0
+! CHECK: acc.serial private(
+! CHECK: %[[LOCAL:.*]]:2 = hlfir.declare %[[PRIVATE]]
+! CHECK: %[[ARG:.*]] = fir.convert %[[LOCAL]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+! CHECK: %[[PRESENT:.*]] = acc.present varPtr(%[[HOST]]#0
+! CHECK: acc.kernels dataOperands(%[[PRESENT]]
+! CHECK: %[[LOCAL:.*]]:2 = hlfir.declare %[[PRESENT]]
+! CHECK: %[[ARG:.*]] = fir.convert %[[LOCAL]]#0
+! CHECK: fir.call @_QPdevice_sub(%[[ARG]])
+
+ subroutine allocatable_mapping(a)
+ real, allocatable :: a(:)
+ !$acc data copy(a)
+ !$acc parallel
+ call device_c(a)
+ !$acc end parallel
+ !$acc end data
+ end subroutine
+
+! CHECK-LABEL: func.func @_QMcompute_callsPallocatable_mapping
+! CHECK: %[[HOST:.*]]:2 = hlfir.declare %{{.*}} uniq_name("_QMcompute_callsFallocatable_mappingEa")
+! CHECK: acc.data
+! CHECK: acc.parallel
+! CHECK: %[[BOX:.*]] = fir.load %[[HOST]]#0
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[BOX]]
+! CHECK: %[[ARG:.*]] = fir.convert %[[ADDR]]
+! CHECK: fir.call @device_c(%[[ARG]])
+end module
>From 8e28512cb132ef9905eb149351fb1d46d33d8c83 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 09:44:22 +0200
Subject: [PATCH 03/52] MachineLICM: Stop checking kill flags in register
pressure estimate (#229201)
This was checking kill flags, or hasOneNonDBGUse as a kill approximation.
The kill flag case appears to be of no practical use. InstrEmitter does not
emit the kill flag in the multiple user cases that would be required and I've
only managed to trigger a different hoisting decision with hand modified MIR.
The flag disagrees with hasOneNonDBGUse in 40 CodeGen tests (mostly
custom-inserter loops such as AMDGPU waterfalls and atomic expansions),
but removing it does not change the output of any CodeGen test.
---
llvm/lib/CodeGen/MachineLICM.cpp | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 50b4856d30d77..d30d2c6bbeee9 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -923,10 +923,6 @@ void MachineLICMImpl::HoistOutOfLoop(MachineDomTreeNode *HeaderN,
}
}
-static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) {
- return MO.isKill() || MRI->hasOneNonDBGUse(MO.getReg());
-}
-
/// Find all virtual register references that are liveout of the preheader to
/// initialize the starting "register pressure". Note this does not count live
/// through (livein but not used) registers.
@@ -989,7 +985,7 @@ MachineLICMImpl::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen,
if (MO.isDef())
RCCost = W.RegWeight;
else {
- bool isKill = isOperandKill(MO, MRI);
+ bool isKill = MRI->hasOneNonDBGUse(Reg);
if (isNew && !isKill && ConsiderUnseenAsDef)
// Haven't seen this, it must be a livein.
RCCost = W.RegWeight;
>From 35546a4476250c1f309cbf8bdb5f5c20044beb6d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 09:44:37 +0200
Subject: [PATCH 04/52] AMDGPU/GlobalISel: Use integer types when narrowing
loads and stores (#229547)
The narrowScalar mutation for loads and stores produced untyped scalars.
For FP-typed values this resulted in an untyped G_OR in the lowerLoad
expansion for unaligned private accesses, which failed to select.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 6 +-
.../AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll | 32 +-
.../AMDGPU/GlobalISel/legalize-load-flat.mir | 538 ++++++-----
.../AMDGPU/GlobalISel/legalize-load-local.mir | 24 +-
.../GlobalISel/legalize-load-private.mir | 889 +++++++++---------
.../load-store-private-unaligned-f64.ll | 361 +++++++
.../regbankselect-amdgcn.s.buffer.load.ll | 32 +-
7 files changed, 1117 insertions(+), 765 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/load-store-private-unaligned-f64.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index dd2fb3a5fd7c9..efdc8dadba1db 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1712,16 +1712,16 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// Split extloads.
if (DstSize > MemSize)
- return std::pair(0, LLT::scalar(MemSize));
+ return std::pair(0, LLT::integer(MemSize));
unsigned MaxSize = maxSizeForAddrSpace(
ST, PtrTy.getAddressSpace(), Op == G_LOAD,
Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
if (MemSize > MaxSize)
- return std::pair(0, LLT::scalar(MaxSize));
+ return std::pair(0, LLT::integer(MaxSize));
uint64_t Align = Query.MMODescrs[0].AlignInBits;
- return std::pair(0, LLT::scalar(Align));
+ return std::pair(0, LLT::integer(Align));
})
.fewerElementsIf(
[=](const LegalityQuery &Query) -> bool {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll
index 5831542bead6b..2f9a68804bcda 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw-fmin-fmax.ll
@@ -139,16 +139,16 @@ define void @atomicrmw_fmax_flat_f64_vv_noret(ptr %ptr, double %val) {
; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.7, align 8, addrspace 5)
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (i32) from %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (i32) from %ir.7 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MAX_F64_e64_2:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub0
; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub1
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.7, align 8, addrspace 5)
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (i32) into %ir.7 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: S_BRANCH %bb.5
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: bb.4.atomicrmw.global:
@@ -202,16 +202,16 @@ define double @atomicrmw_fmax_flat_f64_vv_ret(ptr %ptr, double %val) {
; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.8, align 8, addrspace 5)
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (i32) from %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (i32) from %ir.8 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MAX_F64_e64_2:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub0
; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MAX_F64_e64_2]].sub1
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.8, align 8, addrspace 5)
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (i32) into %ir.8 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: S_BRANCH %bb.5
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: bb.4.atomicrmw.global:
@@ -435,16 +435,16 @@ define void @atomicrmw_fmin_flat_f64_vv_noret(ptr %ptr, double %val) {
; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.7, align 8, addrspace 5)
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (i32) from %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (i32) from %ir.7 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MIN_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MIN_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub0
; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub1
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.7, align 8, addrspace 5)
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.7 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.7, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (i32) into %ir.7 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: S_BRANCH %bb.5
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: bb.4.atomicrmw.global:
@@ -498,16 +498,16 @@ define double @atomicrmw_fmin_flat_f64_vv_ret(ptr %ptr, double %val) {
; GFX10-NEXT: [[V_CMP_NE_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_NE_U64_e64 [[REG_SEQUENCE]], [[COPY9]], implicit $exec
; GFX10-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX10-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY10]], 0, [[COPY]], [[V_CMP_NE_U64_e64_]], implicit $exec
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (s32) from %ir.8, align 8, addrspace 5)
- ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (s32) from %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (load (i32) from %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: [[BUFFER_LOAD_DWORD_OFFEN1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (load (i32) from %ir.8 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[BUFFER_LOAD_DWORD_OFFEN]], %subreg.sub0, [[BUFFER_LOAD_DWORD_OFFEN1]], %subreg.sub1
; GFX10-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE2]], 0, [[REG_SEQUENCE2]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MAX_F64_e64_1:%[0-9]+]]:vreg_64 = nofpexcept V_MAX_F64_e64 0, [[REG_SEQUENCE1]], 0, [[REG_SEQUENCE1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[V_MIN_F64_e64_:%[0-9]+]]:vreg_64 = nofpexcept V_MIN_F64_e64 0, [[V_MAX_F64_e64_]], 0, [[V_MAX_F64_e64_1]], 0, 0, implicit $mode, implicit $exec
; GFX10-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub0
; GFX10-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[V_MIN_F64_e64_]].sub1
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.8, align 8, addrspace 5)
- ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into %ir.8 + 4, basealign 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY11]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (i32) into %ir.8, align 8, addrspace 5)
+ ; GFX10-NEXT: BUFFER_STORE_DWORD_OFFEN [[COPY12]], [[V_CNDMASK_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (i32) into %ir.8 + 4, basealign 8, addrspace 5)
; GFX10-NEXT: S_BRANCH %bb.5
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: bb.4.atomicrmw.global:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index 5c187f74ceddf..c3cc357a31506 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -915,48 +915,46 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i16) from unknown-address + 4, align 4)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
- ; CI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 281474976710655
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[MV]], [[C5]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 281474976710655
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[MV]], [[C4]]
; CI-NEXT: $vgpr0_vgpr1 = COPY [[AND2]](i64)
;
; VI-LABEL: name: test_load_flat_s48_align8
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i16) from unknown-address + 4, align 4)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
- ; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 281474976710655
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[MV]], [[C5]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 281474976710655
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[MV]], [[C4]]
; VI-NEXT: $vgpr0_vgpr1 = COPY [[AND2]](i64)
;
; GFX9PLUS-LABEL: name: test_load_flat_s48_align8
@@ -1028,22 +1026,22 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_flat_s64_align8
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9PLUS-LABEL: name: test_load_flat_s64_align8
@@ -1102,22 +1100,22 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_flat_s64_align4
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9PLUS-LABEL: name: test_load_flat_s64_align4
@@ -1182,15 +1180,15 @@ body: |
; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; CI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_flat_s64_align2
@@ -1203,15 +1201,15 @@ body: |
; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; VI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9PLUS-LABEL: name: test_load_flat_s64_align2
@@ -1340,7 +1338,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -1355,8 +1353,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_flat_s64_align1
@@ -1379,7 +1377,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -1394,8 +1392,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9PLUS-LABEL: name: test_load_flat_s64_align1
@@ -3333,22 +3331,22 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_flat_p1_align8
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9PLUS-LABEL: name: test_load_flat_p1_align8
@@ -3407,22 +3405,22 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_flat_p1_align4
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9PLUS-LABEL: name: test_load_flat_p1_align4
@@ -3497,7 +3495,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -3512,8 +3510,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_flat_p1_align1
@@ -3536,7 +3534,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -3551,8 +3549,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9PLUS-LABEL: name: test_load_flat_p1_align1
@@ -3788,22 +3786,22 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; VI-LABEL: name: test_load_flat_p4_align8
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; GFX9PLUS-LABEL: name: test_load_flat_p4_align8
@@ -3862,22 +3860,22 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; VI-LABEL: name: test_load_flat_p4_align4
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; GFX9PLUS-LABEL: name: test_load_flat_p4_align4
@@ -3942,15 +3940,15 @@ body: |
; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; CI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; VI-LABEL: name: test_load_flat_p4_align2
@@ -3963,15 +3961,15 @@ body: |
; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; VI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; GFX9PLUS-LABEL: name: test_load_flat_p4_align2
@@ -4103,7 +4101,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -4118,8 +4116,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; VI-LABEL: name: test_load_flat_p4_align1
@@ -4142,7 +4140,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -4157,8 +4155,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p4)
;
; GFX9PLUS-LABEL: name: test_load_flat_p4_align1
@@ -9087,17 +9085,17 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 16)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 16)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9105,17 +9103,17 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 16)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 16)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9175,17 +9173,17 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9193,17 +9191,17 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9263,17 +9261,17 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9281,17 +9279,17 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32))
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32))
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9357,29 +9355,29 @@ body: |
; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; CI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
; CI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C2]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s16) from unknown-address + 12)
; CI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i64)
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s16) from unknown-address + 14)
; CI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
- ; CI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR3]](s32)
+ ; CI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR3]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9393,29 +9391,29 @@ body: |
; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; VI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: [[ZEXTLOAD2:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s16) from unknown-address + 8)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s16) from unknown-address + 10)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C2]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD5]](p0) :: (load (s16) from unknown-address + 12)
; VI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i64)
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s16) from unknown-address + 14)
; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
- ; VI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR3]](s32)
+ ; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR3]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9602,7 +9600,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -9617,8 +9615,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
@@ -9633,7 +9631,7 @@ body: |
; CI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; CI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; CI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; CI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; CI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; CI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
; CI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i64)
@@ -9647,8 +9645,8 @@ body: |
; CI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; CI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; CI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; CI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; CI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9672,7 +9670,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -9687,8 +9685,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
@@ -9703,7 +9701,7 @@ body: |
; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; VI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; VI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; VI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; VI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
; VI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i64)
@@ -9717,8 +9715,8 @@ body: |
; VI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; VI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; VI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; VI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; VI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -9991,23 +9989,23 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 32)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 32)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 16)
+ ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 16)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; CI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -10016,23 +10014,23 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 32)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 32)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 16)
+ ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 16)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; VI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -10131,23 +10129,23 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 8)
+ ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 8)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; CI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -10156,23 +10154,23 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 8)
+ ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 8)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; VI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -10287,7 +10285,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -10302,8 +10300,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
@@ -10318,7 +10316,7 @@ body: |
; CI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; CI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; CI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; CI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; CI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; CI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
; CI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i64)
@@ -10332,8 +10330,8 @@ body: |
; CI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; CI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; CI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; CI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; CI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; CI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; CI-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
; CI-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
@@ -10348,7 +10346,7 @@ body: |
; CI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[LOAD4]], [[C1]](i32)
; CI-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
; CI-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[OR13]], [[C3]](i32)
- ; CI-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
+ ; CI-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[SHL14]], [[OR12]]
; CI-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD15]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
; CI-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD19]], [[C]](i64)
@@ -10362,8 +10360,8 @@ body: |
; CI-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[LOAD5]], [[C1]](i32)
; CI-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
; CI-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[OR16]], [[C3]](i32)
- ; CI-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](s32), [[OR17]](s32)
+ ; CI-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[SHL17]], [[OR15]]
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](i32), [[OR17]](i32)
; CI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -10388,7 +10386,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -10403,8 +10401,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
@@ -10419,7 +10417,7 @@ body: |
; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; VI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; VI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; VI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; VI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
; VI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i64)
@@ -10433,8 +10431,8 @@ body: |
; VI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; VI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; VI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; VI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; VI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; VI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; VI-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
; VI-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
@@ -10449,7 +10447,7 @@ body: |
; VI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[LOAD4]], [[C1]](i32)
; VI-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
; VI-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[OR13]], [[C3]](i32)
- ; VI-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
+ ; VI-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[SHL14]], [[OR12]]
; VI-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD15]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
; VI-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD19]], [[C]](i64)
@@ -10463,8 +10461,8 @@ body: |
; VI-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[LOAD5]], [[C1]](i32)
; VI-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
; VI-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[OR16]], [[C3]](i32)
- ; VI-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](s32), [[OR17]](s32)
+ ; VI-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[SHL17]], [[OR15]]
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](i32), [[OR17]](i32)
; VI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -10863,29 +10861,29 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 32)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 32)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 16)
+ ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 16)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; CI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 24
; CI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; CI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p0) :: (load (s32) from unknown-address + 24, align 8)
+ ; CI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p0) :: (load (i32) from unknown-address + 24, align 8)
; CI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i64)
- ; CI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s32) from unknown-address + 28)
- ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; CI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p0) :: (load (i32) from unknown-address + 28)
+ ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -10893,29 +10891,29 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 32)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 32)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 16)
+ ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 16)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 24
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p0) :: (load (s32) from unknown-address + 24, align 8)
+ ; VI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p0) :: (load (i32) from unknown-address + 24, align 8)
; VI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i64)
- ; VI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s32) from unknown-address + 28)
- ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; VI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p0) :: (load (i32) from unknown-address + 28)
+ ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -10999,29 +10997,29 @@ body: |
; CI: liveins: $vgpr0_vgpr1
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 8)
+ ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 8)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; CI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 24
; CI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; CI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p0) :: (load (s32) from unknown-address + 24, align 8)
+ ; CI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p0) :: (load (i32) from unknown-address + 24, align 8)
; CI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i64)
- ; CI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s32) from unknown-address + 28)
- ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; CI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p0) :: (load (i32) from unknown-address + 28)
+ ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -11029,29 +11027,29 @@ body: |
; VI: liveins: $vgpr0_vgpr1
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load (s32), align 8)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p0) :: (load (i32), align 8)
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s32) from unknown-address + 4)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i32) from unknown-address + 4)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s32) from unknown-address + 8, align 8)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p0) :: (load (i32) from unknown-address + 8, align 8)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s32) from unknown-address + 12)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p0) :: (load (i32) from unknown-address + 12)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
- ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p0) :: (load (s32) from unknown-address + 16, align 8)
+ ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p0) :: (load (i32) from unknown-address + 16, align 8)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i64)
- ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s32) from unknown-address + 20)
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p0) :: (load (i32) from unknown-address + 20)
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 24
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
- ; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p0) :: (load (s32) from unknown-address + 24, align 8)
+ ; VI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p0) :: (load (i32) from unknown-address + 24, align 8)
; VI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i64)
- ; VI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s32) from unknown-address + 28)
- ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; VI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p0) :: (load (i32) from unknown-address + 28)
+ ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -11151,7 +11149,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -11166,8 +11164,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
@@ -11182,7 +11180,7 @@ body: |
; CI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; CI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; CI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; CI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; CI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; CI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
; CI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i64)
@@ -11196,8 +11194,8 @@ body: |
; CI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; CI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; CI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; CI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; CI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; CI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; CI-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
; CI-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
@@ -11212,7 +11210,7 @@ body: |
; CI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[LOAD4]], [[C1]](i32)
; CI-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
; CI-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[OR13]], [[C3]](i32)
- ; CI-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
+ ; CI-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[SHL14]], [[OR12]]
; CI-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD15]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
; CI-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD19]], [[C]](i64)
@@ -11226,8 +11224,8 @@ body: |
; CI-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[LOAD5]], [[C1]](i32)
; CI-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
; CI-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[OR16]], [[C3]](i32)
- ; CI-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](s32), [[OR17]](s32)
+ ; CI-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[SHL17]], [[OR15]]
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](i32), [[OR17]](i32)
; CI-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 24
; CI-NEXT: [[PTR_ADD23:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C7]](i64)
; CI-NEXT: [[ZEXTLOAD18:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD23]](p0) :: (load (s8) from unknown-address + 24)
@@ -11242,7 +11240,7 @@ body: |
; CI-NEXT: [[SHL19:%[0-9]+]]:_(i32) = G_SHL [[LOAD6]], [[C1]](i32)
; CI-NEXT: [[OR19:%[0-9]+]]:_(i32) = G_OR [[SHL19]], [[ZEXTLOAD20]]
; CI-NEXT: [[SHL20:%[0-9]+]]:_(i32) = G_SHL [[OR19]], [[C3]](i32)
- ; CI-NEXT: [[OR20:%[0-9]+]]:_(s32) = G_OR [[SHL20]], [[OR18]]
+ ; CI-NEXT: [[OR20:%[0-9]+]]:_(i32) = G_OR [[SHL20]], [[OR18]]
; CI-NEXT: [[PTR_ADD27:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD23]], [[C4]](i64)
; CI-NEXT: [[ZEXTLOAD21:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD27]](p0) :: (load (s8) from unknown-address + 28)
; CI-NEXT: [[PTR_ADD28:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD27]], [[C]](i64)
@@ -11256,8 +11254,8 @@ body: |
; CI-NEXT: [[SHL22:%[0-9]+]]:_(i32) = G_SHL [[LOAD7]], [[C1]](i32)
; CI-NEXT: [[OR22:%[0-9]+]]:_(i32) = G_OR [[SHL22]], [[ZEXTLOAD23]]
; CI-NEXT: [[SHL23:%[0-9]+]]:_(i32) = G_SHL [[OR22]], [[C3]](i32)
- ; CI-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[SHL23]], [[OR21]]
- ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR20]](s32), [[OR23]](s32)
+ ; CI-NEXT: [[OR23:%[0-9]+]]:_(i32) = G_OR [[SHL23]], [[OR21]]
+ ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR20]](i32), [[OR23]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -11281,7 +11279,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p0) :: (load (s8) from unknown-address + 4)
@@ -11296,8 +11294,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p0) :: (load (s8) from unknown-address + 8)
@@ -11312,7 +11310,7 @@ body: |
; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; VI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; VI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; VI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; VI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p0) :: (load (s8) from unknown-address + 12)
; VI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i64)
@@ -11326,8 +11324,8 @@ body: |
; VI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; VI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; VI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; VI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; VI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; VI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
; VI-NEXT: [[PTR_ADD15:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
; VI-NEXT: [[ZEXTLOAD12:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD15]](p0) :: (load (s8) from unknown-address + 16)
@@ -11342,7 +11340,7 @@ body: |
; VI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[LOAD4]], [[C1]](i32)
; VI-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[SHL13]], [[ZEXTLOAD14]]
; VI-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[OR13]], [[C3]](i32)
- ; VI-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[SHL14]], [[OR12]]
+ ; VI-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[SHL14]], [[OR12]]
; VI-NEXT: [[PTR_ADD19:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD15]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD15:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD19]](p0) :: (load (s8) from unknown-address + 20)
; VI-NEXT: [[PTR_ADD20:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD19]], [[C]](i64)
@@ -11356,8 +11354,8 @@ body: |
; VI-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[LOAD5]], [[C1]](i32)
; VI-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[SHL16]], [[ZEXTLOAD17]]
; VI-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[OR16]], [[C3]](i32)
- ; VI-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[SHL17]], [[OR15]]
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](s32), [[OR17]](s32)
+ ; VI-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[SHL17]], [[OR15]]
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR14]](i32), [[OR17]](i32)
; VI-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 24
; VI-NEXT: [[PTR_ADD23:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C7]](i64)
; VI-NEXT: [[ZEXTLOAD18:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD23]](p0) :: (load (s8) from unknown-address + 24)
@@ -11372,7 +11370,7 @@ body: |
; VI-NEXT: [[SHL19:%[0-9]+]]:_(i32) = G_SHL [[LOAD6]], [[C1]](i32)
; VI-NEXT: [[OR19:%[0-9]+]]:_(i32) = G_OR [[SHL19]], [[ZEXTLOAD20]]
; VI-NEXT: [[SHL20:%[0-9]+]]:_(i32) = G_SHL [[OR19]], [[C3]](i32)
- ; VI-NEXT: [[OR20:%[0-9]+]]:_(s32) = G_OR [[SHL20]], [[OR18]]
+ ; VI-NEXT: [[OR20:%[0-9]+]]:_(i32) = G_OR [[SHL20]], [[OR18]]
; VI-NEXT: [[PTR_ADD27:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD23]], [[C4]](i64)
; VI-NEXT: [[ZEXTLOAD21:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD27]](p0) :: (load (s8) from unknown-address + 28)
; VI-NEXT: [[PTR_ADD28:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD27]], [[C]](i64)
@@ -11386,8 +11384,8 @@ body: |
; VI-NEXT: [[SHL22:%[0-9]+]]:_(i32) = G_SHL [[LOAD7]], [[C1]](i32)
; VI-NEXT: [[OR22:%[0-9]+]]:_(i32) = G_OR [[SHL22]], [[ZEXTLOAD23]]
; VI-NEXT: [[SHL23:%[0-9]+]]:_(i32) = G_SHL [[OR22]], [[C3]](i32)
- ; VI-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[SHL23]], [[OR21]]
- ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR20]](s32), [[OR23]](s32)
+ ; VI-NEXT: [[OR23:%[0-9]+]]:_(i32) = G_OR [[SHL23]], [[OR21]]
+ ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR20]](i32), [[OR23]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index e2f94c4f90e89..0cbc6093341a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -2086,11 +2086,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; CI-LABEL: name: test_load_local_s64_align4
@@ -7022,11 +7022,11 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; CI-LABEL: name: test_load_local_p1_align4
@@ -16523,17 +16523,17 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load (s32), addrspace 3)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p3) :: (load (i32), addrspace 3)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s32) from unknown-address + 4, addrspace 3)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p3) :: (load (i32) from unknown-address + 4, addrspace 3)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s32) from unknown-address + 8, addrspace 3)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p3) :: (load (i32) from unknown-address + 8, addrspace 3)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s32) from unknown-address + 12, addrspace 3)
- ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p3) :: (load (i32) from unknown-address + 12, addrspace 3)
+ ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 0695600f9e945..5363a45517ff6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -1721,20 +1721,19 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1743,20 +1742,19 @@ body: |
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1765,20 +1763,19 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1787,20 +1784,19 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1809,20 +1805,19 @@ body: |
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1845,20 +1840,19 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNALIGNED_GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; UNALIGNED_GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; UNALIGNED_GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNALIGNED_GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; UNALIGNED_GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1867,20 +1861,19 @@ body: |
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNALIGNED_GFX10-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[LOAD]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; UNALIGNED_GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
- ; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
+ ; UNALIGNED_GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNALIGNED_GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
- ; UNALIGNED_GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
+ ; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1914,55 +1907,55 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; CI-LABEL: name: test_load_private_s64_align8
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_private_s64_align8
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9-LABEL: name: test_load_private_s64_align8
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX10-LABEL: name: test_load_private_s64_align8
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX11-LABEL: name: test_load_private_s64_align8
@@ -1983,22 +1976,22 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_s64_align8
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_s64_align8
@@ -2029,55 +2022,55 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; CI-LABEL: name: test_load_private_s64_align4
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_private_s64_align4
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9-LABEL: name: test_load_private_s64_align4
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX10-LABEL: name: test_load_private_s64_align4
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX11-LABEL: name: test_load_private_s64_align4
@@ -2098,22 +2091,22 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_s64_align4
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_s64_align4
@@ -2150,15 +2143,15 @@ body: |
; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; SI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; CI-LABEL: name: test_load_private_s64_align2
@@ -2171,15 +2164,15 @@ body: |
; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; CI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_private_s64_align2
@@ -2192,37 +2185,37 @@ body: |
; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; VI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9-LABEL: name: test_load_private_s64_align2
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 2, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 2, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX10-LABEL: name: test_load_private_s64_align2
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 2, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 2, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX11-LABEL: name: test_load_private_s64_align2
@@ -2249,15 +2242,15 @@ body: |
; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; UNALIGNED_GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_s64_align2
@@ -2270,15 +2263,15 @@ body: |
; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; UNALIGNED_GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_s64_align2
@@ -2361,7 +2354,7 @@ body: |
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; SI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -2376,8 +2369,8 @@ body: |
; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; CI-LABEL: name: test_load_private_s64_align1
@@ -2400,7 +2393,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -2415,8 +2408,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; VI-LABEL: name: test_load_private_s64_align1
@@ -2439,7 +2432,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -2454,30 +2447,30 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX9-LABEL: name: test_load_private_s64_align1
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 1, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 1, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX10-LABEL: name: test_load_private_s64_align1
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 1, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 1, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; GFX11-LABEL: name: test_load_private_s64_align1
@@ -2514,7 +2507,7 @@ body: |
; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; UNALIGNED_GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -2529,8 +2522,8 @@ body: |
; UNALIGNED_GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; UNALIGNED_GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_s64_align1
@@ -2553,7 +2546,7 @@ body: |
; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; UNALIGNED_GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -2568,8 +2561,8 @@ body: |
; UNALIGNED_GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; UNALIGNED_GFX10-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_s64_align1
@@ -5833,55 +5826,55 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; CI-LABEL: name: test_load_private_p1_align8
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_private_p1_align8
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9-LABEL: name: test_load_private_p1_align8
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX10-LABEL: name: test_load_private_p1_align8
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX11-LABEL: name: test_load_private_p1_align8
@@ -5902,22 +5895,22 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_p1_align8
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_p1_align8
@@ -5948,55 +5941,55 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; CI-LABEL: name: test_load_private_p1_align4
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_private_p1_align4
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9-LABEL: name: test_load_private_p1_align4
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX10-LABEL: name: test_load_private_p1_align4
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX11-LABEL: name: test_load_private_p1_align4
@@ -6017,22 +6010,22 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_p1_align4
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_p1_align4
@@ -6069,15 +6062,15 @@ body: |
; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; SI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; CI-LABEL: name: test_load_private_p1_align2
@@ -6090,15 +6083,15 @@ body: |
; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; CI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; CI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_private_p1_align2
@@ -6111,37 +6104,37 @@ body: |
; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; VI-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9-LABEL: name: test_load_private_p1_align2
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 2, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 2, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX10-LABEL: name: test_load_private_p1_align2
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 2, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 2, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 2, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 2, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX11-LABEL: name: test_load_private_p1_align2
@@ -6168,15 +6161,15 @@ body: |
; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; UNALIGNED_GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_p1_align2
@@ -6189,15 +6182,15 @@ body: |
; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LOAD]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[SHL]], [[ZEXTLOAD]]
+ ; UNALIGNED_GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[SHL]], [[ZEXTLOAD]]
; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_p1_align2
@@ -6282,7 +6275,7 @@ body: |
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; SI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -6297,8 +6290,8 @@ body: |
; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; CI-LABEL: name: test_load_private_p1_align1
@@ -6321,7 +6314,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -6336,8 +6329,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; VI-LABEL: name: test_load_private_p1_align1
@@ -6360,7 +6353,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -6375,30 +6368,30 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX9-LABEL: name: test_load_private_p1_align1
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 1, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 1, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX10-LABEL: name: test_load_private_p1_align1
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 1, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 1, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; GFX11-LABEL: name: test_load_private_p1_align1
@@ -6435,7 +6428,7 @@ body: |
; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; UNALIGNED_GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -6450,8 +6443,8 @@ body: |
; UNALIGNED_GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; UNALIGNED_GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX10-LABEL: name: test_load_private_p1_align1
@@ -6474,7 +6467,7 @@ body: |
; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; UNALIGNED_GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -6489,8 +6482,8 @@ body: |
; UNALIGNED_GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; UNALIGNED_GFX10-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
;
; UNALIGNED_GFX11-LABEL: name: test_load_private_p1_align1
@@ -15229,17 +15222,17 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15247,17 +15240,17 @@ body: |
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15265,17 +15258,17 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15283,17 +15276,17 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15301,17 +15294,17 @@ body: |
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15333,17 +15326,17 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; UNALIGNED_GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15351,17 +15344,17 @@ body: |
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; UNALIGNED_GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15409,7 +15402,7 @@ body: |
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; SI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -15424,8 +15417,8 @@ body: |
; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; SI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; SI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
; SI-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C]](i32)
@@ -15439,7 +15432,7 @@ body: |
; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; SI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; SI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; SI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; SI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; SI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i32)
; SI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
; SI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i32)
@@ -15453,8 +15446,8 @@ body: |
; SI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; SI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; SI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; SI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; SI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15478,7 +15471,7 @@ body: |
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; CI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -15493,8 +15486,8 @@ body: |
; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; CI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; CI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; CI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
; CI-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C]](i32)
@@ -15508,7 +15501,7 @@ body: |
; CI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; CI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; CI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; CI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; CI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; CI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i32)
; CI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
; CI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i32)
@@ -15522,8 +15515,8 @@ body: |
; CI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; CI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; CI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; CI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; CI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15547,7 +15540,7 @@ body: |
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; VI-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -15562,8 +15555,8 @@ body: |
; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; VI-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; VI-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
; VI-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C]](i32)
@@ -15577,7 +15570,7 @@ body: |
; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; VI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; VI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; VI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; VI-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i32)
; VI-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
; VI-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i32)
@@ -15591,8 +15584,8 @@ body: |
; VI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; VI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; VI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; VI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; VI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15600,17 +15593,17 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 1, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 1, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+ ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 1, addrspace 5)
; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
- ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, align 1, addrspace 5)
+ ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15618,17 +15611,17 @@ body: |
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 1, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 1, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, align 1, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, align 1, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 1, addrspace 5)
+ ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 1, addrspace 5)
; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, align 1, addrspace 5)
- ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, align 1, addrspace 5)
+ ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15666,7 +15659,7 @@ body: |
; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; UNALIGNED_GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -15681,8 +15674,8 @@ body: |
; UNALIGNED_GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; UNALIGNED_GFX9-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C]](i32)
@@ -15696,7 +15689,7 @@ body: |
; UNALIGNED_GFX9-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; UNALIGNED_GFX9-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; UNALIGNED_GFX9-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; UNALIGNED_GFX9-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i32)
; UNALIGNED_GFX9-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i32)
@@ -15710,8 +15703,8 @@ body: |
; UNALIGNED_GFX9-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; UNALIGNED_GFX9-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15735,7 +15728,7 @@ body: |
; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[SHL1]], [[ZEXTLOAD2]]
; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[OR1]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[OR]]
+ ; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SHL2]], [[OR]]
; UNALIGNED_GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD3:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD3]](p5) :: (load (s8) from unknown-address + 4, addrspace 5)
@@ -15750,8 +15743,8 @@ body: |
; UNALIGNED_GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LOAD1]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[SHL4]], [[ZEXTLOAD5]]
; UNALIGNED_GFX10-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[OR4]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[SHL5]], [[OR3]]
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](s32), [[OR5]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[SHL5]], [[OR3]]
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR5]](i32)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD6:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD7]](p5) :: (load (s8) from unknown-address + 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD8:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C]](i32)
@@ -15765,7 +15758,7 @@ body: |
; UNALIGNED_GFX10-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[LOAD2]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[SHL7]], [[ZEXTLOAD8]]
; UNALIGNED_GFX10-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[OR7]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[SHL8]], [[OR6]]
+ ; UNALIGNED_GFX10-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[SHL8]], [[OR6]]
; UNALIGNED_GFX10-NEXT: [[PTR_ADD11:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD7]], [[C4]](i32)
; UNALIGNED_GFX10-NEXT: [[ZEXTLOAD9:%[0-9]+]]:_(i32) = G_ZEXTLOAD [[PTR_ADD11]](p5) :: (load (s8) from unknown-address + 12, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD12:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD11]], [[C]](i32)
@@ -15779,8 +15772,8 @@ body: |
; UNALIGNED_GFX10-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[LOAD3]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[SHL10]], [[ZEXTLOAD11]]
; UNALIGNED_GFX10-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[OR10]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[SHL11]], [[OR9]]
- ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](s32), [[OR11]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[SHL11]], [[OR9]]
+ ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR8]](i32), [[OR11]](i32)
; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
;
@@ -15952,23 +15945,23 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; SI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; SI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; SI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; SI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; SI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; SI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; SI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; SI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -15977,23 +15970,23 @@ body: |
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; CI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -16002,23 +15995,23 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; VI-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -16027,23 +16020,23 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -16052,23 +16045,23 @@ body: |
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; GFX10-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -16103,23 +16096,23 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; UNALIGNED_GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; UNALIGNED_GFX9-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -16128,23 +16121,23 @@ body: |
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; UNALIGNED_GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; UNALIGNED_GFX10-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[DEF]](s64)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
@@ -16191,29 +16184,29 @@ body: |
; SI: liveins: $vgpr0
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; SI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; SI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; SI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; SI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; SI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; SI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; SI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; SI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; SI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; SI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; SI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; SI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; SI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; SI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; SI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; SI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; SI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; SI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; SI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; SI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; SI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; SI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -16221,29 +16214,29 @@ body: |
; CI: liveins: $vgpr0
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; CI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; CI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; CI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; CI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; CI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; CI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; CI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; CI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; CI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; CI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; CI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; CI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; CI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; CI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; CI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; CI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; CI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; CI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; CI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -16251,29 +16244,29 @@ body: |
; VI: liveins: $vgpr0
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; VI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; VI-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; VI-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; VI-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; VI-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; VI-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; VI-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; VI-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; VI-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; VI-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; VI-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; VI-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; VI-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; VI-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; VI-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -16281,29 +16274,29 @@ body: |
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; GFX9-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; GFX9-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; GFX9-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; GFX9-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; GFX9-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; GFX9-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -16311,29 +16304,29 @@ body: |
; GFX10: liveins: $vgpr0
; GFX10-NEXT: {{ $}}
; GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; GFX10-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; GFX10-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; GFX10-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; GFX10-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; GFX10-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; GFX10-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -16363,29 +16356,29 @@ body: |
; UNALIGNED_GFX9: liveins: $vgpr0
; UNALIGNED_GFX9-NEXT: {{ $}}
; UNALIGNED_GFX9-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; UNALIGNED_GFX9-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; UNALIGNED_GFX9-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; UNALIGNED_GFX9-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; UNALIGNED_GFX9-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; UNALIGNED_GFX9-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; UNALIGNED_GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
@@ -16393,29 +16386,29 @@ body: |
; UNALIGNED_GFX10: liveins: $vgpr0
; UNALIGNED_GFX10-NEXT: {{ $}}
; UNALIGNED_GFX10-NEXT: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
- ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load (s32), align 32, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD:%[0-9]+]]:_(i32) = G_LOAD [[COPY]](p5) :: (load (i32), align 32, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; UNALIGNED_GFX10-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s32) from unknown-address + 4, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i32) from unknown-address + 4, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](i32), [[LOAD1]](i32)
; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
; UNALIGNED_GFX10-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s32) from unknown-address + 8, align 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD2:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD1]](p5) :: (load (i32) from unknown-address + 8, align 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s32) from unknown-address + 12, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD3:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD2]](p5) :: (load (i32) from unknown-address + 12, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](i32), [[LOAD3]](i32)
; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p5) :: (load (s32) from unknown-address + 16, align 16, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD4:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD3]](p5) :: (load (i32) from unknown-address + 16, align 16, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD3]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s32) from unknown-address + 20, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD5:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD4]](p5) :: (load (i32) from unknown-address + 20, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](i32), [[LOAD5]](i32)
; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
; UNALIGNED_GFX10-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p5) :: (load (s32) from unknown-address + 24, align 8, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD6:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD5]](p5) :: (load (i32) from unknown-address + 24, align 8, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD5]], [[C]](i32)
- ; UNALIGNED_GFX10-NEXT: [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s32) from unknown-address + 28, addrspace 5)
- ; UNALIGNED_GFX10-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](s32), [[LOAD7]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[LOAD7:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD6]](p5) :: (load (i32) from unknown-address + 28, addrspace 5)
+ ; UNALIGNED_GFX10-NEXT: [[MV3:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD6]](i32), [[LOAD7]](i32)
; UNALIGNED_GFX10-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64), [[MV3]](s64)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-store-private-unaligned-f64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-store-private-unaligned-f64.ll
new file mode 100644
index 0000000000000..40cf5f8650deb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-store-private-unaligned-f64.ll
@@ -0,0 +1,361 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdhsa < %s | FileCheck --check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa -mattr=-unaligned-access-mode < %s | FileCheck --check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa -mattr=+unaligned-access-mode < %s | FileCheck --check-prefix=GFX9-UNALIGNED %s
+
+define double @load_private_f64_align1(ptr addrspace(5) %ptr) {
+; GFX6-LABEL: load_private_f64_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 3, v0
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 5, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 6, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 7, v0
+; GFX6-NEXT: buffer_load_ubyte v1, v1, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v2, v2, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v3, v3, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v4, v4, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v5, v5, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v6, v6, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v7, v0, s[0:3], 0 offen
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+; GFX6-NEXT: buffer_load_ubyte v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(7)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT: s_waitcnt vmcnt(6)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: s_waitcnt vmcnt(5)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: s_waitcnt vmcnt(4)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX6-NEXT: s_waitcnt vmcnt(3)
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: s_waitcnt vmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v7
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v0
+; GFX6-NEXT: v_or_b32_e32 v4, v6, v5
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v3
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_private_f64_align1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: buffer_load_ubyte v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: buffer_load_ubyte v2, v0, s[0:3], 0 offen offset:1
+; GFX9-NEXT: buffer_load_ubyte v3, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT: buffer_load_ubyte v4, v0, s[0:3], 0 offen offset:3
+; GFX9-NEXT: buffer_load_ubyte v5, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: buffer_load_ubyte v6, v0, s[0:3], 0 offen offset:5
+; GFX9-NEXT: buffer_load_ubyte v7, v0, s[0:3], 0 offen offset:7
+; GFX9-NEXT: buffer_load_ubyte v8, v0, s[0:3], 0 offen offset:6
+; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 8, v1
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 24, v4
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshl_or_b32 v3, v6, 8, v5
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 24, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX9-NEXT: v_or3_b32 v1, v4, v5, v3
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-UNALIGNED-LABEL: load_private_f64_align1:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(1)
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0)
+; GFX9-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
+ %val = load double, ptr addrspace(5) %ptr, align 1
+ ret double %val
+}
+
+define double @load_private_f64_align2(ptr addrspace(5) %ptr) {
+; GFX6-LABEL: load_private_f64_align2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 6, v0
+; GFX6-NEXT: buffer_load_ushort v1, v1, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ushort v3, v3, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ushort v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ushort v2, v2, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(3)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_waitcnt vmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_private_f64_align2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: buffer_load_ushort v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: buffer_load_ushort v2, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT: buffer_load_ushort v3, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:6
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v1
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v3
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-UNALIGNED-LABEL: load_private_f64_align2:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(1)
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0)
+; GFX9-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
+ %val = load double, ptr addrspace(5) %ptr, align 2
+ ret double %val
+}
+
+define i64 @load_private_i64_align1(ptr addrspace(5) %ptr) {
+; GFX6-LABEL: load_private_i64_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 3, v0
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 5, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 6, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 7, v0
+; GFX6-NEXT: buffer_load_ubyte v1, v1, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v2, v2, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v3, v3, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v4, v4, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v5, v5, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v6, v6, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v7, v0, s[0:3], 0 offen
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+; GFX6-NEXT: buffer_load_ubyte v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(7)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT: s_waitcnt vmcnt(6)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: s_waitcnt vmcnt(5)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: s_waitcnt vmcnt(4)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX6-NEXT: s_waitcnt vmcnt(3)
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: s_waitcnt vmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v7
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v0
+; GFX6-NEXT: v_or_b32_e32 v4, v6, v5
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v3
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_private_i64_align1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: buffer_load_ubyte v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: buffer_load_ubyte v2, v0, s[0:3], 0 offen offset:1
+; GFX9-NEXT: buffer_load_ubyte v3, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT: buffer_load_ubyte v4, v0, s[0:3], 0 offen offset:3
+; GFX9-NEXT: buffer_load_ubyte v5, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: buffer_load_ubyte v6, v0, s[0:3], 0 offen offset:5
+; GFX9-NEXT: buffer_load_ubyte v7, v0, s[0:3], 0 offen offset:7
+; GFX9-NEXT: buffer_load_ubyte v8, v0, s[0:3], 0 offen offset:6
+; GFX9-NEXT: s_waitcnt vmcnt(6)
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 8, v1
+; GFX9-NEXT: s_waitcnt vmcnt(5)
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX9-NEXT: s_waitcnt vmcnt(4)
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 24, v4
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshl_or_b32 v3, v6, 8, v5
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 24, v7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX9-NEXT: v_or3_b32 v1, v4, v5, v3
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-UNALIGNED-LABEL: load_private_i64_align1:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(1)
+; GFX9-UNALIGNED-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0)
+; GFX9-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
+ %val = load i64, ptr addrspace(5) %ptr, align 1
+ ret i64 %val
+}
+
+define float @load_private_f32_align1(ptr addrspace(5) %ptr) {
+; GFX6-LABEL: load_private_f32_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 3, v0
+; GFX6-NEXT: buffer_load_ubyte v1, v1, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v3, v3, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v2, v2, s[0:3], 0 offen
+; GFX6-NEXT: buffer_load_ubyte v0, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(3)
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX6-NEXT: s_waitcnt vmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: load_private_f32_align1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: buffer_load_ubyte v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: buffer_load_ubyte v2, v0, s[0:3], 0 offen offset:1
+; GFX9-NEXT: buffer_load_ubyte v3, v0, s[0:3], 0 offen offset:3
+; GFX9-NEXT: buffer_load_ubyte v4, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT: s_waitcnt vmcnt(2)
+; GFX9-NEXT: v_lshl_or_b32 v0, v2, 8, v1
+; GFX9-NEXT: s_waitcnt vmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-UNALIGNED-LABEL: load_private_f32_align1:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0)
+; GFX9-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
+ %val = load float, ptr addrspace(5) %ptr, align 1
+ ret float %val
+}
+
+define void @store_private_f64_align1(ptr addrspace(5) %ptr, double %val) {
+; GFX6-LABEL: store_private_f64_align1:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 2, v0
+; GFX6-NEXT: v_bfe_u32 v5, v1, 8, 8
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 1, v0
+; GFX6-NEXT: buffer_store_byte v1, v0, s[0:3], 0 offen
+; GFX6-NEXT: buffer_store_byte v5, v6, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt expcnt(1)
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 3, v0
+; GFX6-NEXT: buffer_store_byte v3, v4, s[0:3], 0 offen
+; GFX6-NEXT: buffer_store_byte v1, v5, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 6, v0
+; GFX6-NEXT: v_bfe_u32 v5, v2, 8, 8
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 5, v0
+; GFX6-NEXT: buffer_store_byte v2, v1, s[0:3], 0 offen
+; GFX6-NEXT: buffer_store_byte v5, v6, s[0:3], 0 offen
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 7, v0
+; GFX6-NEXT: buffer_store_byte v3, v4, s[0:3], 0 offen
+; GFX6-NEXT: buffer_store_byte v1, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_private_f64_align1:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_lshrrev_b16_e32 v3, 8, v1
+; GFX9-NEXT: buffer_store_byte v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: buffer_store_byte v3, v0, s[0:3], 0 offen offset:1
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v1
+; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT: buffer_store_byte v3, v0, s[0:3], 0 offen offset:3
+; GFX9-NEXT: v_lshrrev_b16_e32 v1, 8, v2
+; GFX9-NEXT: buffer_store_byte v2, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: buffer_store_byte v1, v0, s[0:3], 0 offen offset:5
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 24, v2
+; GFX9-NEXT: buffer_store_byte_d16_hi v2, v0, s[0:3], 0 offen offset:6
+; GFX9-NEXT: buffer_store_byte v1, v0, s[0:3], 0 offen offset:7
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-UNALIGNED-LABEL: store_private_f64_align1:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GFX9-UNALIGNED-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0)
+; GFX9-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
+ store double %val, ptr addrspace(5) %ptr, align 1
+ ret void
+}
+
+define void @store_private_f64_align2(ptr addrspace(5) %ptr, double %val) {
+; GFX6-LABEL: store_private_f64_align2:
+; GFX6: ; %bb.0:
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 2, v0
+; GFX6-NEXT: buffer_store_short v1, v0, s[0:3], 0 offen
+; GFX6-NEXT: buffer_store_short v3, v4, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt expcnt(1)
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 6, v0
+; GFX6-NEXT: buffer_store_short v2, v1, s[0:3], 0 offen
+; GFX6-NEXT: buffer_store_short v3, v0, s[0:3], 0 offen
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: store_private_f64_align2:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: buffer_store_short v1, v0, s[0:3], 0 offen
+; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], 0 offen offset:2
+; GFX9-NEXT: buffer_store_short v2, v0, s[0:3], 0 offen offset:4
+; GFX9-NEXT: buffer_store_short_d16_hi v2, v0, s[0:3], 0 offen offset:6
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-UNALIGNED-LABEL: store_private_f64_align2:
+; GFX9-UNALIGNED: ; %bb.0:
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-UNALIGNED-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GFX9-UNALIGNED-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:4
+; GFX9-UNALIGNED-NEXT: s_waitcnt vmcnt(0)
+; GFX9-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
+ store double %val, ptr addrspace(5) %ptr, align 2
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
index c212e1dc69ac0..f8521960ce259 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.buffer.load.ll
@@ -705,11 +705,11 @@ define amdgpu_ps void @s_buffer_load_i256_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
; GFX7-NEXT: [[MV:%[0-9]+]]:vgpr(i256) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128)
- ; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i256)
- ; GFX7-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
+ ; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(i128), [[UV1:%[0-9]+]]:vgpr(i128) = G_UNMERGE_VALUES [[MV]](i256)
+ ; GFX7-NEXT: G_STORE [[UV]](i128), [[DEF]](p1) :: (store (i128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
; GFX7-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C2]](i64)
- ; GFX7-NEXT: G_STORE [[UV1]](s128), [[PTR_ADD]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
+ ; GFX7-NEXT: G_STORE [[UV1]](i128), [[PTR_ADD]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
; GFX7-NEXT: S_ENDPGM 0
;
; GFX1200_1250-LABEL: name: s_buffer_load_i256_vgpr_offset
@@ -728,11 +728,11 @@ define amdgpu_ps void @s_buffer_load_i256_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 0, 0, 0 :: (dereferenceable invariant load (s128), align 8)
; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 16, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 16, align 8)
; GFX1200_1250-NEXT: [[MV:%[0-9]+]]:vgpr(i256) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128)
- ; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i256)
- ; GFX1200_1250-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
+ ; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(i128), [[UV1:%[0-9]+]]:vgpr(i128) = G_UNMERGE_VALUES [[MV]](i256)
+ ; GFX1200_1250-NEXT: G_STORE [[UV]](i128), [[DEF]](p1) :: (store (i128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
; GFX1200_1250-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C2]](i64)
- ; GFX1200_1250-NEXT: G_STORE [[UV1]](s128), [[PTR_ADD]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
+ ; GFX1200_1250-NEXT: G_STORE [[UV1]](i128), [[PTR_ADD]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
; GFX1200_1250-NEXT: S_ENDPGM 0
%val = call i256 @llvm.amdgcn.s.buffer.load.i256(<4 x i32> %rsrc, i32 %soffset, i32 0)
store i256 %val, ptr addrspace(1) poison
@@ -759,17 +759,17 @@ define amdgpu_ps void @s_buffer_load_i512_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 8)
; GFX7-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48, align 8)
; GFX7-NEXT: [[MV:%[0-9]+]]:vgpr(i512) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128), [[AMDGPU_BUFFER_LOAD2]](i128), [[AMDGPU_BUFFER_LOAD3]](i128)
- ; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128), [[UV2:%[0-9]+]]:vgpr(s128), [[UV3:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i512)
- ; GFX7-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
+ ; GFX7-NEXT: [[UV:%[0-9]+]]:vgpr(i128), [[UV1:%[0-9]+]]:vgpr(i128), [[UV2:%[0-9]+]]:vgpr(i128), [[UV3:%[0-9]+]]:vgpr(i128) = G_UNMERGE_VALUES [[MV]](i512)
+ ; GFX7-NEXT: G_STORE [[UV]](i128), [[DEF]](p1) :: (store (i128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
; GFX7-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
; GFX7-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C2]](i64)
- ; GFX7-NEXT: G_STORE [[UV1]](s128), [[PTR_ADD]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
+ ; GFX7-NEXT: G_STORE [[UV1]](i128), [[PTR_ADD]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
; GFX7-NEXT: [[C3:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
; GFX7-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C3]](i64)
- ; GFX7-NEXT: G_STORE [[UV2]](s128), [[PTR_ADD1]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 32, align 8, addrspace 1)
+ ; GFX7-NEXT: G_STORE [[UV2]](i128), [[PTR_ADD1]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 32, align 8, addrspace 1)
; GFX7-NEXT: [[C4:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
; GFX7-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C4]](i64)
- ; GFX7-NEXT: G_STORE [[UV3]](s128), [[PTR_ADD2]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 48, align 8, addrspace 1)
+ ; GFX7-NEXT: G_STORE [[UV3]](i128), [[PTR_ADD2]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 48, align 8, addrspace 1)
; GFX7-NEXT: S_ENDPGM 0
;
; GFX1200_1250-LABEL: name: s_buffer_load_i512_vgpr_offset
@@ -790,17 +790,17 @@ define amdgpu_ps void @s_buffer_load_i512_vgpr_offset(<4 x i32> inreg %rsrc, i32
; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD2:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 32, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 32, align 8)
; GFX1200_1250-NEXT: [[AMDGPU_BUFFER_LOAD3:%[0-9]+]]:vgpr(i128) = G_AMDGPU_BUFFER_LOAD [[BUILD_VECTOR]](<4 x i32>), [[C1]](i32), [[COPY4]], [[C]], 48, 0, 0 :: (dereferenceable invariant load (s128) from unknown-address + 48, align 8)
; GFX1200_1250-NEXT: [[MV:%[0-9]+]]:vgpr(i512) = G_MERGE_VALUES [[AMDGPU_BUFFER_LOAD]](i128), [[AMDGPU_BUFFER_LOAD1]](i128), [[AMDGPU_BUFFER_LOAD2]](i128), [[AMDGPU_BUFFER_LOAD3]](i128)
- ; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(s128), [[UV1:%[0-9]+]]:vgpr(s128), [[UV2:%[0-9]+]]:vgpr(s128), [[UV3:%[0-9]+]]:vgpr(s128) = G_UNMERGE_VALUES [[MV]](i512)
- ; GFX1200_1250-NEXT: G_STORE [[UV]](s128), [[DEF]](p1) :: (store (s128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
+ ; GFX1200_1250-NEXT: [[UV:%[0-9]+]]:vgpr(i128), [[UV1:%[0-9]+]]:vgpr(i128), [[UV2:%[0-9]+]]:vgpr(i128), [[UV3:%[0-9]+]]:vgpr(i128) = G_UNMERGE_VALUES [[MV]](i512)
+ ; GFX1200_1250-NEXT: G_STORE [[UV]](i128), [[DEF]](p1) :: (store (i128) into `ptr addrspace(1) poison`, align 8, addrspace 1)
; GFX1200_1250-NEXT: [[C2:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 16
; GFX1200_1250-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C2]](i64)
- ; GFX1200_1250-NEXT: G_STORE [[UV1]](s128), [[PTR_ADD]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
+ ; GFX1200_1250-NEXT: G_STORE [[UV1]](i128), [[PTR_ADD]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 16, align 8, addrspace 1)
; GFX1200_1250-NEXT: [[C3:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 32
; GFX1200_1250-NEXT: [[PTR_ADD1:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C3]](i64)
- ; GFX1200_1250-NEXT: G_STORE [[UV2]](s128), [[PTR_ADD1]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 32, align 8, addrspace 1)
+ ; GFX1200_1250-NEXT: G_STORE [[UV2]](i128), [[PTR_ADD1]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 32, align 8, addrspace 1)
; GFX1200_1250-NEXT: [[C4:%[0-9]+]]:sgpr(i64) = G_CONSTANT i64 48
; GFX1200_1250-NEXT: [[PTR_ADD2:%[0-9]+]]:sgpr(p1) = nuw inbounds G_PTR_ADD [[DEF]], [[C4]](i64)
- ; GFX1200_1250-NEXT: G_STORE [[UV3]](s128), [[PTR_ADD2]](p1) :: (store (s128) into `ptr addrspace(1) poison` + 48, align 8, addrspace 1)
+ ; GFX1200_1250-NEXT: G_STORE [[UV3]](i128), [[PTR_ADD2]](p1) :: (store (i128) into `ptr addrspace(1) poison` + 48, align 8, addrspace 1)
; GFX1200_1250-NEXT: S_ENDPGM 0
%val = call i512 @llvm.amdgcn.s.buffer.load.i512(<4 x i32> %rsrc, i32 %soffset, i32 0)
store i512 %val, ptr addrspace(1) poison
>From 7016730d795c610d5e62b2662f292e2ffecdd60d Mon Sep 17 00:00:00 2001
From: Abid Qadeer <haqadeer at amd.com>
Date: Wed, 7 Oct 2026 08:47:30 +0100
Subject: [PATCH 05/52] [flang] Take the debug record's location from the
declaration. (#229484)
`DeclareOpConversion` gives the `DbgDeclareOp` it creates the location
of the memref (the alloca) instead of the location of the `XDeclareOp`.
This is problematic in 2 ways.
1. The `XDeclareOp` better represents where the variable is declared.
The memref is where the variable's memory happens to be allocated, and
for a dummy argument it is a block argument, so the record ends up on
the procedure statement. For a subroutine whose dummies `n` and `a` are
declared on lines 2 and 3, their `#dbg_declare` records are both at line
1 today, and at lines 2 and 3 with this change.
2. It requires the alloca to have a valid location. If it does not, the
`DbgDeclareOp` gets none either, and a debug record without a location
is invalid, so it is dropped during the MLIR to LLVM IR translation and
the variable disappears from the debug info altogether.
This PR makes `DeclareOpConversion` use the location of the
`XDeclareOp`. A similar change is done for `DeclareValueOpConversion`.
Assisted By: Cursor
Co-authored-by: Cursor <cursoragent at cursor.com>
---
flang/lib/Optimizer/CodeGen/CodeGen.cpp | 6 ++---
flang/test/Fir/declare-codegen-debug.fir | 31 ++++++++++++++++++++++++
2 files changed, 34 insertions(+), 3 deletions(-)
create mode 100644 flang/test/Fir/declare-codegen-debug.fir
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index e5ba1ae7ff974..4923cfe2e1878 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -322,8 +322,8 @@ struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
if (auto varAttr =
mlir::dyn_cast_or_null<mlir::LLVM::DILocalVariableAttr>(
fusedLoc.getMetadata())) {
- mlir::LLVM::DbgDeclareOp::create(rewriter, memRef.getLoc(), memRef,
- varAttr, nullptr);
+ mlir::LLVM::DbgDeclareOp::create(rewriter, fusedLoc, memRef, varAttr,
+ nullptr);
}
}
rewriter.replaceOp(declareOp, memRef);
@@ -343,7 +343,7 @@ struct DeclareValueOpConversion
if (auto varAttr =
mlir::dyn_cast_or_null<mlir::LLVM::DILocalVariableAttr>(
fusedLoc.getMetadata())) {
- mlir::LLVM::DbgValueOp::create(rewriter, value.getLoc(), value, varAttr,
+ mlir::LLVM::DbgValueOp::create(rewriter, fusedLoc, value, varAttr,
nullptr);
}
}
diff --git a/flang/test/Fir/declare-codegen-debug.fir b/flang/test/Fir/declare-codegen-debug.fir
new file mode 100644
index 0000000000000..5e981dcf95e46
--- /dev/null
+++ b/flang/test/Fir/declare-codegen-debug.fir
@@ -0,0 +1,31 @@
+// Test that the debug intrinsic for a declared variable gets its location from
+// the declaration and not from the allocation.
+// RUN: fir-opt --fir-to-llvm-ir="target=x86_64-unknown-linux-gnu" --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+ func.func @test_codegen() {
+ %0 = fir.alloca i32 loc(#loc_alloca)
+ %1 = fircg.ext_declare %0 uniq_name("x") : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc_x)
+ %2 = fir.alloca i32 loc(unknown)
+ %3 = fircg.ext_declare %2 uniq_name("y") : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc_y)
+ return
+ }
+}
+
+#di_file = #llvm.di_file<"test.f90" in "">
+#di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "Flang", isOptimized = false, emissionKind = Full>
+#di_subprogram = #llvm.di_subprogram<compileUnit = #di_compile_unit, scope = #di_file, name = "test_codegen", file = #di_file, subprogramFlags = Definition>
+#di_int = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer(kind=4)", sizeInBits = 32, encoding = DW_ATE_signed>
+#di_x = #llvm.di_local_variable<scope = #di_subprogram, name = "x", file = #di_file, line = 3, type = #di_int>
+#di_y = #llvm.di_local_variable<scope = #di_subprogram, name = "y", file = #di_file, line = 4, type = #di_int>
+#loc_alloca = loc("test.f90":99:1)
+#loc_x = loc(fused<#di_x>["test.f90":3:11])
+#loc_y = loc(fused<#di_y>["test.f90":4:11])
+
+// CHECK-LABEL: llvm.func @test_codegen
+// CHECK: llvm.intr.dbg.declare #[[X:[^ ]+]] = %{{[^ ]+}} : !llvm.ptr loc(#[[X_LOC:[^)]+]])
+// CHECK: llvm.intr.dbg.declare #[[Y:[^ ]+]] = %{{[^ ]+}} : !llvm.ptr loc(#[[Y_LOC:[^)]+]])
+// CHECK: #[[X_FILE:[^ ]+]] = loc("test.f90":3:11)
+// CHECK: #[[Y_FILE:[^ ]+]] = loc("test.f90":4:11)
+// CHECK: #[[X_LOC]] = loc(fused<#[[X]]>[#[[X_FILE]]])
+// CHECK: #[[Y_LOC]] = loc(fused<#[[Y]]>[#[[Y_FILE]]])
>From b0ba49d8c5433320684dede908bfabe3f2caf633 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i at maskray.me>
Date: Wed, 7 Oct 2026 00:58:29 -0700
Subject: [PATCH 06/52] [clang] Honor -discard-value-names for LLVM IR input
(#229674)
For IR input, CodeGenAction forces setDiscardValueNames(false) to avoid
the crash in #44241, so names created by the optimizer (e.g. the
inliner's `.i`) are kept. c53d807321f6 fixed that crash in
Value::setNameImpl. Apply CodeGenOpts.DiscardValueNames after loading
the module and remove the "ignoring -fdiscard-value-names" warning.
LLM-aided. `clang -O3 -c sqlite3.bc` executes 0.7% fewer instructions.
---
.../clang/Basic/DiagnosticDriverKinds.td | 3 ---
clang/lib/CodeGen/CodeGenAction.cpp | 5 ++---
clang/lib/Driver/ToolChains/Clang.cpp | 9 +-------
clang/test/CodeGen/PR44896.ll | 16 --------------
clang/test/CodeGen/discard-name-values.ll | 21 +++++++++++++++++++
5 files changed, 24 insertions(+), 30 deletions(-)
delete mode 100644 clang/test/CodeGen/PR44896.ll
create mode 100644 clang/test/CodeGen/discard-name-values.ll
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index df6ff0c2cf399..e8a66dae748ff 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -487,9 +487,6 @@ def warn_drv_dwarf_version_limited_by_target : Warning<
InGroup<UnsupportedTargetOpt>;
def warn_c_kext : Warning<
"ignoring -fapple-kext which is valid for C++ and Objective-C++ only">;
-def warn_ignoring_fdiscard_for_bitcode : Warning<
- "ignoring -fdiscard-value-names for LLVM Bitcode">,
- InGroup<UnusedCommandLineArgument>;
def warn_drv_input_file_unused : Warning<
"%0: '%1' input unused%select{ when '%3' is present|}2">,
InGroup<UnusedCommandLineArgument>;
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index de677a385d832..1abd8b0455a19 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -617,9 +617,8 @@ void CodeGenAction::ExecuteAction() {
if (!CodeGenOpts.LinkBitcodePostopt && Result.LinkInModules(&*TheModule))
return;
- // PR44896: Force DiscardValueNames as false. DiscardValueNames cannot be
- // true here because the valued names are needed for reading textual IR.
- Ctx.setDiscardValueNames(false);
+ // Textual IR needs value names while parsing; discard new names afterwards.
+ Ctx.setDiscardValueNames(CodeGenOpts.DiscardValueNames);
Ctx.setDiagnosticHandler(Result.createDiagnosticHandler());
Ctx.setDefaultTargetCPU(TargetOpts.CPU);
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 33507396561e2..2325e7cc552ef 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -5797,15 +5797,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
// Discard value names in no-asserts builds unless otherwise specified.
if (Args.hasFlag(options::OPT_fdiscard_value_names,
- options::OPT_fno_discard_value_names, !IsAssertBuild)) {
- if (Args.hasArg(options::OPT_fdiscard_value_names) &&
- llvm::any_of(Inputs, [](const clang::driver::InputInfo &II) {
- return types::isLLVMIR(II.getType());
- })) {
- D.Diag(diag::warn_ignoring_fdiscard_for_bitcode);
- }
+ options::OPT_fno_discard_value_names, !IsAssertBuild))
CmdArgs.push_back("-discard-value-names");
- }
// Set the main file name, so that debug info works even with
// -save-temps.
diff --git a/clang/test/CodeGen/PR44896.ll b/clang/test/CodeGen/PR44896.ll
deleted file mode 100644
index 6547ea12168f6..0000000000000
--- a/clang/test/CodeGen/PR44896.ll
+++ /dev/null
@@ -1,16 +0,0 @@
-; REQUIRES: x86-registered-target
-; RUN: %clang --target=x86_64-linux-gnu -fdiscard-value-names -S %s -o /dev/null 2>&1 | FileCheck --check-prefix=WARNING %s
-; RUN: %clang --target=x86_64-linux-gnu -S %s -o /dev/null 2>&1 | FileCheck --check-prefix=NOWARNING %s
-; RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm %s -discard-value-names -o /dev/null
-; PR 44896
-
-; WARNING: ignoring -fdiscard-value-names for LLVM Bitcode
-; NOWARNING-NOT: ignoring -fdiscard-value-names for LLVM Bitcode
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64--linux-gnu"
-
-define linkonce ptr @b(ptr %a) {
- ret ptr %a
-}
-
diff --git a/clang/test/CodeGen/discard-name-values.ll b/clang/test/CodeGen/discard-name-values.ll
new file mode 100644
index 0000000000000..246bca2d102a5
--- /dev/null
+++ b/clang/test/CodeGen/discard-name-values.ll
@@ -0,0 +1,21 @@
+; REQUIRES: x86-registered-target
+;; Textual IR keeps its names while parsing; -discard-value-names only drops
+;; names created afterwards (here by the inliner).
+; RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,KEEP
+; RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -emit-llvm %s -discard-value-names -o - | FileCheck %s --check-prefixes=CHECK,DISCARD
+
+; CHECK-LABEL: define {{.*}} @g(i32 {{.*}}%a)
+; KEEP: %y.i = add i32 %a, 1
+; DISCARD: %1 = add i32 %a, 1
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define internal i32 @f(i32 %x) {
+ %y = add i32 %x, 1
+ ret i32 %y
+}
+
+define i32 @g(i32 %a) {
+ %r = call i32 @f(i32 %a)
+ ret i32 %r
+}
>From b7fedffec4c88cc573529ccd82a7422ad2298bc6 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 7 Oct 2026 09:06:05 +0100
Subject: [PATCH 07/52] [GlobalISel] Ensure integer trunc and anyext in call
lowering. (#229346)
This ensures that the trunc and anyext created for argument lowering
have integer types, possibly casting back to fp types. This helps the
Arm backend but also comes up for WebAssembly and AMDGPU too.
---
llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 24 +++++++++++++++++--
.../GlobalISel/irtranslator-byte-type.ll | 3 ++-
.../GlobalISel/irtranslator/args.ll | 3 ++-
.../GlobalISel/irtranslator/call-basics.ll | 12 ++++++----
.../GlobalISel/irtranslator/ret-basics.ll | 3 ++-
5 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 4d97001eb7420..4993056d14d0b 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -460,7 +460,13 @@ void CallLowering::buildCopyFromRegs(MachineIRBuilder &B,
return;
}
- B.buildTrunc(OrigRegs[0], SrcReg);
+ LLT OrigITy = OrigTy.changeToInteger();
+ if (OrigTy == OrigITy) {
+ B.buildTrunc(OrigRegs[0], SrcReg);
+ } else {
+ auto Trunc = B.buildTrunc(OrigITy, SrcReg);
+ B.buildBitcast(OrigRegs[0], Trunc);
+ }
return;
}
@@ -613,7 +619,21 @@ void CallLowering::buildCopyToRegs(MachineIRBuilder &B,
if (PartTy.isVector() == SrcTy.isVector() &&
PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
assert(DstRegs.size() == 1);
- B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
+ // Convert float to integer if needed
+ LLT SrcITy = SrcTy.changeToInteger();
+ LLT PartITy = PartTy.changeToInteger();
+ if (SrcTy != SrcITy)
+ SrcReg = B.buildBitcast(SrcITy, SrcReg).getReg(0);
+
+ // Emit the sext/zext/anyext
+ Register DstIReg =
+ B.buildInstr(ExtendOp,
+ PartITy == PartTy ? DstOp(DstRegs[0]) : DstOp(PartITy),
+ {SrcReg})
+ .getReg(0);
+ // Convert back to the original type if needed
+ if (PartITy != PartTy)
+ B.buildBitcast(DstRegs[0], DstIReg);
return;
}
diff --git a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll
index 2e5bafac09d3b..5d3646f35373f 100644
--- a/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll
+++ b/llvm/test/CodeGen/Generic/GlobalISel/irtranslator-byte-type.ll
@@ -912,7 +912,8 @@ define half @bytecast_b16_to_half(b16 %b) {
; AMDGPU-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; AMDGPU-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY]](i32)
; AMDGPU-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; AMDGPU-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST]](f16)
+ ; AMDGPU-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[BITCAST]](f16)
+ ; AMDGPU-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[BITCAST1]](i16)
; AMDGPU-NEXT: $vgpr0 = COPY [[ANYEXT]](i32)
; AMDGPU-NEXT: SI_RETURN implicit $vgpr0
;
diff --git a/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/args.ll b/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/args.ll
index 844a74686eaea..b40df18d7a0d8 100644
--- a/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/args.ll
+++ b/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/args.ll
@@ -105,7 +105,8 @@ define void @test_f16_arg(half %arg) {
; CHECK-NEXT: liveins: $arguments
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[ARGUMENT_i32_:%[0-9]+]]:i32(i32) = ARGUMENT_i32 0, implicit $arguments
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[ARGUMENT_i32_]](i32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[ARGUMENT_i32_]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
; CHECK-NEXT: RETURN implicit-def $arguments
ret void
}
diff --git a/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/call-basics.ll b/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/call-basics.ll
index ba887f6e6f603..f11f84605806d 100644
--- a/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/call-basics.ll
+++ b/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/call-basics.ll
@@ -132,8 +132,10 @@ define half @call_ret_f16_args_none() {
; CHECK-NEXT: liveins: $arguments
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[CALL:%[0-9]+]]:i32(i32) = CALL @ret_f16_args_none, implicit-def $arguments, implicit $sp32, implicit $sp64
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[CALL]](i32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:i32(i32) = G_ANYEXT [[TRUNC]](f16)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[CALL]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[BITCAST]](f16)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:i32(i32) = G_ANYEXT [[BITCAST1]](i16)
; CHECK-NEXT: RETURN [[ANYEXT]](i32), implicit-def $arguments
%ret = call half @ret_f16_args_none()
ret half %ret
@@ -239,8 +241,10 @@ define void @call_ret_void_args_f16(half %a) {
; CHECK-NEXT: liveins: $arguments
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[ARGUMENT_i32_:%[0-9]+]]:i32(i32) = ARGUMENT_i32 0, implicit $arguments
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(f16) = G_TRUNC [[ARGUMENT_i32_]](i32)
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:i32(i32) = G_ANYEXT [[TRUNC]](f16)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[ARGUMENT_i32_]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i16) = G_BITCAST [[BITCAST]](f16)
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:i32(i32) = G_ANYEXT [[BITCAST1]](i16)
; CHECK-NEXT: CALL @ret_void_args_f16, [[ANYEXT]](i32), implicit-def $arguments, implicit $sp32, implicit $sp64
; CHECK-NEXT: RETURN implicit-def $arguments
call void @ret_void_args_f16(half %a)
diff --git a/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/ret-basics.ll b/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/ret-basics.ll
index 19192f94cc427..a910ae5980bb0 100644
--- a/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/ret-basics.ll
+++ b/llvm/test/CodeGen/WebAssembly/GlobalISel/irtranslator/ret-basics.ll
@@ -99,7 +99,8 @@ define half @test_ret_f16() {
; CHECK-NEXT: liveins: $arguments
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[C:%[0-9]+]]:_(f16) = G_FCONSTANT half 0.000000e+00
- ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:i32(i32) = G_ANYEXT [[C]](f16)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:i32(i32) = G_ANYEXT [[C1]](i16)
; CHECK-NEXT: RETURN [[ANYEXT]](i32), implicit-def $arguments
ret half 0.0
}
>From ee22ca56d6dec9d820ec006aac805a0d2691fc4f Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 10:09:12 +0200
Subject: [PATCH 08/52] AMDGPU/GlobalISel: Fix buffer stores of bf16 (#229673)
The 16-bit store source fixup only handled i8, i16 and f16, so bf16
values reached register bank legalization unextended and failed. Bitcast
16-bit FP sources to i16 before any-extending, rather than any-extending
the FP value directly.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 11 +-
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 14 +-
.../llvm.amdgcn.raw.buffer.store.ll | 131 ++++++++++-
.../llvm.amdgcn.raw.tbuffer.store.f16.ll | 214 ++++++++----------
...m.amdgcn.struct.buffer.store.format.f16.ll | 22 +-
5 files changed, 247 insertions(+), 145 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index efdc8dadba1db..360ca956f22b5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6727,12 +6727,15 @@ Register AMDGPULegalizerInfo::fixStoreSourceType(MachineIRBuilder &B,
Ty = getBitcastRegisterType(Ty);
VData = B.buildBitcast(Ty, VData).getReg(0);
}
- // Fixup illegal register types for i8 stores.
- if (Ty == LLT::integer(8) || Ty == LLT::integer(16) || Ty == F16) {
- Register AnyExt = B.buildAnyExt(LLT::integer(32), VData).getReg(0);
- return AnyExt;
+ if (Ty.isFloat(16)) {
+ Ty = LLT::integer(16);
+ VData = B.buildBitcast(Ty, VData).getReg(0);
}
+ // Fixup illegal register types for i8 stores.
+ if (Ty == LLT::integer(8) || Ty == LLT::integer(16))
+ return B.buildAnyExt(LLT::integer(32), VData).getReg(0);
+
if (Ty.isVector()) {
if (Ty.getElementType().getSizeInBits() == 16 && Ty.getNumElements() <= 4) {
if (IsFormat)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index bef4b13865585..7103a6bd32ed4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -44,11 +44,9 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -93,10 +91,8 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[COPY7]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index eda53baacb1b6..273b50c460ca5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -968,16 +968,48 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
}
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_bf16(<4 x i32> inreg %rsrc, bfloat %val, i32 %voffset, i32 inreg %soffset) {
+ ; GFX8-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_bf16
+ ; GFX8: bb.1 (%ir-block.0):
+ ; GFX8-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX8-NEXT: {{ $}}
+ ; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX8-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX8-NEXT: BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (bf16), align 1, addrspace 8)
+ ; GFX8-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_bf16
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (bf16), align 1, addrspace 8)
+ ; GFX12-NEXT: S_ENDPGM 0
+ call void @llvm.amdgcn.raw.buffer.store.bf16(bfloat %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+ ret void
+}
+
define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
; GFX8-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
; GFX8: bb.1 (%ir-block.0):
@@ -1067,6 +1099,95 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
ret void
}
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2bf16(<4 x i32> inreg %rsrc, <2 x bfloat> %val, i32 %voffset, i32 inreg %soffset) {
+ ; GFX8-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2bf16
+ ; GFX8: bb.1 (%ir-block.0):
+ ; GFX8-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX8-NEXT: {{ $}}
+ ; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX8-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX8-NEXT: BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x bf16>), align 1, addrspace 8)
+ ; GFX8-NEXT: S_ENDPGM 0
+ ;
+ ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2bf16
+ ; GFX12: bb.1 (%ir-block.0):
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: {{ $}}
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x bf16>), align 1, addrspace 8)
+ ; GFX12-NEXT: S_ENDPGM 0
+ call void @llvm.amdgcn.raw.buffer.store.v2bf16(<2 x bfloat> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+ ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4bf16(<4 x i32> inreg %rsrc, <4 x bfloat> %val, i32 %voffset, i32 inreg %soffset) {
+ ; GFX8-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4bf16
+ ; GFX8: bb.1 (%ir-block.0):
+ ; GFX8-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX8-NEXT: {{ $}}
+ ; GFX8-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX8-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX8-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX8-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX8-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX8-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX8-NEXT: BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x bf16>), align 1, addrspace 8)
+ ; GFX8-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4bf16
+ ; GFX1200: bb.1 (%ir-block.0):
+ ; GFX1200-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX1200-NEXT: {{ $}}
+ ; GFX1200-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1200-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1200-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1200-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1200-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1200-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1200-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX1200-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1200-NEXT: BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x bf16>), align 1, addrspace 8)
+ ; GFX1200-NEXT: S_ENDPGM 0
+ ;
+ ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4bf16
+ ; GFX1250: bb.1 (%ir-block.0):
+ ; GFX1250-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX1250-NEXT: {{ $}}
+ ; GFX1250-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX1250-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX1250-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX1250-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; GFX1250-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX1250-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX1250-NEXT: BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x bf16>), align 1, addrspace 8)
+ ; GFX1250-NEXT: S_ENDPGM 0
+ call void @llvm.amdgcn.raw.buffer.store.v4bf16(<4 x bfloat> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+ ret void
+}
+
define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
; GFX8-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
; GFX8: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 363f1c2bf96bc..e4dba0cfa3e47 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -39,16 +39,14 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -327,38 +325,36 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -481,41 +477,39 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
; GFX12-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -640,42 +634,40 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def dead $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY9]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def dead $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -726,16 +718,14 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret void
@@ -777,16 +767,14 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret void
@@ -828,16 +816,14 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret void
@@ -879,16 +865,14 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index 52a81b32b6d97..1c255f19a2a5f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -43,18 +43,16 @@ define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY9]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
>From 2cc7a05005b9ac07ab8a6d99d1750954b4e7a84e Mon Sep 17 00:00:00 2001
From: Mikael Holmen <mikael.holmen at ericsson.com>
Date: Wed, 7 Oct 2026 10:12:57 +0200
Subject: [PATCH 09/52] [MachineCombiner] Don't count debug instructions in
block size (#225765)
The method MachineCombinerImpl::combineInstructions made decisions based
on
MBB->size() > inc_threshold
in two places, and since MachineBasicBlock::size() includes DBG_VALUE,
the existence of debug info can affect the resulting code.
Use sizeWithoutDebugLargerThan() instead to get the same code with and
without debug info.
I originally found this problem for my out-of-tree target. Then I used
Claude Opus 5 to convert the mir test for my out-of-tree target to one
for AArch64.
The other place, in the register pressure reduction part, was found by
code inspection. It is guarded by MustReduceRegisterPressure, which only
PowerPC uses. The PowerPC test for it was created by Claude Opus 5.
---
llvm/lib/CodeGen/MachineCombiner.cpp | 4 +-
.../AArch64/machine-combiner-dbg-value.mir | 66 +++++++++++++++
.../PowerPC/machine-combiner-dbg-value.mir | 81 +++++++++++++++++++
3 files changed, 149 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/machine-combiner-dbg-value.mir
create mode 100644 llvm/test/CodeGen/PowerPC/machine-combiner-dbg-value.mir
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index ec2a9efa7e09c..fd0a84e9fc514 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -632,7 +632,7 @@ bool MachineCombinerImpl::combineInstructions(MachineBasicBlock *MBB) {
if (DoRegPressureReduce &&
getCombinerObjective(P) ==
CombinerObjective::MustReduceRegisterPressure) {
- if (MBB->size() > inc_threshold) {
+ if (MBB->sizeWithoutDebugLargerThan(inc_threshold)) {
// Use incremental depth updates for basic blocks above threshold
IncrementalUpdate = true;
LastUpdate = BlockIter;
@@ -678,7 +678,7 @@ bool MachineCombinerImpl::combineInstructions(MachineBasicBlock *MBB) {
InstrIdxForVirtReg, P,
!IncrementalUpdate) &&
preservesResourceLen(MBB, BlockTrace, InsInstrs, DelInstrs)) {
- if (MBB->size() > inc_threshold) {
+ if (MBB->sizeWithoutDebugLargerThan(inc_threshold)) {
// Use incremental depth updates for basic blocks above treshold
IncrementalUpdate = true;
LastUpdate = BlockIter;
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner-dbg-value.mir b/llvm/test/CodeGen/AArch64/machine-combiner-dbg-value.mir
new file mode 100644
index 0000000000000..9ec8556b8dc4c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-combiner-dbg-value.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -mcpu=neoverse-n2 -run-pass mir-strip-debug,machine-combiner -mir-strip-debugify-only=0 -machine-combiner-inc-threshold=7 %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mcpu=neoverse-n2 -run-pass machine-combiner,mir-strip-debug -mir-strip-debugify-only=0 -machine-combiner-inc-threshold=7 %s -o - | FileCheck %s
+
+# We should get the same result when machine-combiner sees debug info as when
+# it does not. "-mir-strip-debugify-only=0" is needed for mir-strip-debug to
+# remove all debug info.
+#
+# This covers the MBB->size() check in the critical path part of
+# MachineCombinerImpl::combineInstructions.
+
+--- |
+ define void @test() !dbg !5 {
+ ret void
+ }
+ !llvm.dbg.cu = !{!0}
+ !llvm.module.flags = !{!2}
+ !2 = !{i32 2, !"Debug Info Version", i32 3}
+ !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, emissionKind: FullDebug)
+ !1 = !DIFile(filename: "t.c", directory: "/")
+ !5 = distinct !DISubprogram(name: "test", scope: !1, file: !1, type: !6, spFlags: DISPFlagDefinition, unit: !0)
+ !6 = !DISubroutineType(types: !7)
+ !7 = !{null}
+ !9 = !DILocation(line: 0, scope: !5)
+ !10 = !DILocalVariable(name: "x", scope: !5)
+...
+---
+name: test
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $s0, $s1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr32 = COPY $s0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr32 = COPY $s1
+ ; CHECK-NEXT: [[FADDSrr:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[COPY]], [[COPY]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDSrr1:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[COPY1]], [[COPY]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDSrr2:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[FADDSrr]], killed [[FADDSrr1]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDSrr3:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[FADDSrr2]], [[COPY1]], implicit $fpcr
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:fpr32 = COPY [[FADDSrr3]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[FADDSrr4:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[COPY1]], [[COPY]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDSrr5:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[COPY2]], [[FADDSrr4]], implicit $fpcr
+ ; CHECK-NEXT: [[FADDSrr6:%[0-9]+]]:fpr32 = nsz reassoc FADDSrr [[FADDSrr5]], [[COPY2]], implicit $fpcr
+ ; CHECK-NEXT: B %bb.1
+ bb.0:
+ liveins: $s0, $s1
+ DBG_VALUE 0, $noreg, !10, !DIExpression(), debug-location !9
+ %0:fpr32 = COPY $s0
+ %1:fpr32 = COPY $s1
+ %2:fpr32 = nsz reassoc FADDSrr %0, %0, implicit $fpcr
+ %3:fpr32 = nsz reassoc FADDSrr %2, %1, implicit $fpcr
+ %4:fpr32 = nsz reassoc FADDSrr %3, %0, implicit $fpcr
+ %5:fpr32 = nsz reassoc FADDSrr %4, %1, implicit $fpcr
+ %6:fpr32 = COPY %5
+ bb.1:
+ %7:fpr32 = nsz reassoc FADDSrr %1, %0, implicit $fpcr
+ %8:fpr32 = nsz reassoc FADDSrr %6, %7, implicit $fpcr
+ %9:fpr32 = nsz reassoc FADDSrr %8, %6, implicit $fpcr
+ B %bb.1
+...
diff --git a/llvm/test/CodeGen/PowerPC/machine-combiner-dbg-value.mir b/llvm/test/CodeGen/PowerPC/machine-combiner-dbg-value.mir
new file mode 100644
index 0000000000000..0c1266ff0aaa4
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/machine-combiner-dbg-value.mir
@@ -0,0 +1,81 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -O3 -ppc-fma-rp-factor=0.0 -run-pass mir-strip-debug,machine-combiner -mir-strip-debugify-only=0 -machine-combiner-inc-threshold=11 %s -o - | FileCheck %s
+# RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -O3 -ppc-fma-rp-factor=0.0 -run-pass machine-combiner,mir-strip-debug -mir-strip-debugify-only=0 -machine-combiner-inc-threshold=11 %s -o - | FileCheck %s
+
+# We should get the same result when machine-combiner sees debug info as when
+# it does not. "-mir-strip-debugify-only=0" is needed for mir-strip-debug to
+# remove all debug info. "-ppc-fma-rp-factor=0.0" is needed to make
+# PPCInstrInfo::shouldReduceRegisterPressure return true.
+#
+# This covers the MBB->size() check in the register pressure reduction part of
+# MachineCombinerImpl::combineInstructions.
+
+--- |
+ define void @test() !dbg !5 {
+ ret void
+ }
+ !llvm.dbg.cu = !{!0}
+ !llvm.module.flags = !{!2}
+ !2 = !{i32 2, !"Debug Info Version", i32 3}
+ !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, emissionKind: FullDebug)
+ !1 = !DIFile(filename: "t.c", directory: "/")
+ !5 = distinct !DISubprogram(name: "test", scope: !1, file: !1, type: !6, spFlags: DISPFlagDefinition, unit: !0)
+ !6 = !DISubroutineType(types: !7)
+ !7 = !{null}
+ !9 = !DILocation(line: 0, scope: !5)
+ !10 = !DILocalVariable(name: "x", scope: !5)
+...
+---
+name: test
+tracksRegLiveness: true
+constants:
+ - id: 0
+ value: float f0x2D9299FF
+ alignment: 4
+body: |
+ ; CHECK-LABEL: name: test
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $f2, $x2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vssrc = COPY $f2
+ ; CHECK-NEXT: [[XSMULSP:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSMULSP [[COPY]], [[COPY]]
+ ; CHECK-NEXT: [[XSADDSP:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSADDSP [[XSMULSP]], [[COPY]]
+ ; CHECK-NEXT: [[XSMULSP1:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSMULSP [[XSADDSP]], [[COPY]]
+ ; CHECK-NEXT: [[ADDIStocHA8_:%[0-9]+]]:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
+ ; CHECK-NEXT: [[DFLOADf32_:%[0-9]+]]:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0, killed [[ADDIStocHA8_]] :: (load (s32) from constant-pool)
+ ; CHECK-NEXT: [[ADDIStocHA8_1:%[0-9]+]]:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.1
+ ; CHECK-NEXT: [[DFLOADf32_1:%[0-9]+]]:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.1, killed [[ADDIStocHA8_1]] :: (load (s32) from constant-pool)
+ ; CHECK-NEXT: [[XSMADDASP:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSMADDASP killed [[XSMULSP1]], [[COPY]], [[DFLOADf32_1]]
+ ; CHECK-NEXT: [[XSMADDASP1:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSMADDASP killed [[XSMADDASP]], [[COPY]], killed [[DFLOADf32_]]
+ ; CHECK-NEXT: [[XSADDSP1:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSADDSP [[COPY]], [[COPY]]
+ ; CHECK-NEXT: [[XSADDSP2:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSADDSP [[XSADDSP1]], [[XSADDSP]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vssrc = COPY [[XSMADDASP1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[XSADDSP3:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSADDSP [[COPY]], [[COPY]]
+ ; CHECK-NEXT: [[XSMULSP2:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSMULSP [[XSADDSP3]], [[COPY1]]
+ ; CHECK-NEXT: [[XSMULSP3:%[0-9]+]]:vssrc = nsz reassoc nofpexcept XSMULSP [[XSMULSP2]], [[COPY]]
+ ; CHECK-NEXT: B %bb.1
+ bb.0:
+ liveins: $f2, $x2
+ DBG_VALUE 0, $noreg, !10, !DIExpression(), debug-location !9
+ %0:vssrc = COPY $f2
+ %1:vssrc = nsz reassoc nofpexcept XSMULSP %0, %0
+ %2:vssrc = nsz reassoc nofpexcept XSADDSP %1, %0
+ %3:vssrc = nsz reassoc nofpexcept XSMULSP %2, %0
+ %4:vssrc = nsz reassoc nofpexcept XSSUBSP %0, %0
+ %5:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
+ %6:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0, killed %5 :: (load (s32) from constant-pool)
+ %7:vssrc = nsz reassoc nofpexcept XSMADDASP %3, killed %4, killed %6
+ %8:vssrc = nsz reassoc nofpexcept XSADDSP %0, %0
+ %9:vssrc = nsz reassoc nofpexcept XSADDSP %8, %2
+ %10:vssrc = COPY %7
+ bb.1:
+ %11:vssrc = nsz reassoc nofpexcept XSADDSP %0, %0
+ %12:vssrc = nsz reassoc nofpexcept XSMULSP %11, %10
+ %13:vssrc = nsz reassoc nofpexcept XSMULSP %12, %0
+ B %bb.1
+...
>From f2e7e3f1b81c57bdeddbe890f088b2e829b0956e Mon Sep 17 00:00:00 2001
From: algol83 <drom.home at gmail.com>
Date: Wed, 7 Oct 2026 11:15:52 +0300
Subject: [PATCH 10/52] [clang-tidy] Make range source classes configurable
(#226193)
The check currently hardcodes the container types accepted as sources in
range-based for loops. This prevents it from recognizing
project-specific range-like containers even when they expose the
operations needed by the check.
Add a `ForRangeLoopClasses` option, analogous to `VectorLikeClasses`,
for configuring those source container classes. The existing
standard-library types remain the defaults, so the current behavior is
preserved.
The test covers a custom range-like source and verifies the generated
`reserve(range.size())` fix.
Tests:
- `ninja check-clang-extra-clang-tidy-checkers-performance` (48/48
passed)
**AI assistance:** I used OpenAI Codex to help implement, test, and
document this change. I reviewed the resulting code and text, understand
the changes, and take full responsibility for the contribution.
---------
Co-authored-by: EugeneZelenko <eugene.zelenko at gmail.com>
Co-authored-by: Baranov Victor <bar.victor.2002 at gmail.com>
---
.../InefficientVectorOperationCheck.cpp | 21 +++++---
.../InefficientVectorOperationCheck.h | 1 +
clang-tools-extra/docs/ReleaseNotes.md | 5 ++
.../inefficient-vector-operation.rst | 15 ++++--
...nt-vector-operation-vectorlike-classes.cpp | 53 ++++++++++++++++++-
5 files changed, 83 insertions(+), 12 deletions(-)
diff --git a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
index 93cd3a2229a25..744fa8486d21f 100644
--- a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
@@ -58,10 +58,9 @@ static constexpr char LoopInitVarName[] = "loop_init_var";
static constexpr char LoopEndExprName[] = "loop_end_expr";
static constexpr char RangeLoopName[] = "for_range_loop";
-static ast_matchers::internal::Matcher<Expr> supportedContainerTypesMatcher() {
- return hasType(cxxRecordDecl(hasAnyName(
- "::std::vector", "::std::set", "::std::unordered_set", "::std::map",
- "::std::unordered_map", "::std::array", "::std::deque")));
+static ast_matchers::internal::Matcher<Expr>
+supportedContainerTypesMatcher(ArrayRef<StringRef> ContainerTypes) {
+ return hasType(cxxRecordDecl(hasAnyName(ContainerTypes)));
}
namespace {
@@ -77,12 +76,18 @@ InefficientVectorOperationCheck::InefficientVectorOperationCheck(
: ClangTidyCheck(Name, Context),
VectorLikeClasses(utils::options::parseStringList(
Options.get("VectorLikeClasses", "::std::vector"))),
+ ForRangeLoopClasses(utils::options::parseStringList(Options.get(
+ "ForRangeLoopClasses",
+ "::std::array;::std::deque;::std::map;::std::set;"
+ "::std::unordered_map;::std::unordered_set;::std::vector"))),
EnableProto(Options.get("EnableProto", false)) {}
void InefficientVectorOperationCheck::storeOptions(
ClangTidyOptions::OptionMap &Opts) {
Options.store(Opts, "VectorLikeClasses",
utils::options::serializeStringList(VectorLikeClasses));
+ Options.store(Opts, "ForRangeLoopClasses",
+ utils::options::serializeStringList(ForRangeLoopClasses));
Options.store(Opts, "EnableProto", EnableProto);
}
@@ -146,10 +151,10 @@ void InefficientVectorOperationCheck::addMatcher(
// FIXME: Support more complex range-expressions.
Finder->addMatcher(
cxxForRangeStmt(
- hasRangeInit(
- anyOf(declRefExpr(supportedContainerTypesMatcher()),
- memberExpr(hasObjectExpression(unless(hasSideEffects())),
- supportedContainerTypesMatcher()))),
+ hasRangeInit(anyOf(
+ declRefExpr(supportedContainerTypesMatcher(ForRangeLoopClasses)),
+ memberExpr(hasObjectExpression(unless(hasSideEffects())),
+ supportedContainerTypesMatcher(ForRangeLoopClasses)))),
HasInterestingLoopBody, InInterestingCompoundStmt)
.bind(RangeLoopName),
this);
diff --git a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
index 5f3b88f51d626..d976f6b28e363 100644
--- a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
@@ -37,6 +37,7 @@ class InefficientVectorOperationCheck : public ClangTidyCheck {
const ast_matchers::DeclarationMatcher &AppendMethodDecl,
StringRef AppendCallName, ast_matchers::MatchFinder *Finder);
const std::vector<StringRef> VectorLikeClasses;
+ const std::vector<StringRef> ForRangeLoopClasses;
// If true, also check inefficient operations for proto repeated fields.
bool EnableProto;
diff --git a/clang-tools-extra/docs/ReleaseNotes.md b/clang-tools-extra/docs/ReleaseNotes.md
index 42451c4590592..20b612509d7f7 100644
--- a/clang-tools-extra/docs/ReleaseNotes.md
+++ b/clang-tools-extra/docs/ReleaseNotes.md
@@ -268,6 +268,11 @@ infrastructure are described first, followed by tool-specific sections.
offered when an argument covers only part of a macro expansion, as it then
has no source text of its own.
+- Improved {doc}`performance-inefficient-vector-operation
+ <clang-tidy/checks/performance/inefficient-vector-operation>` by adding the
+ {option}`ForRangeLoopClasses` to configure container classes that can be used
+ as sources in range-based `for` loops.
+
- Improved {doc}`readability-convert-member-functions-to-static
<clang-tidy/checks/readability/convert-member-functions-to-static>` check by
fixing a crash when checking a const-qualified method declared with the
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance/inefficient-vector-operation.rst b/clang-tools-extra/docs/clang-tidy/checks/performance/inefficient-vector-operation.rst
index 75016eb72d279..99a250c01a9f2 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance/inefficient-vector-operation.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance/inefficient-vector-operation.rst
@@ -34,9 +34,9 @@ statement body:
}
* For-range loops like ``for (range-declaration : range_expression)``, the type
- of ``range_expression`` can be ``std::vector``, ``std::array``,
- ``std::deque``, ``std::set``, ``std::unordered_set``, ``std::map``,
- ``std::unordered_set``:
+ of ``range_expression`` can be ``std::array``, ``std::deque``, ``std::map``,
+ ``std::set``, ``std::unordered_map``, ``std::unordered_set``, or
+ ``std::vector``:
.. code-block:: c++
@@ -59,6 +59,15 @@ Options
Semicolon-separated list of names of vector-like classes. By default only
``::std::vector`` is considered.
+.. option:: ForRangeLoopClasses
+
+ Semicolon-separated list of names of container classes that can be used as
+ sources in range-based ``for`` loops. Each configured class must provide an
+ accessible ``size()`` method whose result is the number of elements visited
+ by the loop. Defaults to
+ ``::std::array``, ``::std::deque``, ``::std::map``, ``::std::set``,
+ ``::std::unordered_map``, ``::std::unordered_set``, and ``::std::vector``.
+
.. option:: EnableProto
When `true`, the check will also warn on inefficient operations for proto
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-vector-operation-vectorlike-classes.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-vector-operation-vectorlike-classes.cpp
index 41d2d3733e07a..75d2a760ef371 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-vector-operation-vectorlike-classes.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance/inefficient-vector-operation-vectorlike-classes.cpp
@@ -1,7 +1,9 @@
// RUN: %check_clang_tidy %s performance-inefficient-vector-operation %t -- \
// RUN: -config='{CheckOptions: \
// RUN: {performance-inefficient-vector-operation.VectorLikeClasses: \
-// RUN: "VectorLikeInheritedPushBack;VectorLikeDirectPushBack;VectorLikeInheritedEmplaceBack"}}'
+// RUN: "VectorLikeInheritedPushBack;VectorLikeDirectPushBack;VectorLikeInheritedEmplaceBack", \
+// RUN: performance-inefficient-vector-operation.ForRangeLoopClasses: \
+// RUN: "RangeLike"}}'
class VectorLikePushBackBase {
public:
@@ -29,6 +31,25 @@ class VectorLikeInheritedEmplaceBack : public VectorLikeEmplaceBackBase {
void reserve(int);
};
+class RangeLike {
+public:
+ int *begin();
+ int *end();
+ int size() const;
+};
+
+class UnconfiguredRangeLike {
+public:
+ int *begin();
+ int *end();
+ int size() const;
+};
+
+class RangeHolder {
+public:
+ RangeLike range;
+};
+
void testVectorLikeClasses() {
{
VectorLikeInheritedPushBack inheritedPushBackVector;
@@ -56,4 +77,34 @@ void testVectorLikeClasses() {
// CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'emplace_back' is called inside a loop; consider pre-allocating the container capacity before the loop
}
}
+
+ {
+ RangeLike range;
+ VectorLikeDirectPushBack vector;
+ // CHECK-FIXES: vector.reserve(range.size());
+ for (int value : range) {
+ vector.push_back(value);
+ // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'push_back' is called inside a loop; consider pre-allocating the container capacity before the loop
+ }
+ }
+
+ {
+ UnconfiguredRangeLike range;
+ VectorLikeDirectPushBack vector;
+ // CHECK-FIXES-NOT: vector.reserve(range.size());
+ for (int value : range) {
+ vector.push_back(value);
+ // CHECK-MESSAGES-NOT: :[[@LINE-1]]:7: warning: 'push_back' is called inside a loop
+ }
+ }
+
+ {
+ RangeHolder holder;
+ VectorLikeDirectPushBack vector;
+ // CHECK-FIXES: vector.reserve(holder.range.size());
+ for (int value : holder.range) {
+ vector.push_back(value);
+ // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'push_back' is called inside a loop; consider pre-allocating the container capacity before the loop
+ }
+ }
}
>From f4b6e8cb75a06814cb8647c8177a4257f772c3cd Mon Sep 17 00:00:00 2001
From: Alexander Shaposhnikov <ashaposhnikov at google.com>
Date: Wed, 7 Oct 2026 01:20:04 -0700
Subject: [PATCH 11/52] [SPIRV] Re-sort basic blocks after LoopSimplify for
non-shader targets (#229678)
SPIRVPrepareFunctions sorts basic blocks in reverse post-order so that
dominators appear before all blocks
they dominate, as required by SPIR-V.
However, since #187519 (4a773b9f35fc), LoopSimplifyPass runs later in
addISelPrepare for non-shader
targets as well. For shader targets, SPIRVStructurizer subsequently
re-runs sortBlocks(F),
but for non-shader targets no pass restored the block order, allowing
dedicated exit or preheader blocks
inserted by LoopSimplify to appear before their dominators.
Call sortBlocks(Func) for non-shader targets at the start of
SPIRVEmitIntrinsicsImpl::runOnFunction.
---
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 5 ++++
llvm/test/CodeGen/SPIRV/dominator-order.ll | 28 +++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index e22a1a0948c2e..27675e8e6e75f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -3751,6 +3751,11 @@ bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
return false;
const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(Func);
+ // LoopSimplify runs after SPIRVPrepareFunctions sorted the blocks, and the
+ // preheaders/dedicated exits it creates can end up before their dominator,
+ // which SPIR-V forbids.
+ if (!ST.isShader())
+ sortBlocks(Func);
GR = ST.getSPIRVGlobalRegistry();
if (!CurrF)
diff --git a/llvm/test/CodeGen/SPIRV/dominator-order.ll b/llvm/test/CodeGen/SPIRV/dominator-order.ll
index 2ecdddcd71ff0..235b40a859756 100644
--- a/llvm/test/CodeGen/SPIRV/dominator-order.ll
+++ b/llvm/test/CodeGen/SPIRV/dominator-order.ll
@@ -8,6 +8,10 @@
; CHECK-DAG: OpName %[[#ENTRY:]] "entry"
; CHECK-DAG: OpName %[[#FOR_BODY137_LR_PH:]] "for.body137.lr.ph"
; CHECK-DAG: OpName %[[#FOR_BODY:]] "for.body"
+; CHECK-DAG: OpName %[[#OUTER:]] "outer"
+; CHECK-DAG: OpName %[[#INNER_PREHEADER:]] "inner.preheader"
+; CHECK-DAG: OpName %[[#INNER:]] "inner"
+; CHECK-DAG: OpName %[[#OUTER_LOOPEXIT:]] "outer.loopexit"
; CHECK: %[[#ENTRY]] = OpLabel
; CHECK: %[[#FOR_BODY]] = OpLabel
@@ -23,3 +27,27 @@ for.body137.lr.ph: ; preds = %for.body
for.body: ; preds = %for.body, %entry
br i1 %cond, label %for.body, label %for.body137.lr.ph
}
+
+; Check that blocks inserted by LoopSimplify (such as outer.loopexit, which is
+; inserted before outer but dominated by inner) are re-sorted after their
+; dominators.
+; CHECK: OpFunction
+; CHECK: %[[#OUTER]] = OpLabel
+; CHECK: %[[#INNER_PREHEADER]] = OpLabel
+; CHECK: %[[#INNER]] = OpLabel
+; CHECK: %[[#OUTER_LOOPEXIT]] = OpLabel
+; CHECK: OpFunctionEnd
+
+define spir_kernel void @test_loop_simplify(i1 %c1, i1 %c2) {
+entry:
+ br label %outer
+
+outer:
+ br i1 %c1, label %inner, label %exit
+
+inner:
+ br i1 %c2, label %inner, label %outer
+
+exit:
+ ret void
+}
>From ecc201f1d0d5372d0a3a77a516c0530e880170fb Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Wed, 7 Oct 2026 10:20:33 +0200
Subject: [PATCH 12/52] [IRBuilder] Accept Module instead of LLVMContext in
ctor (#229430)
This adds new constructor overloads, which accept `Module &` instead of
`LLVMContext &` in the ctor, and replaces nearly all usages of the old
constructors. A few more complicated cases (C API, SandboxIR context and
one unit test) are left alone for now.
The intent is to remove the old LLVMContext constructors and make
availability of the Module required at time of construction (either
directly passed to the ctor or implied by the insertion point).
The motivation for this is to ensure that IRBuilder always has an
available DataLayout. IRBuilder used to be DL-agnostic, but at this
point we've accumulated a number of key places which require a data
layout, and get it from the insertion point instead. This is not great,
because the insertion point is not actually required to be set when
producing instructions (they just won't be inserted).
Additionally, not having a required DataLayout also introduces the
ConstantFolder (default) vs TargetFolder dichotomy, and is also a big
part of the reason why we still have separate ConstantFold and
ConstantFolding APIs that differ in DL-awareness.
Rather than accepting LLVMContext + DataLayout, this PR proposes to
accept Module instead. This is both a matter of convenience, but also
some future-proofing: For example, if we decide that constant
expressions should be owned by the Module instead of LLVMContext (which
they should be...) it will be easier to make such a change in the
future. Additionally, we have API's like
`IRBuilder::CreateGlobalString()`, which currently accept an explicit
Module, because IRBuilder itself is not module-aware.
---
clang/lib/CodeGen/CGBuilder.h | 6 +-
clang/lib/CodeGen/CGHLSLRuntime.cpp | 8 +-
.../BuildingAJIT/Chapter1/toy.cpp | 2 +-
.../BuildingAJIT/Chapter2/toy.cpp | 2 +-
.../BuildingAJIT/Chapter3/toy.cpp | 2 +-
.../BuildingAJIT/Chapter4/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter3/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter4/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter5/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter6/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter7/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter8/toy.cpp | 2 +-
llvm/examples/Kaleidoscope/Chapter9/toy.cpp | 2 +-
llvm/include/llvm/Analysis/MemoryBuiltins.h | 4 +-
llvm/include/llvm/Analysis/ScalarEvolution.h | 2 +
.../Frontend/HLSL/RootSignatureMetadata.h | 6 +-
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 4 +-
llvm/include/llvm/IR/IRBuilder.h | 18 +
.../llvm/Transforms/IPO/InstrumentorUtils.h | 2 +-
.../Transforms/InstCombine/InstCombiner.h | 3 +-
.../llvm/Transforms/Utils/EscapeEnumerator.h | 2 +-
.../Utils/ScalarEvolutionExpander.h | 2 +-
llvm/lib/Analysis/MemoryBuiltins.cpp | 10 +-
llvm/lib/CodeGen/InlineAsmPrepare.cpp | 2 +-
llvm/lib/CodeGen/InterleavedAccessPass.cpp | 2 +-
llvm/lib/CodeGen/SanitizerBinaryMetadata.cpp | 2 +-
llvm/lib/CodeGen/TypePromotion.cpp | 18 +-
llvm/lib/CodeGen/WasmEHPrepare.cpp | 4 +-
.../ExecutionEngine/Orc/ReOptimizeLayer.cpp | 2 +-
llvm/lib/ExecutionEngine/Orc/Speculation.cpp | 2 +-
.../Frontend/HLSL/RootSignatureMetadata.cpp | 12 +-
llvm/lib/IR/AutoUpgrade.cpp | 3 +-
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 2 +-
.../AMDGPU/AMDGPULowerBufferFatPointers.cpp | 25 +-
.../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 9 +-
.../AMDGPU/AMDGPUPrintfRuntimeBinding.cpp | 2 +-
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 2 +-
llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp | 2 +-
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 2 +-
.../Target/DirectX/DXILDataScalarization.cpp | 2 +-
llvm/lib/Target/DirectX/DXILOpBuilder.cpp | 2 +-
.../Target/DirectX/DXILTranslateMetadata.cpp | 9 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 4 +-
.../Hexagon/HexagonLoopIdiomRecognition.cpp | 345 +++++++++---------
llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp | 2 +-
llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 9 +-
.../Target/SPIRV/SPIRVLegalizePointerCast.cpp | 2 +-
.../Target/SPIRV/SPIRVPrepareFunctions.cpp | 4 +-
.../WebAssemblyLowerEmscriptenEHSjLj.cpp | 21 +-
.../AggressiveInstCombine.cpp | 2 +-
.../lib/Transforms/Coroutines/CoroCleanup.cpp | 6 +-
llvm/lib/Transforms/Coroutines/CoroCloner.h | 4 +-
llvm/lib/Transforms/Coroutines/CoroEarly.cpp | 2 +-
llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 5 +-
llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 2 +-
.../Transforms/IPO/AttributorAttributes.cpp | 3 +-
llvm/lib/Transforms/IPO/ExpandVariadics.cpp | 3 +-
llvm/lib/Transforms/IPO/GlobalOpt.cpp | 2 +-
.../InstCombine/InstCombineCompares.cpp | 2 +-
.../InstCombine/InstCombineInternal.h | 4 +-
.../InstCombine/InstCombineNegator.cpp | 10 +-
.../Instrumentation/AddressSanitizer.cpp | 10 +-
.../Instrumentation/BoundsChecking.cpp | 2 +-
.../Transforms/Instrumentation/CopyProf.cpp | 4 +-
.../Instrumentation/HWAddressSanitizer.cpp | 4 +-
.../MemProfInstrumentation.cpp | 2 +-
.../Instrumentation/MemorySanitizer.cpp | 8 +-
.../SanitizerBinaryMetadata.cpp | 2 +-
.../Instrumentation/SanitizerCoverage.cpp | 2 +-
.../Instrumentation/ThreadSanitizer.cpp | 2 +-
.../Instrumentation/TypeSanitizer.cpp | 2 +-
llvm/lib/Transforms/Scalar/SROA.cpp | 2 +-
llvm/lib/Transforms/Utils/InlineFunction.cpp | 2 +-
llvm/lib/Transforms/Utils/LoopVersioning.cpp | 2 +-
llvm/lib/Transforms/Utils/ModuleUtils.cpp | 6 +-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 2 +-
.../Vectorize/LoadStoreVectorizer.cpp | 4 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
.../Transforms/Vectorize/VectorCombine.cpp | 2 +-
llvm/tools/lli/lli.cpp | 4 +-
.../Analysis/BasicAliasAnalysisTest.cpp | 2 +-
llvm/unittests/Analysis/MemorySSATest.cpp | 12 +-
llvm/unittests/Analysis/SparsePropagation.cpp | 3 +-
llvm/unittests/Analysis/ValueTrackingTest.cpp | 2 +-
.../Orc/ReOptimizeLayerTest.cpp | 2 +-
llvm/unittests/IR/IRBuilderTest.cpp | 6 +-
llvm/unittests/IR/InstructionsTest.cpp | 14 +-
.../Transforms/Utils/CloningTest.cpp | 4 +-
.../Utils/FunctionComparatorTest.cpp | 2 +-
.../Transforms/Utils/IntegerDivisionTest.cpp | 16 +-
llvm/unittests/Transforms/Utils/LocalTest.cpp | 10 +-
.../Transforms/Utils/SSAUpdaterBulkTest.cpp | 4 +-
mlir/lib/ExecutionEngine/ExecutionEngine.cpp | 2 +-
.../LLVMIR/Dialect/GPU/SelectObjectAttr.cpp | 2 +-
mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 13 +-
polly/lib/CodeGen/PerfMonitor.cpp | 3 +-
97 files changed, 401 insertions(+), 387 deletions(-)
diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h
index 18e738f10ba3b..2dc70a9071e96 100644
--- a/clang/lib/CodeGen/CGBuilder.h
+++ b/clang/lib/CodeGen/CGBuilder.h
@@ -92,11 +92,13 @@ class CGBuilderTy : public CGBuilderBaseTy {
public:
CGBuilderTy(const CodeGenModule &CGM, llvm::LLVMContext &C)
- : CGBuilderBaseTy(C, llvm::TargetFolder(CGM.getDataLayout())),
+ : CGBuilderBaseTy(CGM.getModule(),
+ llvm::TargetFolder(CGM.getDataLayout())),
TypeCache(CGM) {}
CGBuilderTy(const CodeGenModule &CGM, llvm::LLVMContext &C,
const CGBuilderInserterTy &Inserter)
- : CGBuilderBaseTy(C, llvm::TargetFolder(CGM.getDataLayout()), Inserter),
+ : CGBuilderBaseTy(CGM.getModule(),
+ llvm::TargetFolder(CGM.getDataLayout()), Inserter),
TypeCache(CGM) {}
CGBuilderTy(const CodeGenModule &CGM, llvm::Instruction *I)
: CGBuilderBaseTy(I->getIterator(),
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 52fde18dc3ebd..b69fbc907293a 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -78,7 +78,7 @@ void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
uint64_t Minor = *Version.getMinor();
auto &Ctx = M.getContext();
- IRBuilder<> B(M.getContext());
+ IRBuilder<> B(M);
MDNode *Val = MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(Major)),
ConstantAsMetadata::get(B.getInt32(Minor))});
StringRef DXILValKey = "dx.valver";
@@ -91,7 +91,7 @@ void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
llvm::Function *Fn, llvm::Module &M) {
auto &Ctx = M.getContext();
- llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);
+ llvm::hlsl::rootsig::MetadataBuilder RSBuilder(M, Elements);
MDNode *RootSignature = RSBuilder.BuildRootSignature();
ConstantAsMetadata *Version = ConstantAsMetadata::get(ConstantInt::get(
@@ -1075,7 +1075,7 @@ static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
unsigned BuiltIn) {
LLVMContext &Ctx = GV->getContext();
- IRBuilder<> B(GV->getContext());
+ IRBuilder<> B(*GV->getParent());
MDNode *Operands = MDNode::get(
Ctx,
{ConstantAsMetadata::get(B.getInt32(/* Spirv::Decoration::BuiltIn */ 11)),
@@ -1086,7 +1086,7 @@ static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,
static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {
LLVMContext &Ctx = GV->getContext();
- IRBuilder<> B(GV->getContext());
+ IRBuilder<> B(*GV->getParent());
MDNode *Operands =
MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(/* Location */ 30)),
ConstantAsMetadata::get(B.getInt32(Location))});
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp
index ed0770bb3d5a5..92e64b34c2150 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp
@@ -1108,7 +1108,7 @@ static void InitializeModule() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
static void HandleDefinition() {
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp
index ed0770bb3d5a5..92e64b34c2150 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp
@@ -1108,7 +1108,7 @@ static void InitializeModule() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
static void HandleDefinition() {
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp
index ed0770bb3d5a5..92e64b34c2150 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp
@@ -1108,7 +1108,7 @@ static void InitializeModule() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
static void HandleDefinition() {
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp
index 1a7f1475330a8..a4cfc4b9d5b7b 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp
@@ -1099,7 +1099,7 @@ static void InitializeModule() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
ThreadSafeModule irgenAndTakeOwnership(FunctionAST &FnAST,
diff --git a/llvm/examples/Kaleidoscope/Chapter3/toy.cpp b/llvm/examples/Kaleidoscope/Chapter3/toy.cpp
index 66973ee3ebcfa..c23f21e3bc0c0 100644
--- a/llvm/examples/Kaleidoscope/Chapter3/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter3/toy.cpp
@@ -527,7 +527,7 @@ static void InitializeModule() {
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
static void HandleDefinition() {
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index 0f58391c50667..b73dea67511a3 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -566,7 +566,7 @@ static void InitializeModuleAndManagers() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
// Create new pass and analysis managers.
TheFPM = std::make_unique<FunctionPassManager>();
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index 7117eaf4982b0..c33e4da68d875 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -840,7 +840,7 @@ static void InitializeModuleAndManagers() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
// Create new pass and analysis managers.
TheFPM = std::make_unique<FunctionPassManager>();
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index cb7b6cc8651c1..18cba9447cf04 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -959,7 +959,7 @@ static void InitializeModuleAndManagers() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
// Create new pass and analysis managers.
TheFPM = std::make_unique<FunctionPassManager>();
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 21c504a7357b8..6a8c4840bf4f5 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1127,7 +1127,7 @@ static void InitializeModuleAndManagers() {
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
// Create new pass and analysis managers.
TheFPM = std::make_unique<FunctionPassManager>();
diff --git a/llvm/examples/Kaleidoscope/Chapter8/toy.cpp b/llvm/examples/Kaleidoscope/Chapter8/toy.cpp
index 2634c1891c8fc..d8fb8e8cd1d59 100644
--- a/llvm/examples/Kaleidoscope/Chapter8/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter8/toy.cpp
@@ -1111,7 +1111,7 @@ static void InitializeModuleAndPassManager() {
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
// Create a new builder for the module.
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
static void HandleDefinition() {
diff --git a/llvm/examples/Kaleidoscope/Chapter9/toy.cpp b/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
index d25c776401b99..1bd7f8bc0ae49 100644
--- a/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
@@ -1325,7 +1325,7 @@ static void InitializeModule() {
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
TheModule->setDataLayout(TheJIT->getDataLayout());
- Builder = std::make_unique<IRBuilder<>>(*TheContext);
+ Builder = std::make_unique<IRBuilder<>>(*TheModule);
}
static void HandleDefinition() {
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index ebe644cee53e1..065dc00da2439 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -354,9 +354,7 @@ class ObjectSizeOffsetEvaluator
SizeOffsetValue compute_(Value *V);
public:
- LLVM_ABI ObjectSizeOffsetEvaluator(const DataLayout &DL,
- const TargetLibraryInfo *TLI,
- LLVMContext &Context,
+ LLVM_ABI ObjectSizeOffsetEvaluator(Module &M, const TargetLibraryInfo *TLI,
ObjectSizeOpts EvalOpts = {});
static SizeOffsetValue unknown() { return SizeOffsetValue(); }
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index 79944d6870158..9ef4d0b0542f3 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -665,6 +665,8 @@ class ScalarEvolution {
LLVMContext &getContext() const { return F.getContext(); }
+ Module &getModule() const { return *F.getParent(); }
+
/// Test if values of the given type are analyzable within the SCEV
/// framework. This primarily includes integer types, and it can optionally
/// include pointer types if the ScalarEvolution class has access to
diff --git a/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
index 509c0b816f79e..b3b5fd5a3a0d1 100644
--- a/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
+++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
@@ -17,6 +17,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/Module.h"
#include "llvm/MC/DXContainerRootSignature.h"
#include "llvm/Support/Compiler.h"
@@ -44,8 +45,8 @@ class RootSignatureValidationError
class MetadataBuilder {
public:
- MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
- : Ctx(Ctx), Elements(Elements) {}
+ MetadataBuilder(llvm::Module &M, ArrayRef<RootElement> Elements)
+ : M(M), Ctx(M.getContext()), Elements(Elements) {}
/// Iterates through elements and dispatches onto the correct Build* method
///
@@ -62,6 +63,7 @@ class MetadataBuilder {
MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
MDNode *BuildStaticSampler(const StaticSampler &Sampler);
+ llvm::Module &M;
llvm::LLVMContext &Ctx;
ArrayRef<RootElement> Elements;
SmallVector<Metadata *> GeneratedMetadata;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 5acbb69248309..4cd5b88c64540 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -513,8 +513,8 @@ class OpenMPIRBuilder {
/// Create a new OpenMPIRBuilder operating on the given module \p M. This will
/// not have an effect on \p M (see initialize)
OpenMPIRBuilder(Module &M)
- : M(M), Builder(M.getContext()), OffloadInfoManager(this),
- T(M.getTargetTriple()), IsFinalized(false) {}
+ : M(M), Builder(M), OffloadInfoManager(this), T(M.getTargetTriple()),
+ IsFinalized(false) {}
LLVM_ABI ~OpenMPIRBuilder();
class AtomicInfo : public llvm::AtomicInfo {
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index bbeefa8ef90c0..c370433331d5a 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -34,6 +34,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
@@ -2910,6 +2911,7 @@ class IRBuilder : public IRBuilderBase {
InserterTy Inserter;
public:
+ // TODO: Deprecate ctors accepting LLVMContext.
IRBuilder(LLVMContext &C, FolderTy Folder, InserterTy Inserter)
: IRBuilderBase(C, this->Folder, this->Inserter), Folder(Folder),
Inserter(Inserter) {}
@@ -2920,6 +2922,17 @@ class IRBuilder : public IRBuilderBase {
explicit IRBuilder(LLVMContext &C)
: IRBuilderBase(C, this->Folder, this->Inserter) {}
+ IRBuilder(Module &M, FolderTy Folder, InserterTy Inserter)
+ : IRBuilderBase(M.getContext(), this->Folder, this->Inserter),
+ Folder(Folder), Inserter(Inserter) {}
+
+ IRBuilder(Module &M, FolderTy Folder)
+ : IRBuilderBase(M.getContext(), this->Folder, this->Inserter),
+ Folder(Folder) {}
+
+ explicit IRBuilder(Module &M)
+ : IRBuilderBase(M.getContext(), this->Folder, this->Inserter) {}
+
explicit IRBuilder(BasicBlock *TheBB, FolderTy Folder)
: IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter),
Folder(Folder) {
@@ -2981,6 +2994,11 @@ template <typename FolderTy, typename InserterTy>
IRBuilder(LLVMContext &, FolderTy, InserterTy)
-> IRBuilder<FolderTy, InserterTy>;
IRBuilder(LLVMContext &) -> IRBuilder<>;
+template <typename FolderTy, typename InserterTy>
+IRBuilder(Module &, FolderTy, InserterTy) -> IRBuilder<FolderTy, InserterTy>;
+template <typename FolderTy>
+IRBuilder(Module &, FolderTy) -> IRBuilder<FolderTy>;
+IRBuilder(Module &) -> IRBuilder<>;
template <typename FolderTy>
IRBuilder(BasicBlock *, FolderTy) -> IRBuilder<FolderTy>;
IRBuilder(BasicBlock *) -> IRBuilder<>;
diff --git a/llvm/include/llvm/Transforms/IPO/InstrumentorUtils.h b/llvm/include/llvm/Transforms/IPO/InstrumentorUtils.h
index 44c6c87057850..c8b66e7719fb5 100644
--- a/llvm/include/llvm/Transforms/IPO/InstrumentorUtils.h
+++ b/llvm/include/llvm/Transforms/IPO/InstrumentorUtils.h
@@ -38,7 +38,7 @@ struct InstrumentorIRBuilderTy {
/// Construct an IR builder for the module \p M.
InstrumentorIRBuilderTy(Module &M)
: M(M), Ctx(M.getContext()),
- IRB(Ctx, ConstantFolder(),
+ IRB(M, ConstantFolder(),
// Save the inserted instructions in a structure.
IRBuilderCallbackInserter(
[&](Instruction *I) { NewInsts[I] = Epoch; })) {}
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
index ce9d42c58e3f2..42924a0648efc 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -123,7 +123,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
const DataLayout &DL,
ReversePostOrderTraversal<BasicBlock *> &RPOT)
: TTIForTargetIntrinsicsOnly(TTI),
- Builder(F.getContext(), TargetFolder(DL),
+ Builder(*F.getParent(), TargetFolder(DL),
IRBuilderInstCombineInserter(*this)),
Worklist(Worklist), F(F), MinimizeSize(F.hasMinSize()), AA(AA), AC(AC),
TLI(TLI), DT(DT), DL(DL),
@@ -362,6 +362,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner {
void addToWorklist(Instruction *I) { Worklist.push(I); }
+ Module &getModule() const { return *F.getParent(); }
AssumptionCache &getAssumptionCache() const { return AC; }
TargetLibraryInfo &getTargetLibraryInfo() const { return TLI; }
DominatorTree &getDominatorTree() const { return DT; }
diff --git a/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h b/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h
index f3f7e9743fa7b..660dbd47f9ad8 100644
--- a/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h
+++ b/llvm/include/llvm/Transforms/Utils/EscapeEnumerator.h
@@ -41,7 +41,7 @@ class EscapeEnumerator {
EscapeEnumerator(Function &F, const char *N = "cleanup",
bool HandleExceptions = true, DomTreeUpdater *DTU = nullptr)
: F(F), CleanupBBName(N), StateBB(F.begin()), StateE(F.end()),
- Builder(F.getContext()), HandleExceptions(HandleExceptions), DTU(DTU) {}
+ Builder(*F.getParent()), HandleExceptions(HandleExceptions), DTU(DTU) {}
LLVM_ABI IRBuilder<> *Next();
};
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index e72235fbf6d42..181f9dfe4247f 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -196,7 +196,7 @@ class SCEVExpander : public SCEVUseVisitor<SCEVExpander, Value *> {
: SE(SE), DL(SE.getDataLayout()), IVName(Name),
PreserveLCSSA(PreserveLCSSA), IVIncInsertLoop(nullptr),
IVIncInsertPos(nullptr), CanonicalMode(true), LSRMode(false),
- Builder(SE.getContext(), InstSimplifyFolder(DL),
+ Builder(SE.getModule(), InstSimplifyFolder(DL),
IRBuilderCallbackInserter(
[this](Instruction *I) { rememberInstruction(I); })) {
#if LLVM_ENABLE_ABI_BREAKING_CHECKS
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 2a8b4e7d81910..c9eaa82802fe6 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -673,8 +673,7 @@ Value *llvm::lowerObjectSizeCall(
isUIntN(ResultType->getBitWidth(), Size))
return ConstantInt::get(ResultType, Size);
} else {
- LLVMContext &Ctx = ObjectSize->getFunction()->getContext();
- ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, EvalOptions);
+ ObjectSizeOffsetEvaluator Eval(*ObjectSize->getModule(), TLI, EvalOptions);
SizeOffsetValue SizeOffsetPair = Eval.compute(ObjectSize->getArgOperand(0));
if (SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown()) {
@@ -1231,10 +1230,9 @@ SizeOffsetValue::SizeOffsetValue(const SizeOffsetWeakTrackingVH &SOT)
: SizeOffsetType(SOT.Size, SOT.Offset) {}
ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator(
- const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context,
- ObjectSizeOpts EvalOpts)
- : DL(DL), TLI(TLI), Context(Context),
- Builder(Context, TargetFolder(DL),
+ Module &M, const TargetLibraryInfo *TLI, ObjectSizeOpts EvalOpts)
+ : DL(M.getDataLayout()), TLI(TLI), Context(M.getContext()),
+ Builder(M, TargetFolder(DL),
IRBuilderCallbackInserter(
[&](Instruction *I) { InsertedInstructions.insert(I); })),
EvalOpts(EvalOpts) {
diff --git a/llvm/lib/CodeGen/InlineAsmPrepare.cpp b/llvm/lib/CodeGen/InlineAsmPrepare.cpp
index 9587bf34649c1..ca58eb0e8d794 100644
--- a/llvm/lib/CodeGen/InlineAsmPrepare.cpp
+++ b/llvm/lib/CodeGen/InlineAsmPrepare.cpp
@@ -173,7 +173,7 @@ static bool splitCriticalEdges(CallBrInst *CBR, DominatorTree *DT) {
static bool insertIntrinsicCalls(CallBrInst *CBR, DominatorTree &DT) {
bool Changed = false;
SmallPtrSet<const BasicBlock *, 4> Visited;
- IRBuilder<> Builder(CBR->getContext());
+ IRBuilder<> Builder(*CBR->getModule());
if (!CBR->getNumIndirectDests())
return false;
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 5498ce1d81647..4ece88a0014bc 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -490,7 +490,7 @@ bool InterleavedAccessImpl::tryReplaceExtracts(
}
// Finally, perform the replacements.
- IRBuilder<> Builder(Extracts[0]->getContext());
+ IRBuilder<> Builder(*Extracts[0]->getModule());
for (auto &Replacement : ReplacementMap) {
auto *Extract = Replacement.first;
auto *Vector = Replacement.second.first;
diff --git a/llvm/lib/CodeGen/SanitizerBinaryMetadata.cpp b/llvm/lib/CodeGen/SanitizerBinaryMetadata.cpp
index af445af9403ef..da9b469249977 100644
--- a/llvm/lib/CodeGen/SanitizerBinaryMetadata.cpp
+++ b/llvm/lib/CodeGen/SanitizerBinaryMetadata.cpp
@@ -93,7 +93,7 @@ bool MachineSanitizerBinaryMetadata::run(MachineFunction &MF) {
return false;
// Non-zero size, update metadata.
auto &F = MF.getFunction();
- IRBuilder<> IRB(F.getContext());
+ IRBuilder<> IRB(*F.getParent());
MDBuilder MDB(F.getContext());
// Keep the features and append size of stack args to the metadata.
APInt NewFeatures = Features->getUniqueInteger();
diff --git a/llvm/lib/CodeGen/TypePromotion.cpp b/llvm/lib/CodeGen/TypePromotion.cpp
index 2736ff3f8a299..77ba91e55ba62 100644
--- a/llvm/lib/CodeGen/TypePromotion.cpp
+++ b/llvm/lib/CodeGen/TypePromotion.cpp
@@ -102,6 +102,7 @@ static cl::opt<bool> DisablePromotion("disable-type-promotion", cl::Hidden,
namespace {
class IRPromoter {
+ Module &M;
LLVMContext &Ctx;
unsigned PromotedWidth = 0;
SetVector<Value *> &Visited;
@@ -122,12 +123,13 @@ class IRPromoter {
void Cleanup();
public:
- IRPromoter(LLVMContext &C, unsigned Width, SetVector<Value *> &visited,
+ IRPromoter(Module &M, unsigned Width, SetVector<Value *> &visited,
SetVector<Value *> &sources, SetVector<Instruction *> &sinks,
SmallPtrSetImpl<Instruction *> &wrap,
SmallPtrSetImpl<Instruction *> &instsToRemove)
- : Ctx(C), PromotedWidth(Width), Visited(visited), Sources(sources),
- Sinks(sinks), SafeWrap(wrap), InstsToRemove(instsToRemove) {
+ : M(M), Ctx(M.getContext()), PromotedWidth(Width), Visited(visited),
+ Sources(sources), Sinks(sinks), SafeWrap(wrap),
+ InstsToRemove(instsToRemove) {
ExtTy = IntegerType::get(Ctx, PromotedWidth);
}
@@ -137,6 +139,7 @@ class IRPromoter {
class TypePromotionImpl {
unsigned TypeSize = 0;
const TargetLowering *TLI = nullptr;
+ Module *M = nullptr;
LLVMContext *Ctx = nullptr;
unsigned RegisterBitWidth = 0;
SmallPtrSet<Value *, 16> AllVisited;
@@ -444,7 +447,7 @@ void IRPromoter::ReplaceAllUsersOfWith(Value *From, Value *To) {
}
void IRPromoter::ExtendSources() {
- IRBuilder<> Builder{Ctx};
+ IRBuilder<> Builder(M);
auto InsertZExt = [&](Value *V, BasicBlock::iterator InsertPt) {
assert(V->getType() != ExtTy && "zext already extends to i32");
@@ -555,7 +558,7 @@ void IRPromoter::PromoteTree() {
void IRPromoter::TruncateSinks() {
LLVM_DEBUG(dbgs() << "IR Promotion: Fixing up the sinks:\n");
- IRBuilder<> Builder{Ctx};
+ IRBuilder<> Builder(M);
auto InsertTrunc = [&](Value *V, Type *TruncTy) -> Instruction * {
if (!isa<Instruction>(V) || !isa<IntegerType>(V->getType()))
@@ -660,7 +663,7 @@ void IRPromoter::Cleanup() {
void IRPromoter::ConvertTruncs() {
LLVM_DEBUG(dbgs() << "IR Promotion: Converting truncs..\n");
- IRBuilder<> Builder{Ctx};
+ IRBuilder<> Builder(M);
for (auto *V : Visited) {
if (!isa<TruncInst>(V) || isTruncToI1(V) || Sources.count(V))
@@ -941,7 +944,7 @@ bool TypePromotionImpl::TryToPromote(Value *V, unsigned PromotedWidth,
(ToPromote < 2 || (Blocks.size() == 1 && NonFreeArgs > SafeWrap.size())))
return false;
- IRPromoter Promoter(*Ctx, PromotedWidth, CurrentVisited, Sources, Sinks,
+ IRPromoter Promoter(*M, PromotedWidth, CurrentVisited, Sources, Sinks,
SafeWrap, InstsToRemove);
Promoter.Mutate();
return true;
@@ -964,6 +967,7 @@ bool TypePromotionImpl::run(Function &F, const TargetMachine *TM,
TLI = SubtargetInfo->getTargetLowering();
RegisterBitWidth =
TTI.getRegisterBitWidth(TargetTransformInfo::RGK_Scalar).getFixedValue();
+ M = F.getParent();
Ctx = &F.getContext();
// Return the preferred integer width of the instruction, or zero if we
diff --git a/llvm/lib/CodeGen/WasmEHPrepare.cpp b/llvm/lib/CodeGen/WasmEHPrepare.cpp
index 76cdac4609e38..9356287b33983 100644
--- a/llvm/lib/CodeGen/WasmEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WasmEHPrepare.cpp
@@ -145,7 +145,7 @@ INITIALIZE_PASS_END(WasmEHPrepare, DEBUG_TYPE, "Prepare WebAssembly exceptions",
FunctionPass *llvm::createWasmEHPass() { return new WasmEHPrepare(); }
bool WasmEHPrepare::doInitialization(Module &M) {
- IRBuilder<> IRB(M.getContext());
+ IRBuilder<> IRB(M);
P.LPadContextTy = StructType::get(IRB.getInt32Ty(), // lpad_index
IRB.getPtrTy(), // lsda
IRB.getInt32Ty() // selector
@@ -176,7 +176,7 @@ bool WasmEHPrepareImpl::runOnFunction(Function &F) {
bool WasmEHPrepareImpl::prepareThrows(Function &F) {
Module &M = *F.getParent();
- IRBuilder<> IRB(F.getContext());
+ IRBuilder<> IRB(M);
bool Changed = false;
// wasm.throw() intinsic, which will be lowered to wasm 'throw' instruction.
diff --git a/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp
index cb73ee1a61e33..92e87f58c1a1c 100644
--- a/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ReOptimizeLayer.cpp
@@ -59,7 +59,7 @@ Error ReOptimizeLayer::addOrcRTLiteSupport(JITDylib &PlatformJD,
auto Mod = std::make_unique<Module>("orc-rt-lite-reoptimize.ll", *Ctx);
Mod->setDataLayout(DL);
- IRBuilder<> Builder(*Ctx);
+ IRBuilder<> Builder(*Mod);
// Create basic types portably
Type *VoidTy = Type::getVoidTy(*Ctx);
diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
index fee94b96a9e8a..b22998afef810 100644
--- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
@@ -75,7 +75,7 @@ void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
M, SpeculatorVTy, false, GlobalValue::LinkageTypes::ExternalLinkage,
nullptr, "__orc_speculator");
- IRBuilder<> Mutator(MContext);
+ IRBuilder<> Mutator(M);
// QueryAnalysis allowed to transform the IR source, one such example is
// Simplify CFG helps the static branch prediction heuristics!
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
index 014601c08447b..108b57c34af16 100644
--- a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
+++ b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
@@ -138,7 +138,7 @@ MDNode *MetadataBuilder::BuildRootSignature() {
}
MDNode *MetadataBuilder::BuildRootFlags(const dxbc::RootFlags &Flags) {
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
Metadata *Operands[] = {
MDString::get(Ctx, "RootFlags"),
ConstantAsMetadata::get(Builder.getInt32(to_underlying(Flags))),
@@ -147,7 +147,7 @@ MDNode *MetadataBuilder::BuildRootFlags(const dxbc::RootFlags &Flags) {
}
MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
Metadata *Operands[] = {
MDString::get(Ctx, "RootConstants"),
ConstantAsMetadata::get(
@@ -160,7 +160,7 @@ MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
}
MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
StringRef ResName = dxil::getResourceClassName(Descriptor.Type);
assert(!ResName.empty() && "Provided an invalid Resource Class");
SmallString<7> Name({"Root", ResName});
@@ -177,7 +177,7 @@ MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
}
MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
SmallVector<Metadata *> TableOperands;
// Set the mandatory arguments
TableOperands.push_back(MDString::get(Ctx, "DescriptorTable"));
@@ -200,7 +200,7 @@ MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
MDNode *MetadataBuilder::BuildDescriptorTableClause(
const DescriptorTableClause &Clause) {
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
StringRef ResName = dxil::getResourceClassName(Clause.Type);
assert(!ResName.empty() && "Provided an invalid Resource Class");
Metadata *Operands[] = {
@@ -215,7 +215,7 @@ MDNode *MetadataBuilder::BuildDescriptorTableClause(
}
MDNode *MetadataBuilder::BuildStaticSampler(const StaticSampler &Sampler) {
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
Metadata *Operands[] = {
MDString::get(Ctx, "StaticSampler"),
ConstantAsMetadata::get(Builder.getInt32(to_underlying(Sampler.Filter))),
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 2765934ee02d9..0e0c6cf10eeac 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -2499,8 +2499,7 @@ GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
if (!STy || STy->getNumElements() != 2)
return nullptr;
- LLVMContext &C = GV->getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(*GV->getParent());
auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
IRB.getPtrTy());
Constant *Init = GV->getInitializer();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 6b5976bab177a..cbbe7bcde0be5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -340,7 +340,7 @@ simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST,
II.mutateType(HalfVecTy);
II.setCalledFunction(HalfDecl);
- IRBuilder<> Builder(II.getContext());
+ IRBuilder<> Builder(*M);
for (auto &[Ext, Tr] : ExtractTruncPairs) {
Value *Idx = Ext->getIndexOperand();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index a00f3372c1f60..aa0f896b8c107 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -453,9 +453,9 @@ class StoreFatPtrsAsIntsAndExpandMemcpyVisitor
public:
StoreFatPtrsAsIntsAndExpandMemcpyVisitor(BufferFatPtrToIntTypeMap *TypeMap,
- const DataLayout &DL,
- LLVMContext &Ctx)
- : TypeMap(TypeMap), IRB(Ctx, InstSimplifyFolder(DL)), DL(DL) {}
+ Module &M)
+ : TypeMap(TypeMap), IRB(M, InstSimplifyFolder(M.getDataLayout())),
+ DL(M.getDataLayout()) {}
bool processFunction(Function &F, const TargetTransformInfo *TTI,
ScalarEvolution *SE);
@@ -832,9 +832,9 @@ class LegalizeBufferContentTypesVisitor
bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASCI);
public:
- LegalizeBufferContentTypesVisitor(const DataLayout &DL, LLVMContext &Ctx,
- const TargetMachine *TM)
- : IRB(Ctx, InstSimplifyFolder(DL)), DL(DL), TM(TM) {}
+ LegalizeBufferContentTypesVisitor(Module &M, const TargetMachine *TM)
+ : IRB(M, InstSimplifyFolder(M.getDataLayout())), DL(M.getDataLayout()),
+ TM(TM) {}
bool processFunction(Function &F, ScalarEvolution *SE);
};
} // namespace
@@ -1603,9 +1603,8 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
bool IsVolatile, SyncScope::ID SSID);
public:
- SplitPtrStructs(const DataLayout &DL, LLVMContext &Ctx,
- const TargetMachine *TM)
- : TM(TM), IRB(Ctx, InstSimplifyFolder(DL)) {}
+ SplitPtrStructs(Module &M, const TargetMachine *TM)
+ : TM(TM), IRB(M, InstSimplifyFolder(M.getDataLayout())) {}
void processFunction(Function &F);
@@ -2789,10 +2788,8 @@ bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM,
/*RemoveDeadConstants=*/false, /*IncludeSelf=*/true);
}
- StoreFatPtrsAsIntsAndExpandMemcpyVisitor MemOpsRewrite(&IntTM, DL,
- M.getContext());
- LegalizeBufferContentTypesVisitor BufferContentsTypeRewrite(
- DL, M.getContext(), &TM);
+ StoreFatPtrsAsIntsAndExpandMemcpyVisitor MemOpsRewrite(&IntTM, M);
+ LegalizeBufferContentTypesVisitor BufferContentsTypeRewrite(M, &TM);
for (Function &F : M.functions()) {
bool InterfaceChange = hasFatPointerInterface(F, &StructTM);
bool BodyChanges = containsBufferFatPointers(F, &StructTM);
@@ -2837,7 +2834,7 @@ bool AMDGPULowerBufferFatPointers::run(Module &M, const TargetMachine &TM,
IntTM.clear();
CloneMap.clear();
- SplitPtrStructs Splitter(DL, M.getContext(), &TM);
+ SplitPtrStructs Splitter(M, &TM);
for (Function *F : NeedsPostProcess)
Splitter.processFunction(*F);
for (Function *F : Intrinsics) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index fc6f2ac69c000..2313a955d52b9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -402,7 +402,7 @@ class AMDGPULowerModuleLDS {
GlobalVariable *LookupTable) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
Type *I32 = Type::getInt32Ty(Ctx);
for (size_t Index = 0; Index < ModuleScopeVariables.size(); Index++) {
@@ -556,7 +556,7 @@ class AMDGPULowerModuleLDS {
// Annotate the kernels with their order in this vector
LLVMContext &Ctx = M->getContext();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(*M);
if (OrderedKernels.size() > UINT32_MAX) {
// 32 bit keeps it in one SGPR. > 2**32 kernels won't fit on the GPU
@@ -855,7 +855,7 @@ class AMDGPULowerModuleLDS {
DenseMap<Function *, GlobalVariable *> KernelToCreatedDynamicLDS;
if (!KernelsThatIndirectlyAllocateDynamicLDS.empty()) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
Type *LocalPtrTy = PointerType::get(Ctx, AMDGPUAS::LOCAL_ADDRESS);
std::vector<Constant *> newDynamicLDS;
@@ -1138,8 +1138,7 @@ class AMDGPULowerModuleLDS {
KernelsThatIndirectlyAllocateDynamicLDS);
if (!KernelsThatAllocateTableLDS.empty()) {
- LLVMContext &Ctx = M.getContext();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
// The order must be consistent between lookup table and accesses to
// lookup table
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
index 98928e26f8de0..1c7f6761d4f30 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
@@ -137,7 +137,7 @@ static void diagnoseInvalidFormatString(const CallBase *CI) {
bool AMDGPUPrintfRuntimeBindingImpl::lowerPrintfForGpu(Module &M) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
Type *I32Ty = Type::getInt32Ty(Ctx);
// Instead of creating global variables, the printf format strings are
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 5498f3a20d8a7..0cac1374b2db9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -469,7 +469,7 @@ static bool isSupportedMemset(MemSetInst *I, AllocaInst *AI,
}
static Value *calculateVectorIndex(Value *Ptr, AllocaAnalysis &AA) {
- IRBuilder<> B(Ptr->getContext());
+ IRBuilder<> B(*AA.Alloca->getModule());
Ptr = Ptr->stripPointerCasts();
if (Ptr == AA.Alloca)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
index a32a4ea7e721a..9e2e7ffa23ebe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
@@ -168,7 +168,7 @@ struct FunctionsAndLDSAccess {
class AMDGPUSwLowerLDS {
public:
AMDGPUSwLowerLDS(Module &Mod, DomTreeCallback Callback)
- : M(Mod), IRB(M.getContext()), DTCallback(Callback) {}
+ : M(Mod), IRB(M), DTCallback(Callback) {}
bool run();
void getUsesOfLDSByNonKernels();
void getNonKernelsWithLDSArguments(const CallGraph &CG);
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index 888828dd57c80..21a88c3341563 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -443,7 +443,7 @@ static Value *aspaceWrapValue(DenseMap<Value *, Value *> &Cache, Function *F,
return NewGEP;
}
- IRBuilder IB(F->getContext());
+ IRBuilder IB(*F->getParent());
if (Instruction *InsnPtr = dyn_cast<Instruction>(ToWrap))
IB.SetInsertPoint(*InsnPtr->getInsertionPointAfterDef());
else
diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
index deaef574a828a..666d04cec7f38 100644
--- a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
@@ -431,7 +431,7 @@ static Constant *transformInitializer(Constant *Init, Type *OrigType,
static bool findAndReplaceVectors(Module &M) {
bool MadeChange = false;
LLVMContext &Ctx = M.getContext();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
DataScalarizerVisitor Impl;
for (GlobalVariable &G : M.globals()) {
Type *OrigType = G.getValueType();
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 9bb2521d7a9fd..93527c576495c 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -504,7 +504,7 @@ namespace dxil {
// Triple is well-formed or that the target is supported since these checks
// would have been done at the time the module M is constructed in the earlier
// stages of compilation.
-DXILOpBuilder::DXILOpBuilder(Module &M) : M(M), IRB(M.getContext()) {
+DXILOpBuilder::DXILOpBuilder(Module &M) : M(M), IRB(M) {
const Triple &TT = M.getTargetTriple();
DXILVersion = TT.getDXILVersion();
ShaderStage = TT.getEnvironment();
diff --git a/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp b/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp
index bde0729e38e0c..1bdfb7d7cc0ec 100644
--- a/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp
+++ b/llvm/lib/Target/DirectX/DXILTranslateMetadata.cpp
@@ -297,7 +297,7 @@ static void emitValidatorVersionMD(Module &M, const ModuleMetadataInfo &MMDI) {
return;
LLVMContext &Ctx = M.getContext();
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
Metadata *MDVals[2];
MDVals[0] =
ConstantAsMetadata::get(IRB.getInt32(MMDI.ValidatorVersion.getMajor()));
@@ -312,7 +312,7 @@ static void emitValidatorVersionMD(Module &M, const ModuleMetadataInfo &MMDI) {
static void emitShaderModelVersionMD(Module &M,
const ModuleMetadataInfo &MMDI) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
Metadata *SMVals[3];
VersionTuple SM = MMDI.ShaderModelVersion;
SMVals[0] = MDString::get(Ctx, getShortShaderStage(MMDI.ShaderProfile));
@@ -324,7 +324,7 @@ static void emitShaderModelVersionMD(Module &M,
static void emitDXILVersionTupleMD(Module &M, const ModuleMetadataInfo &MMDI) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
VersionTuple DXILVer = MMDI.DXILVersion;
Metadata *DXILVals[2];
DXILVals[0] = ConstantAsMetadata::get(IRB.getInt32(DXILVer.getMajor()));
@@ -535,8 +535,7 @@ static void translateGlobalMetadata(Module &M, DXILResourceMap &DRM,
DXILResourceTypeMap &DRTM,
const ModuleShaderFlags &ShaderFlags,
const ModuleMetadataInfo &MMDI) {
- LLVMContext &Ctx = M.getContext();
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
SmallVector<MDNode *> EntryFnMDNodes;
emitValidatorVersionMD(M, MMDI);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 26f44bb8f4cc6..41a6afa2f59ca 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1190,13 +1190,13 @@ HexagonTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
bool isVTi1Type = false;
if (auto *CV = dyn_cast<ConstantVector>(CPN->getConstVal())) {
if (cast<VectorType>(CV->getType())->getElementType()->isIntegerTy(1)) {
- IRBuilder<> IRB(CV->getContext());
SmallVector<Constant*, 128> NewConst;
unsigned VecLen = CV->getNumOperands();
assert(isPowerOf2_32(VecLen) &&
"conversion only supported for pow2 VectorSize");
for (unsigned i = 0; i < VecLen; ++i)
- NewConst.push_back(IRB.getInt8(CV->getOperand(i)->isNullValue()));
+ NewConst.push_back(ConstantInt::get(Type::getInt8Ty(CV->getContext()),
+ CV->getOperand(i)->isNullValue()));
CVal = ConstantVector::get(NewConst);
isVTi1Type = true;
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index db99ec5a32fb3..f2175483f24d6 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -166,7 +166,7 @@ class HexagonLoopIdiomRecognizeLegacyPass : public LoopPass {
struct Simplifier {
struct Rule {
- using FuncType = std::function<Value *(Instruction *, LLVMContext &)>;
+ using FuncType = std::function<Value *(Instruction *, Module &)>;
Rule(StringRef N, FuncType F) : Name(N), Fn(F) {}
StringRef Name; // For debugging.
FuncType Fn;
@@ -211,12 +211,9 @@ struct Simplifier {
Value *Root;
ValueSetType Used; // The set of all cloned values used by Root.
ValueSetType Clones; // The set of all cloned values.
- LLVMContext &Ctx;
+ Module &M;
- Context(Instruction *Exp)
- : Ctx(Exp->getParent()->getParent()->getContext()) {
- initialize(Exp);
- }
+ Context(Instruction *Exp) : M(*Exp->getModule()) { initialize(Exp); }
~Context() { cleanup(); }
@@ -537,7 +534,7 @@ Value *Simplifier::simplify(Context &C) {
continue;
bool Changed = false;
for (Rule &R : Rules) {
- Value *W = R.Fn(U, C.Ctx);
+ Value *W = R.Fn(U, C.M);
if (!W)
continue;
Changed = true;
@@ -1580,154 +1577,160 @@ static bool hasZeroSignBit(const Value *V) {
void PolynomialMultiplyRecognize::setupPreSimplifier(Simplifier &S) {
S.addRule("sink-zext",
- // Sink zext past bitwise operations.
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- if (I->getOpcode() != Instruction::ZExt)
- return nullptr;
- Instruction *T = dyn_cast<Instruction>(I->getOperand(0));
- if (!T)
- return nullptr;
- switch (T->getOpcode()) {
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor:
- break;
- default:
- return nullptr;
- }
- IRBuilder<> B(Ctx);
- return B.CreateBinOp(cast<BinaryOperator>(T)->getOpcode(),
- B.CreateZExt(T->getOperand(0), I->getType()),
- B.CreateZExt(T->getOperand(1), I->getType()));
- });
+ // Sink zext past bitwise operations.
+ [](Instruction *I, Module &M) -> Value * {
+ if (I->getOpcode() != Instruction::ZExt)
+ return nullptr;
+ Instruction *T = dyn_cast<Instruction>(I->getOperand(0));
+ if (!T)
+ return nullptr;
+ switch (T->getOpcode()) {
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ break;
+ default:
+ return nullptr;
+ }
+ IRBuilder<> B(M);
+ return B.CreateBinOp(
+ cast<BinaryOperator>(T)->getOpcode(),
+ B.CreateZExt(T->getOperand(0), I->getType()),
+ B.CreateZExt(T->getOperand(1), I->getType()));
+ });
S.addRule("xor/and -> and/xor",
- // (xor (and x a) (and y a)) -> (and (xor x y) a)
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- if (I->getOpcode() != Instruction::Xor)
- return nullptr;
- Instruction *And0 = dyn_cast<Instruction>(I->getOperand(0));
- Instruction *And1 = dyn_cast<Instruction>(I->getOperand(1));
- if (!And0 || !And1)
- return nullptr;
- if (And0->getOpcode() != Instruction::And ||
- And1->getOpcode() != Instruction::And)
- return nullptr;
- if (And0->getOperand(1) != And1->getOperand(1))
- return nullptr;
- IRBuilder<> B(Ctx);
- return B.CreateAnd(B.CreateXor(And0->getOperand(0), And1->getOperand(0)),
- And0->getOperand(1));
- });
- S.addRule("sink binop into select",
- // (Op (select c x y) z) -> (select c (Op x z) (Op y z))
- // (Op x (select c y z)) -> (select c (Op x y) (Op x z))
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- BinaryOperator *BO = dyn_cast<BinaryOperator>(I);
- if (!BO)
+ // (xor (and x a) (and y a)) -> (and (xor x y) a)
+ [](Instruction *I, Module &M) -> Value * {
+ if (I->getOpcode() != Instruction::Xor)
+ return nullptr;
+ Instruction *And0 = dyn_cast<Instruction>(I->getOperand(0));
+ Instruction *And1 = dyn_cast<Instruction>(I->getOperand(1));
+ if (!And0 || !And1)
+ return nullptr;
+ if (And0->getOpcode() != Instruction::And ||
+ And1->getOpcode() != Instruction::And)
+ return nullptr;
+ if (And0->getOperand(1) != And1->getOperand(1))
+ return nullptr;
+ IRBuilder<> B(M);
+ return B.CreateAnd(
+ B.CreateXor(And0->getOperand(0), And1->getOperand(0)),
+ And0->getOperand(1));
+ });
+ S.addRule(
+ "sink binop into select",
+ // (Op (select c x y) z) -> (select c (Op x z) (Op y z))
+ // (Op x (select c y z)) -> (select c (Op x y) (Op x z))
+ [](Instruction *I, Module &M) -> Value * {
+ BinaryOperator *BO = dyn_cast<BinaryOperator>(I);
+ if (!BO)
+ return nullptr;
+ Instruction::BinaryOps Op = BO->getOpcode();
+ if (SelectInst *Sel = dyn_cast<SelectInst>(BO->getOperand(0))) {
+ IRBuilder<> B(M);
+ Value *X = Sel->getTrueValue(), *Y = Sel->getFalseValue();
+ Value *Z = BO->getOperand(1);
+ return B.CreateSelect(Sel->getCondition(), B.CreateBinOp(Op, X, Z),
+ B.CreateBinOp(Op, Y, Z));
+ }
+ if (SelectInst *Sel = dyn_cast<SelectInst>(BO->getOperand(1))) {
+ IRBuilder<> B(M);
+ Value *X = BO->getOperand(0);
+ Value *Y = Sel->getTrueValue(), *Z = Sel->getFalseValue();
+ return B.CreateSelect(Sel->getCondition(), B.CreateBinOp(Op, X, Y),
+ B.CreateBinOp(Op, X, Z));
+ }
return nullptr;
- Instruction::BinaryOps Op = BO->getOpcode();
- if (SelectInst *Sel = dyn_cast<SelectInst>(BO->getOperand(0))) {
- IRBuilder<> B(Ctx);
- Value *X = Sel->getTrueValue(), *Y = Sel->getFalseValue();
- Value *Z = BO->getOperand(1);
- return B.CreateSelect(Sel->getCondition(),
- B.CreateBinOp(Op, X, Z),
- B.CreateBinOp(Op, Y, Z));
- }
- if (SelectInst *Sel = dyn_cast<SelectInst>(BO->getOperand(1))) {
- IRBuilder<> B(Ctx);
- Value *X = BO->getOperand(0);
- Value *Y = Sel->getTrueValue(), *Z = Sel->getFalseValue();
- return B.CreateSelect(Sel->getCondition(),
- B.CreateBinOp(Op, X, Y),
- B.CreateBinOp(Op, X, Z));
- }
- return nullptr;
- });
- S.addRule("fold select-select",
- // (select c (select c x y) z) -> (select c x z)
- // (select c x (select c y z)) -> (select c x z)
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- SelectInst *Sel = dyn_cast<SelectInst>(I);
- if (!Sel)
+ });
+ S.addRule(
+ "fold select-select",
+ // (select c (select c x y) z) -> (select c x z)
+ // (select c x (select c y z)) -> (select c x z)
+ [](Instruction *I, Module &M) -> Value * {
+ SelectInst *Sel = dyn_cast<SelectInst>(I);
+ if (!Sel)
+ return nullptr;
+ IRBuilder<> B(M);
+ Value *C = Sel->getCondition();
+ if (SelectInst *Sel0 = dyn_cast<SelectInst>(Sel->getTrueValue())) {
+ if (Sel0->getCondition() == C)
+ return B.CreateSelect(C, Sel0->getTrueValue(),
+ Sel->getFalseValue());
+ }
+ if (SelectInst *Sel1 = dyn_cast<SelectInst>(Sel->getFalseValue())) {
+ if (Sel1->getCondition() == C)
+ return B.CreateSelect(C, Sel->getTrueValue(),
+ Sel1->getFalseValue());
+ }
return nullptr;
- IRBuilder<> B(Ctx);
- Value *C = Sel->getCondition();
- if (SelectInst *Sel0 = dyn_cast<SelectInst>(Sel->getTrueValue())) {
- if (Sel0->getCondition() == C)
- return B.CreateSelect(C, Sel0->getTrueValue(), Sel->getFalseValue());
- }
- if (SelectInst *Sel1 = dyn_cast<SelectInst>(Sel->getFalseValue())) {
- if (Sel1->getCondition() == C)
- return B.CreateSelect(C, Sel->getTrueValue(), Sel1->getFalseValue());
- }
- return nullptr;
- });
+ });
S.addRule("or-signbit -> xor-signbit",
- // (or (lshr x 1) 0x800.0) -> (xor (lshr x 1) 0x800.0)
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- if (I->getOpcode() != Instruction::Or)
- return nullptr;
- ConstantInt *Msb = dyn_cast<ConstantInt>(I->getOperand(1));
- if (!Msb || !Msb->getValue().isSignMask())
- return nullptr;
- if (!hasZeroSignBit(I->getOperand(0)))
- return nullptr;
- return IRBuilder<>(Ctx).CreateXor(I->getOperand(0), Msb);
- });
+ // (or (lshr x 1) 0x800.0) -> (xor (lshr x 1) 0x800.0)
+ [](Instruction *I, Module &M) -> Value * {
+ if (I->getOpcode() != Instruction::Or)
+ return nullptr;
+ ConstantInt *Msb = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!Msb || !Msb->getValue().isSignMask())
+ return nullptr;
+ if (!hasZeroSignBit(I->getOperand(0)))
+ return nullptr;
+ return IRBuilder<>(M).CreateXor(I->getOperand(0), Msb);
+ });
S.addRule("sink lshr into binop",
- // (lshr (BitOp x y) c) -> (BitOp (lshr x c) (lshr y c))
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- if (I->getOpcode() != Instruction::LShr)
- return nullptr;
- BinaryOperator *BitOp = dyn_cast<BinaryOperator>(I->getOperand(0));
- if (!BitOp)
- return nullptr;
- switch (BitOp->getOpcode()) {
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor:
- break;
- default:
- return nullptr;
- }
- IRBuilder<> B(Ctx);
- Value *S = I->getOperand(1);
- return B.CreateBinOp(BitOp->getOpcode(),
- B.CreateLShr(BitOp->getOperand(0), S),
- B.CreateLShr(BitOp->getOperand(1), S));
- });
+ // (lshr (BitOp x y) c) -> (BitOp (lshr x c) (lshr y c))
+ [](Instruction *I, Module &M) -> Value * {
+ if (I->getOpcode() != Instruction::LShr)
+ return nullptr;
+ BinaryOperator *BitOp =
+ dyn_cast<BinaryOperator>(I->getOperand(0));
+ if (!BitOp)
+ return nullptr;
+ switch (BitOp->getOpcode()) {
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ break;
+ default:
+ return nullptr;
+ }
+ IRBuilder<> B(M);
+ Value *S = I->getOperand(1);
+ return B.CreateBinOp(BitOp->getOpcode(),
+ B.CreateLShr(BitOp->getOperand(0), S),
+ B.CreateLShr(BitOp->getOperand(1), S));
+ });
S.addRule("expose bitop-const",
- // (BitOp1 (BitOp2 x a) b) -> (BitOp2 x (BitOp1 a b))
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- auto IsBitOp = [](unsigned Op) -> bool {
- switch (Op) {
- case Instruction::And:
- case Instruction::Or:
- case Instruction::Xor:
- return true;
- }
- return false;
- };
- BinaryOperator *BitOp1 = dyn_cast<BinaryOperator>(I);
- if (!BitOp1 || !IsBitOp(BitOp1->getOpcode()))
- return nullptr;
- BinaryOperator *BitOp2 = dyn_cast<BinaryOperator>(BitOp1->getOperand(0));
- if (!BitOp2 || !IsBitOp(BitOp2->getOpcode()))
- return nullptr;
- ConstantInt *CA = dyn_cast<ConstantInt>(BitOp2->getOperand(1));
- ConstantInt *CB = dyn_cast<ConstantInt>(BitOp1->getOperand(1));
- if (!CA || !CB)
- return nullptr;
- IRBuilder<> B(Ctx);
- Value *X = BitOp2->getOperand(0);
- return B.CreateBinOp(BitOp2->getOpcode(), X,
- B.CreateBinOp(BitOp1->getOpcode(), CA, CB));
- });
+ // (BitOp1 (BitOp2 x a) b) -> (BitOp2 x (BitOp1 a b))
+ [](Instruction *I, Module &M) -> Value * {
+ auto IsBitOp = [](unsigned Op) -> bool {
+ switch (Op) {
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ return true;
+ }
+ return false;
+ };
+ BinaryOperator *BitOp1 = dyn_cast<BinaryOperator>(I);
+ if (!BitOp1 || !IsBitOp(BitOp1->getOpcode()))
+ return nullptr;
+ BinaryOperator *BitOp2 =
+ dyn_cast<BinaryOperator>(BitOp1->getOperand(0));
+ if (!BitOp2 || !IsBitOp(BitOp2->getOpcode()))
+ return nullptr;
+ ConstantInt *CA = dyn_cast<ConstantInt>(BitOp2->getOperand(1));
+ ConstantInt *CB = dyn_cast<ConstantInt>(BitOp1->getOperand(1));
+ if (!CA || !CB)
+ return nullptr;
+ IRBuilder<> B(M);
+ Value *X = BitOp2->getOperand(0);
+ return B.CreateBinOp(BitOp2->getOpcode(), X,
+ B.CreateBinOp(BitOp1->getOpcode(), CA, CB));
+ });
S.addRule("select with trunc cond to select with icmp cond",
// select (trunc x to i1) -> select (icmp ne (and x, 1), 0)
// select (xor (trunc x to i1) 1) -> select (icmp eq (and x, 1), 0)
- [](Instruction *I, LLVMContext &Ctx) -> Value * {
+ [](Instruction *I, Module &M) -> Value * {
SelectInst *Sel = dyn_cast<SelectInst>(I);
if (!Sel)
return nullptr;
@@ -1738,7 +1741,7 @@ void PolynomialMultiplyRecognize::setupPreSimplifier(Simplifier &S) {
match(C, m_Not(m_Trunc(m_Value(X))))))
return nullptr;
- IRBuilder<> B(Ctx);
+ IRBuilder<> B(M);
Type *Ty = X->getType();
Value *And = B.CreateAnd(X, ConstantInt::get(Ty, 1));
Value *Icmp = B.CreateICmp(isa<TruncInst>(C) ? ICmpInst::ICMP_NE
@@ -1751,30 +1754,30 @@ void PolynomialMultiplyRecognize::setupPreSimplifier(Simplifier &S) {
void PolynomialMultiplyRecognize::setupPostSimplifier(Simplifier &S) {
S.addRule("(and (xor (and x a) y) b) -> (and (xor x y) b), if b == b&a",
- [](Instruction *I, LLVMContext &Ctx) -> Value* {
- if (I->getOpcode() != Instruction::And)
- return nullptr;
- Instruction *Xor = dyn_cast<Instruction>(I->getOperand(0));
- ConstantInt *C0 = dyn_cast<ConstantInt>(I->getOperand(1));
- if (!Xor || !C0)
- return nullptr;
- if (Xor->getOpcode() != Instruction::Xor)
- return nullptr;
- Instruction *And0 = dyn_cast<Instruction>(Xor->getOperand(0));
- Instruction *And1 = dyn_cast<Instruction>(Xor->getOperand(1));
- // Pick the first non-null and.
- if (!And0 || And0->getOpcode() != Instruction::And)
- std::swap(And0, And1);
- ConstantInt *C1 = dyn_cast<ConstantInt>(And0->getOperand(1));
- if (!C1)
- return nullptr;
- uint32_t V0 = C0->getZExtValue();
- uint32_t V1 = C1->getZExtValue();
- if (V0 != (V0 & V1))
- return nullptr;
- IRBuilder<> B(Ctx);
- return B.CreateAnd(B.CreateXor(And0->getOperand(0), And1), C0);
- });
+ [](Instruction *I, Module &M) -> Value * {
+ if (I->getOpcode() != Instruction::And)
+ return nullptr;
+ Instruction *Xor = dyn_cast<Instruction>(I->getOperand(0));
+ ConstantInt *C0 = dyn_cast<ConstantInt>(I->getOperand(1));
+ if (!Xor || !C0)
+ return nullptr;
+ if (Xor->getOpcode() != Instruction::Xor)
+ return nullptr;
+ Instruction *And0 = dyn_cast<Instruction>(Xor->getOperand(0));
+ Instruction *And1 = dyn_cast<Instruction>(Xor->getOperand(1));
+ // Pick the first non-null and.
+ if (!And0 || And0->getOpcode() != Instruction::And)
+ std::swap(And0, And1);
+ ConstantInt *C1 = dyn_cast<ConstantInt>(And0->getOperand(1));
+ if (!C1)
+ return nullptr;
+ uint32_t V0 = C0->getZExtValue();
+ uint32_t V1 = C1->getZExtValue();
+ if (V0 != (V0 & V1))
+ return nullptr;
+ IRBuilder<> B(M);
+ return B.CreateAnd(B.CreateXor(And0->getOperand(0), And1), C0);
+ });
}
bool PolynomialMultiplyRecognize::recognize() {
diff --git a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
index 1652fa42ec89d..36b5ba5b30d57 100644
--- a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
+++ b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
@@ -104,7 +104,7 @@ class PPCBoolRetToInt : public FunctionPass {
return Q;
}
- IRBuilder IRB(V->getContext());
+ IRBuilder IRB(*Func->getParent());
if (auto *I = dyn_cast<Instruction>(V))
IRB.SetInsertPoint(I->getNextNode());
else
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 27675e8e6e75f..d56e84a9bd2c0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -614,7 +614,7 @@ Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
Type *ElemTy) {
- IRBuilder<> B(Op->getContext());
+ IRBuilder<> B(*F->getParent());
if (auto *OpI = dyn_cast<Instruction>(Op)) {
// spv_ptrcast's argument Op denotes an instruction that generates
// a value, and we may use getInsertionPointAfterDef()
@@ -1536,8 +1536,7 @@ void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
if (!KnownElemTy || Ops.size() == 0)
return;
- LLVMContext &Ctx = CurrF->getContext();
- IRBuilder<> B(Ctx);
+ IRBuilder<> B(*CurrF->getParent());
for (auto &OpIt : Ops) {
Value *Op = OpIt.first;
if (AskOps && !AskOps->contains(Op))
@@ -3765,7 +3764,7 @@ bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
CanUseAnyVectorRank =
ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
CurrF = &Func;
- IRBuilder<> B(Func.getContext());
+ IRBuilder<> B(*Func.getParent());
AggrConsts.clear();
AggrConstTypes.clear();
AggrStores.clear();
@@ -4187,7 +4186,7 @@ bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
// check if function parameter types are set
CurrF = &F;
if (!F.isDeclaration() && !F.isIntrinsic()) {
- IRBuilder<> B(F.getContext());
+ IRBuilder<> B(M);
processParamTypes(&F, B);
}
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
index 0e86ad2b5108b..6effed3dae94f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp
@@ -777,7 +777,7 @@ class SPIRVLegalizePointerCastImpl {
Value *CastedOperand = II;
Value *OriginalOperand = II->getOperand(0);
- IRBuilder<> B(II->getContext());
+ IRBuilder<> B(*II->getModule());
std::vector<Value *> Users;
for (Use &U : II->uses())
Users.push_back(U.getUser());
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 537a98c43ec59..bba56486fb278 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -590,7 +590,7 @@ SPIRVPrepareFunctionsImpl::removeAggregateTypesFromSignature(Function *F) {
if (F->isIntrinsic())
return F;
- IRBuilder<> B(F->getContext());
+ IRBuilder<> B(*F->getParent());
bool HasAggrArg = llvm::any_of(F->args(), [](Argument &Arg) {
return Arg.getType()->isAggregateType();
@@ -785,7 +785,7 @@ bool SPIRVPrepareFunctionsImpl::removeAggregateTypesFromCalls(Function *F) {
if (Calls.empty())
return false;
- IRBuilder<> B(F->getContext());
+ IRBuilder<> B(*F->getParent());
unsigned MutatedCallIdx = 0;
for (auto &&[CB, NewFnTy] : Calls) {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index c885f3ecc443b..07ddfb1b852f9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -478,7 +478,7 @@ static void markAsImported(Function *F) {
// Returns an integer type for the target architecture's address space.
// i32 for wasm32 and i64 for wasm64.
static Type *getAddrIntType(Module *M) {
- IRBuilder<> IRB(M->getContext());
+ IRBuilder<> IRB(*M);
return IRB.getIntNTy(M->getDataLayout().getPointerSizeInBits());
}
@@ -493,7 +493,7 @@ static Type *getAddrPtrType(Module *M) {
// space. Returns (i32 C) for wasm32 and (i64 C) for wasm64, when C is the
// integer.
static Value *getAddrSizeInt(Module *M, uint64_t C) {
- IRBuilder<> IRB(M->getContext());
+ IRBuilder<> IRB(*M);
return IRB.getIntN(M->getDataLayout().getPointerSizeInBits(), C);
}
@@ -854,8 +854,7 @@ void WebAssemblyLowerEmscriptenEHSjLjImpl::replaceLongjmpWith(
assert(NewF == EmLongjmpF || NewF == WasmLongjmpF);
Module *M = LongjmpF->getParent();
SmallVector<CallInst *, 8> ToErase;
- LLVMContext &C = LongjmpF->getParent()->getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(*LongjmpF->getParent());
// For calls to longjmp, replace it with emscripten_longjmp/__wasm_longjmp and
// cast its first argument (jmp_buf*) appropriately
@@ -900,7 +899,7 @@ static bool containsLongjmpableCalls(const Function *F) {
// returns 0 when called directly.
static void nullifySetjmp(Function *F) {
Module &M = *F->getParent();
- IRBuilder<> IRB(M.getContext());
+ IRBuilder<> IRB(M);
Function *SetjmpF = M.getFunction("setjmp");
SmallVector<Instruction *, 1> ToErase;
@@ -930,8 +929,7 @@ bool WebAssemblyLowerEmscriptenEHSjLjImpl::runOnModule(Module &M) {
// threaded in from the TargetMachine.
EnableEmEH |= M.getExceptionModel() == ExceptionHandling::Emscripten;
- LLVMContext &C = M.getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(M);
Function *SetjmpF = M.getFunction("setjmp");
Function *LongjmpF = M.getFunction("longjmp");
@@ -1127,7 +1125,7 @@ bool WebAssemblyLowerEmscriptenEHSjLjImpl::runOnModule(Module &M) {
bool WebAssemblyLowerEmscriptenEHSjLjImpl::runEHOnFunction(Function &F) {
Module &M = *F.getParent();
LLVMContext &C = F.getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(M);
bool Changed = false;
SmallVector<Instruction *, 64> ToErase;
SmallPtrSet<LandingPadInst *, 32> LandingPads;
@@ -1319,8 +1317,7 @@ static DebugLoc getOrCreateDebugLoc(const Instruction *InsertBefore,
bool WebAssemblyLowerEmscriptenEHSjLjImpl::runSjLjOnFunction(Function &F) {
assert(EnableEmSjLj || EnableWasmSjLj);
Module &M = *F.getParent();
- LLVMContext &C = F.getContext();
- IRBuilder<> IRB(C);
+ IRBuilder IRB(M);
SmallVector<Instruction *, 64> ToErase;
// Setjmp preparation
@@ -1431,7 +1428,7 @@ void WebAssemblyLowerEmscriptenEHSjLjImpl::
SmallVectorImpl<PHINode *> &SetjmpRetPHIs) {
Module &M = *F.getParent();
LLVMContext &C = F.getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(M);
SmallVector<Instruction *, 64> ToErase;
// call.em.longjmp BB that will be shared within the function.
@@ -1626,7 +1623,7 @@ void WebAssemblyLowerEmscriptenEHSjLjImpl::handleLongjmpableCallsForWasmSjLj(
SmallVectorImpl<PHINode *> &SetjmpRetPHIs) {
Module &M = *F.getParent();
LLVMContext &C = F.getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(M);
// A function with catchswitch/catchpad instruction should have a personality
// function attached to it. Search for the wasm personality function, and if
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 2d62534b18696..7353428a058de 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -2040,7 +2040,7 @@ bool StrNCmpInliner::optimizeStrNCmp() {
void StrNCmpInliner::inlineCompare(Value *LHS, StringRef RHS, uint64_t N,
bool Swapped) {
auto &Ctx = CI->getContext();
- IRBuilder<> B(Ctx);
+ IRBuilder<> B(*CI->getModule());
// We want these instructions to be recognized as inlined instructions for the
// compare call, but we don't have a source location for the definition of
// that function, since we're generating that code now. Because the generated
diff --git a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
index 20de75d0ade60..d2d083c489444 100644
--- a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
@@ -29,7 +29,7 @@ struct Lowerer : coro::LowererBase {
IRBuilder<> Builder;
Constant *NoopCoro = nullptr;
- Lowerer(Module &M) : LowererBase(M), Builder(Context) {}
+ Lowerer(Module &M) : LowererBase(M), Builder(M) {}
bool lower(Function &F);
private:
@@ -43,7 +43,7 @@ class NoopCoroElider : public PtrUseVisitor<NoopCoroElider> {
IRBuilder<> Builder;
public:
- NoopCoroElider(const DataLayout &DL, LLVMContext &C) : Base(DL), Builder(C) {}
+ NoopCoroElider(Module &M) : Base(M.getDataLayout()), Builder(M) {}
void run(IntrinsicInst *II);
@@ -95,7 +95,7 @@ bool Lowerer::lower(Function &F) {
bool IsPrivateAndUnprocessed = F.isPresplitCoroutine() && F.hasLocalLinkage();
bool Changed = false;
- NoopCoroElider NCE(F.getDataLayout(), F.getContext());
+ NoopCoroElider NCE(*F.getParent());
SmallPtrSet<Instruction *, 8> DeadInsts{};
for (Instruction &I : instructions(F)) {
if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroCloner.h b/llvm/lib/Transforms/Coroutines/CoroCloner.h
index 1e549f122b6ba..3d01879e4c1ff 100644
--- a/llvm/lib/Transforms/Coroutines/CoroCloner.h
+++ b/llvm/lib/Transforms/Coroutines/CoroCloner.h
@@ -63,7 +63,7 @@ class BaseCloner {
: OrigF(OrigF), Suffix(Suffix), Shape(Shape),
FKind(Shape.ABI == ABI::Async ? CloneKind::Async
: CloneKind::Continuation),
- Builder(OrigF.getContext()), TTI(TTI), NewF(NewF),
+ Builder(*OrigF.getParent()), TTI(TTI), NewF(NewF),
ActiveSuspend(ActiveSuspend) {
assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce ||
Shape.ABI == ABI::Async);
@@ -75,7 +75,7 @@ class BaseCloner {
BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
CloneKind FKind, TargetTransformInfo &TTI)
: OrigF(OrigF), Suffix(Suffix), Shape(Shape), FKind(FKind),
- Builder(OrigF.getContext()), TTI(TTI) {}
+ Builder(*OrigF.getParent()), TTI(TTI) {}
virtual ~BaseCloner() = default;
diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index ecb34fa11e0fc..d15b4dbd7f270 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -29,7 +29,7 @@ class Lowerer : public coro::LowererBase {
public:
Lowerer(Module &M)
- : LowererBase(M), Builder(Context),
+ : LowererBase(M), Builder(M),
AnyResumeFnPtrTy(PointerType::getUnqual(Context)) {}
void lowerEarlyIntrinsics(Function &F);
};
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index bb11b5ee1038a..8c0f105b77d6a 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1065,7 +1065,7 @@ static void handleAccessBeforeCoroBegin(const FrameDataInfo &FrameData,
static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
LLVMContext &C = Shape.CoroBegin->getContext();
Function *F = Shape.CoroBegin->getFunction();
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(*F->getParent());
DominatorTree DT(*F);
SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
@@ -1830,11 +1830,10 @@ static std::optional<std::pair<Value &, DIExpression &>>
salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
bool UseEntryValue, Function *F, Value *Storage,
DIExpression *Expr, bool SkipOutermostLoad) {
- IRBuilder<> Builder(F->getContext());
auto InsertPt = F->getEntryBlock().getFirstInsertionPt();
while (isa<IntrinsicInst>(InsertPt))
++InsertPt;
- Builder.SetInsertPoint(InsertPt);
+ IRBuilder<> Builder(InsertPt);
while (auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
if (auto *LdInst = dyn_cast<LoadInst>(Inst)) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 764e07cd53f2b..dd7d00b7f0c22 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -151,7 +151,7 @@ static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB,
}
static void lowerAwaitSuspends(Function &F, coro::Shape &Shape) {
- IRBuilder<> Builder(F.getContext());
+ IRBuilder<> Builder(*F.getParent());
for (auto *AWS : Shape.CoroAwaitSuspends)
lowerAwaitSuspend(Builder, AWS, Shape);
}
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index ae21725cea2e4..9af51dedab174 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -6934,9 +6934,8 @@ struct AAHeapToStackFunction final : public AAHeapToStack {
if (SizeAPI) {
Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
} else {
- LLVMContext &Ctx = AI.CB->getContext();
ObjectSizeOpts Opts;
- ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
+ ObjectSizeOffsetEvaluator Eval(*AI.CB->getModule(), TLI, Opts);
SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
index 44d944933cc3a..483492fd0feb7 100644
--- a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
+++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
@@ -364,9 +364,8 @@ bool ExpandVariadics::runOnModule(Module &M) {
if (!ABI->enableForTarget())
return Changed;
- auto &Ctx = M.getContext();
const DataLayout &DL = M.getDataLayout();
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(M);
// Lowering needs to run on all functions exactly once.
// Optimize could run on functions containing va_start exactly once.
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 36563e25cc7f7..3072371b9fede 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -1840,7 +1840,7 @@ static void RemovePreallocated(Function *F) {
auto *M = F->getParent();
- IRBuilder<> Builder(M->getContext());
+ IRBuilder<> Builder(*M);
// Cannot modify users() while iterating over it, so make a copy.
SmallVector<User *, 4> PreallocatedCalls(F->users());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index cd95225f5f657..8893744821f7e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -538,7 +538,7 @@ static Value *rewriteGEPAsOffset(Value *Start, Value *Base, GEPNoWrapFlags NW,
PHINode::Create(IndexType, PHI->getNumIncomingValues(),
PHI->getName() + ".idx", PHI->getIterator());
}
- IRBuilder<> Builder(Base->getContext());
+ IRBuilder<> Builder(IC.getModule());
// Create all the other instructions.
for (Value *Val : Explored) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 388530edc10da..07bc7e302c9ec 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -891,8 +891,8 @@ class Negator final {
SmallDenseMap<Value *, Value *> NegationsCache;
- Negator(LLVMContext &C, const DataLayout &DL, const DominatorTree &DT,
- bool IsTrulyNegation, unsigned MaxDepth);
+ Negator(Module &M, const DominatorTree &DT, bool IsTrulyNegation,
+ unsigned MaxDepth);
#if LLVM_ENABLE_STATS
unsigned NumValuesVisitedInThisNegator = 0;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index 9dceeef7bbab3..5e3eb41862f03 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -78,9 +78,9 @@ STATISTIC(NegatorNumInstructionsNegatedSuccess,
DEBUG_COUNTER(NegatorCounter, "instcombine-negator",
"Controls Negator transformations in InstCombine pass");
-Negator::Negator(LLVMContext &C, const DataLayout &DL, const DominatorTree &DT_,
- bool IsTrulyNegation_, unsigned MaxDepth)
- : Builder(C, TargetFolder(DL),
+Negator::Negator(Module &M, const DominatorTree &DT_, bool IsTrulyNegation_,
+ unsigned MaxDepth)
+ : Builder(M, TargetFolder(M.getDataLayout()),
IRBuilderCallbackInserter([&](Instruction *I) {
++NegatorNumInstructionsCreatedTotal;
NewInstructions.push_back(I);
@@ -548,8 +548,8 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
!DebugCounter::shouldExecute(NegatorCounter))
return nullptr;
- Negator N(Root->getContext(), IC.getDataLayout(), IC.getDominatorTree(),
- LHSIsZero, IC.CLOpts.negator_max_depth);
+ Negator N(IC.getModule(), IC.getDominatorTree(), LHSIsZero,
+ IC.CLOpts.negator_max_depth);
std::optional<Result> Res = N.run(Root, IsNSW);
if (!Res) { // Negation failed.
LLVM_DEBUG(dbgs() << "Negator: failed to sink negation into " << *Root
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index c70597ced1370..4d3be3b5834dc 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -2361,7 +2361,7 @@ StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
}
void ModuleAddressSanitizer::initializeCallbacks() {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
// Declare our poisoning and unpoisoning functions.
AsanPoisonGlobals = Inserter.insertFunction(kAsanPoisonGlobalsName,
@@ -2910,7 +2910,7 @@ bool ModuleAddressSanitizer::instrumentModule() {
IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
instrumentGlobals(IRB, &CtorComdat);
} else {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
instrumentGlobals(IRB, &CtorComdat);
}
}
@@ -2940,7 +2940,7 @@ bool ModuleAddressSanitizer::instrumentModule() {
}
void AddressSanitizer::initializeCallbacks(const TargetLibraryInfo *TLI) {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
// Create __asan_report* callbacks.
// IsWrite, TypeSize and Exp are encoded in the function name.
for (int Exp = 0; Exp < 2; Exp++) {
@@ -3273,8 +3273,8 @@ bool AddressSanitizer::LooksLikeCodeInBug11395(Instruction *I) {
return true;
}
-void FunctionStackPoisoner::initializeCallbacks(Module &) {
- IRBuilder<> IRB(*C);
+void FunctionStackPoisoner::initializeCallbacks(Module &M) {
+ IRBuilder<> IRB(M);
if (ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Always ||
ASan.UseAfterReturn == AsanDetectStackUseAfterReturnMode::Runtime) {
const char *MallocNameTemplate =
diff --git a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
index 3aed97bf8363c..66e890ee62dbd 100644
--- a/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/llvm/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -206,7 +206,7 @@ static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI,
ObjectSizeOpts EvalOpts;
EvalOpts.RoundToAlign = true;
EvalOpts.EvalMode = ObjectSizeOpts::Mode::ExactUnderlyingSizeAndOffset;
- ObjectSizeOffsetEvaluator ObjSizeEval(DL, &TLI, F.getContext(), EvalOpts);
+ ObjectSizeOffsetEvaluator ObjSizeEval(*F.getParent(), &TLI, EvalOpts);
// check HANDLE_MEMORY_INST in include/llvm/Instruction.def for memory
// touching instructions
diff --git a/llvm/lib/Transforms/Instrumentation/CopyProf.cpp b/llvm/lib/Transforms/Instrumentation/CopyProf.cpp
index c21703145dff8..840f9c0b7c8b1 100644
--- a/llvm/lib/Transforms/Instrumentation/CopyProf.cpp
+++ b/llvm/lib/Transforms/Instrumentation/CopyProf.cpp
@@ -174,7 +174,7 @@ class CopyProfStores {
CopyProf::CopyProf(Module &M) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
IntPtrTy = IRB.getIntPtrTy(M.getDataLayout());
Type *PtrTy = IRB.getPtrTy();
Type *VoidTy = IRB.getVoidTy();
@@ -254,7 +254,7 @@ void CopyProf::insertCallback(Function &F, size_t ObjSize, unsigned NumArgs,
CopyProfStores::CopyProfStores(Module &M) {
LLVMContext &Ctx = M.getContext();
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
IntPtrTy = IRB.getIntPtrTy(M.getDataLayout());
Type *PtrTy = IRB.getPtrTy();
Type *VoidTy = IRB.getVoidTy();
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index db6bf996a28c6..3ad957c34a2d4 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -691,7 +691,7 @@ void HWAddressSanitizer::initializeModule() {
Mapping.init(TargetTriple, InstrumentWithCalls, CompileKernel);
C = &(M.getContext());
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
HwasanCtorFunction = nullptr;
@@ -750,7 +750,7 @@ void HWAddressSanitizer::initializeModule() {
}
void HWAddressSanitizer::initializeCallbacks(Module &M) {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
*HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
index 72697ffeadf7a..310d8bc77edbc 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp
@@ -546,7 +546,7 @@ bool ModuleMemProfiler::instrumentModule(Module &M) {
}
void MemProfiler::initializeCallbacks(Module &M) {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
const std::string TypeStr = AccessIsWrite ? "store" : "load";
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 360c071bc95f1..59b4903b416d2 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -858,7 +858,7 @@ MemorySanitizer::getOrInsertMsanMetadataFunction(Module &M, StringRef Name,
/// Create KMSAN API callbacks.
void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
// These will be initialized in insertKmsanPrologue().
RetvalTLS = nullptr;
@@ -921,7 +921,7 @@ static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
/// Insert declarations for userspace-specific functions and globals.
void MemorySanitizer::createUserspaceApi(Module &M,
const TargetLibraryInfo &TLI) {
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
// Create the callback.
// FIXME: this function should have "Cold" calling conv,
@@ -998,7 +998,7 @@ void MemorySanitizer::initializeCallbacks(Module &M,
if (CallbacksInitialized)
return;
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
// Initialize callbacks that are common for kernel and userspace
// instrumentation.
MsanChainOriginFn = M.getOrInsertFunction(
@@ -1127,7 +1127,7 @@ void MemorySanitizer::initializeModule(Module &M) {
}
C = &(M.getContext());
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
IntptrTy = IRB.getIntPtrTy(DL);
OriginTy = IRB.getInt32Ty();
PtrTy = IRB.getPtrTy();
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
index 210b1266de23c..c320bb61fd810 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp
@@ -130,7 +130,7 @@ class SanitizerBinaryMetadata {
std::unique_ptr<SpecialCaseList> Ignorelist)
: Mod(M), Options(transformOptionsFromCl(std::move(Opts))),
Ignorelist(std::move(Ignorelist)), TargetTriple(M.getTargetTriple()),
- VersionStr(utostr(getVersion())), IRB(M.getContext()) {
+ VersionStr(utostr(getVersion())), IRB(M) {
// FIXME: Make it work with other formats.
assert(TargetTriple.isOSBinFormatELF() && "ELF only");
assert(!TargetTriple.isGPU() && "Device targets are not supported");
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 9e59f60d3d926..d97aca5d9c0b9 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -443,7 +443,7 @@ bool ModuleSanitizerCoverage::instrumentModule() {
IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits());
PtrTy = PointerType::getUnqual(*C);
Type *VoidTy = Type::getVoidTy(*C);
- IRBuilder<> IRB(*C);
+ IRBuilder<> IRB(M);
Int64Ty = IRB.getInt64Ty();
Int32Ty = IRB.getInt32Ty();
Int16Ty = IRB.getInt16Ty();
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 5f399e25db5ff..7424307829b85 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -205,7 +205,7 @@ void ThreadSanitizer::initialize(Module &M, const TargetLibraryInfo &TLI) {
LLVMContext &Ctx = M.getContext();
IntptrTy = DL.getIntPtrType(Ctx);
- IRBuilder<> IRB(Ctx);
+ IRBuilder<> IRB(M);
AttributeList Attr;
Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind);
// Initialize the callbacks.
diff --git a/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
index 524a9261fbb1f..2e1806fd5b9fa 100644
--- a/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
@@ -145,7 +145,7 @@ TypeSanitizer::TypeSanitizer(Module &M)
void TypeSanitizer::initializeCallbacks(Module &M) {
LLVMContext &C = M.getContext();
- IRBuilder<> IRB(C);
+ IRBuilder<> IRB(M);
OrdTy = IRB.getInt32Ty();
U64Ty = IRB.getInt64Ty();
Type *BoolType = IRB.getInt1Ty();
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 599f9f42f9ca8..e4c8c3029fe05 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -2897,7 +2897,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8
: 0),
PHIUsers(PHIUsers), SelectUsers(SelectUsers),
- IRB(NewAI.getContext(), ConstantFolder()) {
+ IRB(*NewAI.getModule(), ConstantFolder()) {
if (VecTy) {
assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
"Only multiple-of-8 sized vector elements are viable");
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index a8eae7dcd5ec0..41c3c2c3c0e19 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -2284,7 +2284,7 @@ inlineRetainOrClaimRVCalls(CallBase &CB, objcarc::ARCInstKind RVCallKind,
for (auto *RI : Returns) {
Value *RetOpnd = objcarc::GetRCIdentityRoot(RI->getOperand(0));
bool InsertRetainCall = IsRetainRV;
- IRBuilder<> Builder(RI->getContext());
+ IRBuilder<> Builder(*RI->getModule());
// Walk backwards through the basic block looking for either a matching
// autoreleaseRV call or an unannotated call.
diff --git a/llvm/lib/Transforms/Utils/LoopVersioning.cpp b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
index f77c4918d15bd..4a02cf5e8298c 100644
--- a/llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -78,7 +78,7 @@ void LoopVersioning::versionLoop(
Exp.expandCodeForPredicate(&Preds, RuntimeCheckBB->getTerminator());
IRBuilder<InstSimplifyFolder> Builder(
- RuntimeCheckBB->getContext(),
+ *RuntimeCheckBB->getModule(),
InstSimplifyFolder(RuntimeCheckBB->getDataLayout()));
if (MemRuntimeCheck && SCEVRuntimeCheck) {
Builder.SetInsertPoint(RuntimeCheckBB->getTerminator());
diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
index 71e9c70cbf9a5..78315855e394c 100644
--- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
@@ -29,7 +29,7 @@ using namespace llvm;
static void appendToGlobalArray(StringRef ArrayName, Module &M, Function *F,
int Priority, Constant *Data) {
- IRBuilder<> IRB(M.getContext());
+ IRBuilder<> IRB(M);
// Get the current set of static global constructors and add the new ctor
// to the list.
@@ -85,7 +85,7 @@ static void transformGlobalArray(StringRef ArrayName, Module &M,
if (!GVCtor)
return;
- IRBuilder<> IRB(M.getContext());
+ IRBuilder<> IRB(M);
SmallVector<Constant *, 16> CurrentCtors;
bool Changed = false;
StructType *EltTy =
@@ -264,7 +264,7 @@ std::pair<Function *, FunctionCallee> llvm::createSanitizerCtorAndInitFunctions(
FunctionCallee InitFunction =
declareSanitizerInitFunction(M, InitName, InitArgTypes, Weak);
Function *Ctor = createSanitizerCtor(M, CtorName);
- IRBuilder<> IRB(M.getContext());
+ IRBuilder<> IRB(M);
BasicBlock *RetBB = &Ctor->getEntryBlock();
if (Weak) {
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index e3db3b230c238..b7b442f64fda0 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -4139,7 +4139,7 @@ static bool performBranchToCommonDestFolding(CondBrInst *BI, CondBrInst *PBI,
LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
IRBuilder<ConstantFolder, IRBuilderCallbackInserter> Builder(
- BB->getContext(), ConstantFolder{},
+ *BB->getModule(), ConstantFolder{},
IRBuilderCallbackInserter([&BB](Instruction *I) {
// The builder is used to create instructions to eliminate the branch in
// BB. If BB's terminator has !annotation metadata, add it to the new
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index eb0b152f6bb6f..202670ff815b6 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -258,8 +258,8 @@ class Vectorizer {
public:
Vectorizer(Function &F, AliasAnalysis &AA, AssumptionCache &AC,
DominatorTree &DT, ScalarEvolution &SE, TargetTransformInfo &TTI)
- : F(F), AA(AA), AC(AC), DT(DT), SE(SE), TTI(TTI),
- DL(F.getDataLayout()), Builder(SE.getContext()) {}
+ : F(F), AA(AA), AC(AC), DT(DT), SE(SE), TTI(TTI), DL(F.getDataLayout()),
+ Builder(SE.getModule()) {}
bool run();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3cdc144bace67..326f202fd150f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -546,7 +546,7 @@ class InnerLoopVectorizer {
ElementCount VecWidth, unsigned UnrollFactor,
GeneratedRTChecks &RTChecks, VPlan &Plan)
: OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
- VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
+ VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getModule()),
RTChecks(RTChecks), Plan(Plan),
VectorPHVPBB(cast<VPBasicBlock>(
Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c7bd38813b72c..eb9bfb60e2de0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -434,7 +434,7 @@ class slpvectorizer::BoUpSLP {
: AA(Aa), BatchAA(std::in_place, *Aa), F(Func), SE(Se), TTI(Tti),
TLI(TLi), LI(Li), DT(Dt), AC(AC), DB(DB), DL(DL), ORE(ORE),
CostKind(getSLPCostKind(Func)),
- Builder(Se->getContext(), TargetFolder(*DL)) {
+ Builder(Se->getModule(), TargetFolder(*DL)) {
CodeMetrics::collectEphemeralValues(F, AC, EphValues);
// Use the vector register size specified by the target unless overridden
// by a command-line option.
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 72b72e51dbe6b..c24bebadb6506 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -80,7 +80,7 @@ class VectorCombine {
const DominatorTree &DT, AAResults &AA, AssumptionCache &AC,
const DataLayout *DL, TTI::TargetCostKind CostKind,
bool TryEarlyFoldsOnly)
- : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI),
+ : F(F), Builder(*F.getParent(), InstSimplifyFolder(*DL)), TTI(TTI),
DT(DT), AA(AA), DL(DL), CostKind(CostKind),
SQ(*DL, /*TLI=*/nullptr, &DT, &AC),
TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index 6bee74935a115..3a36043aa0504 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -375,12 +375,12 @@ class LLIObjectCache : public ObjectCache {
// an empty '__main' function.
static void addCygMingExtraModule(ExecutionEngine &EE, LLVMContext &Context,
const Triple &TargetTriple) {
- IRBuilder<> Builder(Context);
-
// Create a new module.
std::unique_ptr<Module> M = std::make_unique<Module>("CygMingHelper", Context);
M->setTargetTriple(TargetTriple);
+ IRBuilder<> Builder(*M);
+
// Create an empty function named "__main".
Type *ReturnTy;
if (TargetTriple.isArch64Bit())
diff --git a/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp b/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
index 0541f257b20d2..2f2bb676053a9 100644
--- a/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
@@ -66,7 +66,7 @@ class BasicAATest : public testing::Test {
public:
BasicAATest()
- : M("BasicAATest", C), B(C), DL(DLString), TLII(M.getTargetTriple()),
+ : M("BasicAATest", C), B(M), DL(DLString), TLII(M.getTargetTriple()),
TLI(TLII), F(nullptr) {}
};
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index 450241963af53..49701aaf0168d 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -70,7 +70,7 @@ class MemorySSATest : public testing::Test {
public:
MemorySSATest()
- : M("MemorySSATest", C), B(C), DL(DLString), TLII(M.getTargetTriple()),
+ : M("MemorySSATest", C), B(M), DL(DLString), TLII(M.getTargetTriple()),
TLI(TLII), F(nullptr) {}
};
@@ -896,7 +896,7 @@ TEST_F(MemorySSATest, Irreducible) {
// }
// use(x)
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
F = Function::Create(FunctionType::get(B.getVoidTy(), {B.getPtrTy()}, false),
GlobalValue::ExternalLinkage, "F", &M);
@@ -935,7 +935,7 @@ TEST_F(MemorySSATest, MoveToBeforeLiveOnEntryInvalidatesCache) {
//
// ...And be sure that MSSA's caching doesn't give us `1` for the clobber of
// `2` after `1` is removed.
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
F = Function::Create(FunctionType::get(B.getVoidTy(), {B.getPtrTy()}, false),
GlobalValue::ExternalLinkage, "F", &M);
@@ -980,7 +980,7 @@ TEST_F(MemorySSATest, RemovingDefInvalidatesCache) {
//
// And be sure that MSSA's caching handles the removal of def `1`
// appropriately.
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
F = Function::Create(FunctionType::get(B.getVoidTy(), {B.getPtrTy()}, false),
GlobalValue::ExternalLinkage, "F", &M);
@@ -1130,7 +1130,7 @@ TEST_F(MemorySSATest, LifetimeMarkersAreClobbers) {
// should both be clobbered by the lifetime.start call if they're dominated by
// it.
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
F = Function::Create(FunctionType::get(B.getVoidTy(), {}, false),
GlobalValue::ExternalLinkage, "F", &M);
@@ -1188,7 +1188,7 @@ TEST_F(MemorySSATest, LifetimeMarkersAreClobbers) {
}
TEST_F(MemorySSATest, DefOptimizationsAreInvalidatedOnMoving) {
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
F = Function::Create(FunctionType::get(B.getVoidTy(), {B.getInt1Ty()}, false),
GlobalValue::ExternalLinkage, "F", &M);
diff --git a/llvm/unittests/Analysis/SparsePropagation.cpp b/llvm/unittests/Analysis/SparsePropagation.cpp
index e302f89c5719f..872620fa42145 100644
--- a/llvm/unittests/Analysis/SparsePropagation.cpp
+++ b/llvm/unittests/Analysis/SparsePropagation.cpp
@@ -232,8 +232,7 @@ class SparsePropagationTest : public testing::Test {
SparseSolver<TestLatticeKey, TestLatticeVal> Solver;
public:
- SparsePropagationTest()
- : M("", Context), Builder(Context), Solver(&Lattice) {}
+ SparsePropagationTest() : M("", Context), Builder(M), Solver(&Lattice) {}
};
} // namespace
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index 96576875bd7a5..955fa15f6bc62 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -2980,7 +2980,7 @@ TEST_F(ComputeKnownBitsTest, ComputeKnownBitsAddWithRange) {
TEST_F(ComputeKnownBitsTest, ComputeKnownBitsUnknownVScale) {
Module M("", Context);
- IRBuilder<> Builder(Context);
+ IRBuilder<> Builder(M);
Function *TheFn = Intrinsic::getOrInsertDeclaration(&M, Intrinsic::vscale,
{Builder.getInt32Ty()});
CallInst *CI = Builder.CreateCall(TheFn, {});
diff --git a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
index d10ea6c92c8cb..3c005181d9036 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
@@ -129,7 +129,7 @@ static Function *createRetFunction(Module *M, StringRef Name,
GlobalValue::ExternalLinkage, Name, M);
BasicBlock *BB = BasicBlock::Create(M->getContext(), Name, Result);
- IRBuilder<> Builder(M->getContext());
+ IRBuilder<> Builder(*M);
Builder.SetInsertPoint(BB);
Value *RetValue = ConstantInt::get(M->getContext(), APInt(32, ReturnCode));
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index 14a3065b989ad..063ec08c25913 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -1267,7 +1267,7 @@ TEST_F(IRBuilderTest, DebugLoc) {
auto Br = UncondBrInst::Create(BB2, BB);
Br->setDebugLoc(DL1);
- IRBuilder<> Builder(Ctx);
+ IRBuilder<> Builder(*M);
Builder.SetInsertPoint(Br);
EXPECT_EQ(DL1, Builder.getCurrentDebugLocation());
auto Call1 = Builder.CreateCall(Callee, {});
@@ -1384,10 +1384,10 @@ TEST_F(IRBuilderTest, CTAD) {
};
InstSimplifyFolder Folder(M->getDataLayout());
- IRBuilder Builder1(Ctx, Folder, TestInserter());
+ IRBuilder Builder1(*M, Folder, TestInserter());
static_assert(std::is_same_v<decltype(Builder1),
IRBuilder<InstSimplifyFolder, TestInserter>>);
- IRBuilder Builder2(Ctx);
+ IRBuilder Builder2(*M);
static_assert(std::is_same_v<decltype(Builder2), IRBuilder<>>);
IRBuilder Builder3(BB, Folder);
static_assert(
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index c5a24c1b688a6..fdf46a66afca2 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -543,7 +543,8 @@ TEST(InstructionsTest, VectorGep) {
TEST(InstructionsTest, FPMathOperator) {
LLVMContext Context;
- IRBuilder<> Builder(Context);
+ Module M("", Context);
+ IRBuilder<> Builder(M);
MDBuilder MDHelper(Context);
Instruction *I = Builder.CreatePHI(Builder.getDoubleTy(), 0);
MDNode *MD1 = MDHelper.createFPMath(1.0);
@@ -565,8 +566,7 @@ TEST(InstructionTest, ConstrainedTrans) {
false);
auto *F = Function::Create(FTy, Function::ExternalLinkage, "", M.get());
auto *BB = BasicBlock::Create(Context, "bb", F);
- IRBuilder<> Builder(Context);
- Builder.SetInsertPoint(BB);
+ IRBuilder<> Builder(BB);
auto *Arg0 = F->arg_begin();
auto *Arg1 = F->arg_begin() + 1;
@@ -880,7 +880,7 @@ TEST_F(ModuleWithFunctionTest, DropPoisonGeneratingFlags) {
auto *OnlyBB = BasicBlock::Create(Ctx, "bb", F);
auto *Arg0 = &*F->arg_begin();
- IRBuilder<NoFolder> B(Ctx);
+ IRBuilder<NoFolder> B(*M);
B.SetInsertPoint(OnlyBB);
{
@@ -1629,7 +1629,8 @@ TEST(InstructionsTest, SkipDebug) {
TEST(InstructionsTest, PhiMightNotBeFPMathOperator) {
LLVMContext Context;
- IRBuilder<> Builder(Context);
+ Module M("", Context);
+ IRBuilder<> Builder(M);
MDBuilder MDHelper(Context);
Instruction *I = Builder.CreatePHI(Builder.getInt32Ty(), 0);
EXPECT_FALSE(isa<FPMathOperator>(I));
@@ -1800,7 +1801,8 @@ if.end:
TEST(InstructionsTest, UnaryOperator) {
LLVMContext Context;
- IRBuilder<> Builder(Context);
+ Module M("", Context);
+ IRBuilder<> Builder(M);
Instruction *I = Builder.CreatePHI(Builder.getDoubleTy(), 0);
Value *F = Builder.CreateFNeg(I);
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 30318b02007ab..bc5ffd27a3af8 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -476,7 +476,7 @@ class CloneFunc : public ::testing::Test {
void CreateOldFunctionBodyAndDI() {
DIBuilder DBuilder(*M);
- IRBuilder<> IBuilder(C);
+ IRBuilder<> IBuilder(*M);
// Function DI
auto *File = DBuilder.createFile("filename.c", "/file/dir/");
@@ -1115,7 +1115,7 @@ class CloneModule : public ::testing::Test {
}
DIBuilder DBuilder(*OldM);
- IRBuilder<> IBuilder(C);
+ IRBuilder<> IBuilder(*OldM);
auto *FuncType = FunctionType::get(Type::getVoidTy(C), false);
auto *PersFn = Function::Create(FuncType, GlobalValue::ExternalLinkage,
diff --git a/llvm/unittests/Transforms/Utils/FunctionComparatorTest.cpp b/llvm/unittests/Transforms/Utils/FunctionComparatorTest.cpp
index cd2b4e8046b2e..6606b59ff2d04 100644
--- a/llvm/unittests/Transforms/Utils/FunctionComparatorTest.cpp
+++ b/llvm/unittests/Transforms/Utils/FunctionComparatorTest.cpp
@@ -24,7 +24,7 @@ struct TestFunction {
Type *T;
TestFunction(LLVMContext &Ctx, Module &M, int addVal) {
- IRBuilder<> B(Ctx);
+ IRBuilder<> B(M);
T = B.getInt8Ty();
F = Function::Create(FunctionType::get(T, {B.getPtrTy()}, false),
GlobalValue::ExternalLinkage, "F", &M);
diff --git a/llvm/unittests/Transforms/Utils/IntegerDivisionTest.cpp b/llvm/unittests/Transforms/Utils/IntegerDivisionTest.cpp
index 85dae05c35b5a..7a39f60972234 100644
--- a/llvm/unittests/Transforms/Utils/IntegerDivisionTest.cpp
+++ b/llvm/unittests/Transforms/Utils/IntegerDivisionTest.cpp
@@ -22,7 +22,7 @@ namespace {
TEST(IntegerDivision, SDiv) {
LLVMContext C;
Module M("test division", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt32Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt32Ty(),
@@ -52,7 +52,7 @@ TEST(IntegerDivision, SDiv) {
TEST(IntegerDivision, UDiv) {
LLVMContext C;
Module M("test division", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt32Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt32Ty(),
@@ -82,7 +82,7 @@ TEST(IntegerDivision, UDiv) {
TEST(IntegerDivision, SRem) {
LLVMContext C;
Module M("test remainder", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt32Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt32Ty(),
@@ -112,7 +112,7 @@ TEST(IntegerDivision, SRem) {
TEST(IntegerDivision, URem) {
LLVMContext C;
Module M("test remainder", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt32Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt32Ty(),
@@ -143,7 +143,7 @@ TEST(IntegerDivision, URem) {
TEST(IntegerDivision, SDiv64) {
LLVMContext C;
Module M("test division", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt64Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt64Ty(),
@@ -173,7 +173,7 @@ TEST(IntegerDivision, SDiv64) {
TEST(IntegerDivision, UDiv64) {
LLVMContext C;
Module M("test division", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt64Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt64Ty(),
@@ -203,7 +203,7 @@ TEST(IntegerDivision, UDiv64) {
TEST(IntegerDivision, SRem64) {
LLVMContext C;
Module M("test remainder", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt64Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt64Ty(),
@@ -233,7 +233,7 @@ TEST(IntegerDivision, SRem64) {
TEST(IntegerDivision, URem64) {
LLVMContext C;
Module M("test remainder", C);
- IRBuilder<> Builder(C);
+ IRBuilder<> Builder(M);
SmallVector<Type*, 2> ArgTys(2, Builder.getInt64Ty());
Function *F = Function::Create(FunctionType::get(Builder.getInt64Ty(),
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index a497f75c00896..81c3f20f992dc 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -30,8 +30,9 @@ using namespace llvm;
TEST(Local, RecursivelyDeleteDeadPHINodes) {
LLVMContext C;
+ Module M("", C);
- IRBuilder<> builder(C);
+ IRBuilder<> builder(M);
// Make blocks
BasicBlock *bb0 = BasicBlock::Create(C);
@@ -73,7 +74,8 @@ TEST(Local, RecursivelyDeleteDeadPHINodes) {
TEST(Local, RemoveDuplicatePHINodes) {
LLVMContext C;
- IRBuilder<> B(C);
+ Module M("", C);
+ IRBuilder<> B(M);
std::unique_ptr<Function> F(
Function::Create(FunctionType::get(B.getVoidTy(), false),
@@ -1134,7 +1136,7 @@ TEST(LocalTest, TargetTypeInfoHasNoReplacementProperty) {
TEST(Local, CanReplaceOperandWithVariable) {
LLVMContext Ctx;
Module M("test_module", Ctx);
- IRBuilder<> B(Ctx);
+ IRBuilder<> B(M);
FunctionType *FnType =
FunctionType::get(Type::getVoidTy(Ctx), {}, false);
@@ -1253,7 +1255,7 @@ TEST(Local, ExpressionForConstant) {
EXPECT_EQ(Expr->getElement(1), 0x7FFFFFFFFFFFFFFFU);
GlobalVariable *String =
- IRBuilder<>(Context).CreateGlobalString("hello", "hello", 0, &M);
+ IRBuilder<>(M).CreateGlobalString("hello", "hello", 0, &M);
Expr = createExpression(ConstantExpr::getPtrToInt(String, Int32Ty), Int32Ty);
EXPECT_EQ(Expr, nullptr);
diff --git a/llvm/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp b/llvm/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
index 835e5930ed009..6f494d4067459 100644
--- a/llvm/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
+++ b/llvm/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
@@ -23,7 +23,7 @@ TEST(SSAUpdaterBulk, SimpleMerge) {
SSAUpdaterBulk Updater;
LLVMContext C;
Module M("SSAUpdaterTest", C);
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
Type *I32Ty = B.getInt32Ty();
auto *F = Function::Create(FunctionType::get(B.getVoidTy(), {I32Ty}, false),
GlobalValue::ExternalLinkage, "F", &M);
@@ -112,7 +112,7 @@ TEST(SSAUpdaterBulk, Irreducible) {
SSAUpdaterBulk Updater;
LLVMContext C;
Module M("SSAUpdaterTest", C);
- IRBuilder<> B(C);
+ IRBuilder<> B(M);
Type *I32Ty = B.getInt32Ty();
auto *F = Function::Create(FunctionType::get(B.getVoidTy(), {I32Ty}, false),
GlobalValue::ExternalLinkage, "F", &M);
diff --git a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
index 27458f9d51e90..c8f4e81ddf07b 100644
--- a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -143,7 +143,7 @@ static std::string makePackedFunctionName(StringRef name) {
// pointer to provide a unified invocation interface.
static void packFunctionArguments(Module *module) {
auto &ctx = module->getContext();
- llvm::IRBuilder<> builder(ctx);
+ llvm::IRBuilder<> builder(*module);
DenseSet<llvm::Function *> interfaceFunctions;
for (auto &func : module->getFunctionList()) {
if (func.isDeclaration() || func.hasLocalLinkage())
diff --git a/mlir/lib/Target/LLVMIR/Dialect/GPU/SelectObjectAttr.cpp b/mlir/lib/Target/LLVMIR/Dialect/GPU/SelectObjectAttr.cpp
index 016159391c33f..d40a2789a9feb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/GPU/SelectObjectAttr.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/GPU/SelectObjectAttr.cpp
@@ -122,7 +122,7 @@ static LogicalResult embedBinaryImpl(StringRef moduleName,
optLevel = optAttr.getValue();
}
- IRBuilder<> builder(module.getContext());
+ IRBuilder<> builder(module);
auto *i32Ty = builder.getInt32Ty();
auto *i64Ty = builder.getInt64Ty();
auto *ptrTy = builder.getPtrTy(0);
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 9f992805dc900..5b1bbcd3bdc31 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -1386,8 +1386,7 @@ LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
if (Block *initializer = op.getInitializerBlock()) {
llvm::IRBuilder<llvm::TargetFolder> builder(
- llvmModule->getContext(),
- llvm::TargetFolder(llvmModule->getDataLayout()));
+ *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
[[maybe_unused]] int numConstantsHit = 0;
[[maybe_unused]] int numConstantsErased = 0;
@@ -1478,8 +1477,7 @@ LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
if ((ctorOp && ctorOp.getCtors().empty()) ||
(dtorOp && dtorOp.getDtors().empty())) {
llvm::IRBuilder<llvm::TargetFolder> builder(
- llvmModule->getContext(),
- llvm::TargetFolder(llvmModule->getDataLayout()));
+ *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
llvm::Type *eltTy = llvm::StructType::get(
builder.getInt32Ty(), builder.getPtrTy(), builder.getPtrTy());
llvm::ArrayType *at = llvm::ArrayType::get(eltTy, 0);
@@ -1522,8 +1520,7 @@ LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
Block &initializer = op.getInitializerBlock();
llvm::IRBuilder<llvm::TargetFolder> builder(
- llvmModule->getContext(),
- llvm::TargetFolder(llvmModule->getDataLayout()));
+ *llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
for (mlir::Operation &op : initializer.without_terminator()) {
if (failed(convertOperation(op, builder)))
@@ -1780,7 +1777,7 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
// converted before uses.
auto blocks = getBlocksSortedByDominance(func.getBody());
for (Block *bb : blocks) {
- CapturingIRBuilder builder(llvmContext,
+ CapturingIRBuilder builder(*llvmModule,
llvm::TargetFolder(llvmModule->getDataLayout()));
if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder,
/*recordInsertions=*/true)))
@@ -2740,7 +2737,7 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
ModuleTranslation translator(module, std::move(llvmModule), fs);
llvm::IRBuilder<llvm::TargetFolder> llvmBuilder(
- llvmContext,
+ *translator.getLLVMModule(),
llvm::TargetFolder(translator.getLLVMModule()->getDataLayout()));
// Convert module before functions and operations inside, so dialect
diff --git a/polly/lib/CodeGen/PerfMonitor.cpp b/polly/lib/CodeGen/PerfMonitor.cpp
index 7b559e990b7c0..b3e3b6f1d00e7 100644
--- a/polly/lib/CodeGen/PerfMonitor.cpp
+++ b/polly/lib/CodeGen/PerfMonitor.cpp
@@ -62,8 +62,7 @@ Function *PerfMonitor::getRDTSCP() {
return Intrinsic::getOrInsertDeclaration(M, Intrinsic::x86_rdtscp);
}
-PerfMonitor::PerfMonitor(const Scop &S, Module *M)
- : M(M), Builder(M->getContext()), S(S) {
+PerfMonitor::PerfMonitor(const Scop &S, Module *M) : M(M), Builder(*M), S(S) {
if (M->getTargetTriple().getArch() == llvm::Triple::x86_64)
Supported = true;
else
>From 16792e00e652c4eba68ed02cc3b69ec5b8a9f311 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Wed, 7 Oct 2026 10:25:56 +0200
Subject: [PATCH 13/52] [OCaml] Remove unused open Llvm_target in test (NFC)
(#229680)
Reported at:
https://github.com/llvm/llvm-project/pull/227714#issuecomment-6026833038
---
llvm/test/Bindings/OCaml/executionengine.ml | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/test/Bindings/OCaml/executionengine.ml b/llvm/test/Bindings/OCaml/executionengine.ml
index 0835b229532a9..06ddda7f81207 100644
--- a/llvm/test/Bindings/OCaml/executionengine.ml
+++ b/llvm/test/Bindings/OCaml/executionengine.ml
@@ -9,7 +9,6 @@
open Llvm
open Llvm_executionengine
-open Llvm_target
(* Note that this takes a moment to link, so it's best to keep the number of
individual tests low. *)
>From 389a69b468631086349b03ff3abc1cff9df71db0 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Wed, 7 Oct 2026 01:28:09 -0700
Subject: [PATCH 14/52] [Analysis] Remove unused CallGraphSCC methods (NFC)
(#229683)
The last callers of CallGraphSCC::ReplaceNode and
CallGraphSCC::DeleteNode were removed on July 12, 2024 in commit
58bc98cd3abd72226cdbaa05bd92af9598d491db.
Their removal leaves the private member variable Context unused, so this
patch also removes Context and simplifies the CallGraphSCC constructor.
Assisted-by: Antigravity
---
llvm/include/llvm/Analysis/CallGraphSCCPass.h | 11 +------
llvm/lib/Analysis/CallGraphSCCPass.cpp | 30 +------------------
2 files changed, 2 insertions(+), 39 deletions(-)
diff --git a/llvm/include/llvm/Analysis/CallGraphSCCPass.h b/llvm/include/llvm/Analysis/CallGraphSCCPass.h
index 4ef38cbbcbd64..c620c08859055 100644
--- a/llvm/include/llvm/Analysis/CallGraphSCCPass.h
+++ b/llvm/include/llvm/Analysis/CallGraphSCCPass.h
@@ -82,11 +82,10 @@ class LLVM_ABI CallGraphSCCPass : public Pass {
/// CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
class CallGraphSCC {
const CallGraph &CG; // The call graph for this SCC.
- void *Context; // The CGPassManager object that is vending this.
std::vector<CallGraphNode *> Nodes;
public:
- CallGraphSCC(CallGraph &cg, void *context) : CG(cg), Context(context) {}
+ CallGraphSCC(CallGraph &cg) : CG(cg) {}
void initialize(ArrayRef<CallGraphNode *> NewNodes) {
Nodes.assign(NewNodes.begin(), NewNodes.end());
@@ -95,14 +94,6 @@ class CallGraphSCC {
bool isSingular() const { return Nodes.size() == 1; }
unsigned size() const { return Nodes.size(); }
- /// ReplaceNode - This informs the SCC and the pass manager that the specified
- /// Old node has been deleted, and New is to be used in its place.
- LLVM_ABI void ReplaceNode(CallGraphNode *Old, CallGraphNode *New);
-
- /// DeleteNode - This informs the SCC and the pass manager that the specified
- /// Old node has been deleted.
- LLVM_ABI void DeleteNode(CallGraphNode *Old);
-
using iterator = std::vector<CallGraphNode *>::const_iterator;
iterator begin() const { return Nodes.begin(); }
diff --git a/llvm/lib/Analysis/CallGraphSCCPass.cpp b/llvm/lib/Analysis/CallGraphSCCPass.cpp
index b27a7c5e388c2..41e33ef2300ac 100644
--- a/llvm/lib/Analysis/CallGraphSCCPass.cpp
+++ b/llvm/lib/Analysis/CallGraphSCCPass.cpp
@@ -503,7 +503,7 @@ bool CGPassManager::runOnModule(Module &M) {
// Walk the callgraph in bottom-up SCC order.
scc_iterator<CallGraph*> CGI = scc_begin(&CG);
- CallGraphSCC CurSCC(CG, &CGI);
+ CallGraphSCC CurSCC(CG);
while (!CGI.isAtEnd()) {
// Copy the current SCC and increment past it so that the pass can hack
// on the SCC if it wants to without invalidating our iterator.
@@ -574,34 +574,6 @@ bool CGPassManager::doFinalization(CallGraph &CG) {
return Changed;
}
-//===----------------------------------------------------------------------===//
-// CallGraphSCC Implementation
-//===----------------------------------------------------------------------===//
-
-/// This informs the SCC and the pass manager that the specified
-/// Old node has been deleted, and New is to be used in its place.
-void CallGraphSCC::ReplaceNode(CallGraphNode *Old, CallGraphNode *New) {
- assert(Old != New && "Should not replace node with self");
- for (unsigned i = 0; ; ++i) {
- assert(i != Nodes.size() && "Node not in SCC");
- if (Nodes[i] != Old) continue;
- if (New)
- Nodes[i] = New;
- else
- Nodes.erase(Nodes.begin() + i);
- break;
- }
-
- // Update the active scc_iterator so that it doesn't contain dangling
- // pointers to the old CallGraphNode.
- scc_iterator<CallGraph*> *CGI = (scc_iterator<CallGraph*>*)Context;
- CGI->ReplaceNode(Old, New);
-}
-
-void CallGraphSCC::DeleteNode(CallGraphNode *Old) {
- ReplaceNode(Old, /*New=*/nullptr);
-}
-
//===----------------------------------------------------------------------===//
// CallGraphSCCPass Implementation
//===----------------------------------------------------------------------===//
>From e128a79c81bd4e0274eaf0cae3a318f715882746 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 7 Oct 2026 09:29:01 +0100
Subject: [PATCH 15/52] [AArch64] Add predicate-as-counter loop rewrite pass
(#220960)
This patch adds an AArch64 IR loop pass that rewrites wide loop-carried
`llvm.get.active.lane.mask` phis to predicate-as-counter `whilelo` phis
(for SVE2.1 or streaming SME2 targets).
Users of the original mask are preserved by materializing vector
predicates with `aarch64.sve.pext`.
The element size and vector scale (VLx2 or VLx4) of the
predicate-as-counter is inferred from the mask load/store users within
the loop. These could be optimized to multi-vector loads/stores (though
that is not included in this patch).
For example, a loop like:
```
entry:
%step = vscale x 64
%mask.entry = @get.active.lane.mask(0, %n)
loop:
%iv = phi i64 [0, entry], [%iv.next, loop]
%mask = phi <vscale x 64 x i1> [%mask.entry, entry],
[%mask.next, loop]
...
%iv.next = %iv + %step
%mask.next = @get.active.lane.mask(%iv.next, %n)
br first.active(%mask.next), loop, exit
```
Could be rewritten to:
```
entry:
%step = vscale x 64
%mask.entry = @whilelo.c8(0, %n, VLx4)
loop:
%iv = phi i64 [0, entry], [%iv.next, loop]
%mask = phi target("aarch64.svcount") [%mask.entry, entry],
[%mask.next, loop]
...
%iv.next = %iv + %step
%mask.next = @whilelo.c8(%iv.next, %n, VLx4)
br first.active(@pext(%mask.next, 0)), loop, exit
```
Assisted-by: Codex
---
llvm/lib/Target/AArch64/AArch64.h | 2 +
.../AArch64PredicateAsCounterLoopRewrites.cpp | 430 +++++++++++
.../Target/AArch64/AArch64TargetMachine.cpp | 11 +
llvm/lib/Target/AArch64/CMakeLists.txt | 1 +
.../predicate-as-counter-loop-rewrites.ll | 675 ++++++++++++++++++
5 files changed, 1119 insertions(+)
create mode 100644 llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
create mode 100644 llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
diff --git a/llvm/lib/Target/AArch64/AArch64.h b/llvm/lib/Target/AArch64/AArch64.h
index da05342040697..d267126035aac 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -76,6 +76,7 @@ FunctionPass *createAArch64PTrueCoalescingLegacyPass();
FunctionPass *createAArch64CleanupLocalDynamicTLSPass();
FunctionPass *createAArch64CollectLOHPass();
+Pass *createAArch64PredicateAsCounterLoopRewritesPass();
FunctionPass *createSMEPeepholeOptPass();
FunctionPass *createMachineSMEABIPass(CodeGenOptLevel);
FunctionPass *createAArch64SRLTDefineSuperRegsLegacyPass();
@@ -167,6 +168,7 @@ void initializeAArch64A57FPLoadBalancingLegacyPass(PassRegistry &);
void initializeAArch64AdvSIMDScalarLegacyPass(PassRegistry &);
void initializeAArch64AsmPrinterPass(PassRegistry &);
void initializeAArch64PointerAuthLegacyPass(PassRegistry &);
+void initializeAArch64PredicateAsCounterLoopRewritesPass(PassRegistry &);
void initializeAArch64BranchTargetsLegacyPass(PassRegistry &);
void initializeAArch64CollectLOHLegacyPass(PassRegistry &);
void initializeAArch64CompressJumpTablesLegacyPass(PassRegistry &);
diff --git a/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
new file mode 100644
index 0000000000000..c52c09e71aa89
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64PredicateAsCounterLoopRewrites.cpp
@@ -0,0 +1,430 @@
+//===- AArch64PredicateAsCounterLoopRewrites.cpp --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
+// Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Rewrites IR for loop-carried wide masks that can be represented as
+// predicate-as-counter values. This applies when the mask is used by load/store
+// operations that can be mapped to multi-vector instructions (with +sve2p1).
+//
+// For example, a loop like:
+//
+// entry:
+// %step = vscale x 64
+// %mask.entry = @get.active.lane.mask(0, %n)
+//
+// loop:
+// %iv = phi i64 [0, entry], [%iv.next, loop]
+// %mask = phi <vscale x 64 x i1> [%mask.entry, entry],
+// [%mask.next, loop]
+//
+// %load = load <vscale x 64 x i8> %src[%iv], %mask
+// store <vscale x 64 x i8> %load, %dst[%iv], %mask
+//
+// %iv.next = %iv + %step
+// %mask.next = @get.active.lane.mask(%iv.next, %n)
+// br first.active(%mask.next), loop, exit
+//
+// Could be rewritten to:
+//
+// entry:
+// %step = vscale x 64
+// %mask.entry = @whilelo.c8(0, %n, VLx4)
+//
+// loop:
+// %iv = phi i64 [0, entry], [%iv.next, loop]
+// %mask = phi target("aarch64.svcount") [%mask.entry, entry],
+// [%mask.next, loop]
+//
+// %load = @ld1.pn.x4 <4 x <vscale x 16 x i8>> %src[%iv], %mask
+// @st1.pn.x4 <4 x <vscale x 16 x i8>> %load, %dest[%iv], %mask
+//
+// %iv.next = %iv + %step
+// %mask.next = @whilelo.c8(%iv.next, %n, VLx4)
+// br first.active(@pext(%mask.next, 0)), loop, exit
+//
+// This replaces the `get.active.lane.mask` intrinsics with AArch64
+// predicate-as-counter `whilelo` intrinsics and updates the mask phi to use the
+// `aarch64.svcount` target type. Within the loop, load/store users are mapped
+// to multi-vector load/store intrinsics where possible. Users that cannot be
+// mapped to multi-vector instructions materialize vector masks using the `pext`
+// intrinsic (which extracts vector predicates from a predicate-as-counter).
+//
+// This pass may be a temporary solution that is removed if we gain support
+// for target-specific VPlan transforms in the loop vectorizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64.h"
+#include "AArch64Subtarget.h"
+#include "AArch64TargetMachine.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include <optional>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "aarch64-pn-loop-rewrites"
+namespace {
+
+STATISTIC(LoopsRewritten, "Number of loops rewritten");
+
+struct MaskRewriteCandidate {
+ /// The mask phi node (used by masked operations within the loop).
+ PHINode *MaskPhi = nullptr;
+ /// The initial value for the mask (incoming value from the preheader).
+ IntrinsicInst *StartMask = nullptr;
+ /// The updated value for the mask (incoming value from the loop latch).
+ IntrinsicInst *NextMask = nullptr;
+ /// The multi-vector scale for the predicate-as-counter (2 or 4).
+ unsigned VectorScale = 0;
+ /// The element size (in bits) for the predicate-as-counter.
+ unsigned ElementSizeInBits = 0;
+};
+
+static void logLoopBailout(const Loop &L, const Twine &Reason) {
+ LLVM_DEBUG({
+ dbgs() << "PN loop rewrite: skipping loop with header ";
+ L.getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
+ dbgs() << ": " << Reason << "\n";
+ });
+}
+
+static void logMatchFailure(const PHINode &Phi, const Twine &Reason) {
+ LLVM_DEBUG({
+ dbgs() << "PN loop rewrite: failed to match mask phi ";
+ Phi.printAsOperand(dbgs(), /*PrintType=*/false);
+ dbgs() << ": " << Reason << "\n";
+ });
+}
+
+/// Returns the scalar size in bits for a type according to the data layout.
+static unsigned getScalarSizeInBits(const DataLayout &DL, Type *Ty) {
+ return DL.getTypeSizeInBits(Ty->getScalarType()).getFixedValue();
+}
+
+/// Returns the SVE element count for \p ElementSizeInBits.
+static ElementCount getSVEElementCount(unsigned ElementSizeInBits) {
+ return ElementCount::getScalable(AArch64::SVEBitsPerBlock /
+ ElementSizeInBits);
+}
+
+/// Returns the largest scalar access size of masked loads/stores in loop \p L
+/// where \p MaskPhi is used as the mask. TODO: This heuristic may need
+/// refinement based on the frequency of different access sizes.
+static unsigned getLargestMaskedMemAccessSizeInBits(const Loop &L,
+ PHINode &MaskPhi) {
+ const DataLayout &DL = MaskPhi.getModule()->getDataLayout();
+ unsigned LargestAccessSizeInBits = 0;
+
+ for (User *U : MaskPhi.users()) {
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ if (!II || !L.contains(II))
+ continue;
+
+ Intrinsic::ID IID = II->getIntrinsicID();
+ if (IID != Intrinsic::masked_load && IID != Intrinsic::masked_store)
+ continue;
+
+ unsigned MaskOpIdx = IID == Intrinsic::masked_load ? 1 : 2;
+ if (II->getArgOperand(MaskOpIdx) != &MaskPhi)
+ continue;
+
+ unsigned AccessSizeInBits = getScalarSizeInBits(DL, II->getAccessType());
+ if (AccessSizeInBits > LargestAccessSizeInBits)
+ LargestAccessSizeInBits = AccessSizeInBits;
+ }
+
+ return LargestAccessSizeInBits;
+}
+
+/// Returns the predicate-as-counter whilelo intrinsic ID for
+/// \p ElementSizeInBits.
+static Intrinsic::ID getWhileLOIntrinsic(unsigned ElementSizeInBits) {
+ switch (ElementSizeInBits) {
+ case 8:
+ return Intrinsic::aarch64_sve_whilelo_c8;
+ case 16:
+ return Intrinsic::aarch64_sve_whilelo_c16;
+ case 32:
+ return Intrinsic::aarch64_sve_whilelo_c32;
+ case 64:
+ return Intrinsic::aarch64_sve_whilelo_c64;
+ default:
+ llvm_unreachable("unsupported predicate-as-counter element size");
+ }
+}
+
+/// Expands the predicate-as-counter mask \p Count into a wide vector mask.
+/// Masks are extracted from the counter using the paired pext intrinsics,
+/// then concatenated to form the wide mask value.
+static Value *buildWideMask(IRBuilder<> &Builder, const MaskRewriteCandidate &C,
+ Value *Count) {
+ ElementCount LegalEC = getSVEElementCount(C.ElementSizeInBits);
+ Type *LegalMaskTy = VectorType::get(Builder.getInt1Ty(), LegalEC);
+
+ Value *WideMask = PoisonValue::get(C.MaskPhi->getType());
+ for (unsigned PairOffset = 0; PairOffset != C.VectorScale / 2; ++PairOffset) {
+ auto *Pair =
+ Builder.CreateIntrinsic(Intrinsic::aarch64_sve_pext_x2, {LegalMaskTy},
+ {Count, Builder.getInt32(PairOffset)},
+ /*FMFSource=*/{}, "pn.pext.pair");
+ for (unsigned SliceInPair = 0; SliceInPair != 2; ++SliceInPair) {
+ Value *Part = Builder.CreateExtractValue(Pair, SliceInPair, "pn.pext");
+ unsigned Slice = PairOffset * 2 + SliceInPair;
+ WideMask = Builder.CreateInsertVector(
+ C.MaskPhi->getType(), WideMask, Part,
+ Slice * LegalEC.getKnownMinValue(), "pn.mask");
+ }
+ }
+
+ return WideMask;
+}
+
+/// Creates a predicate-as-counter whilelo for \p ElementSizeInBits between
+/// \p Start and \p End with multi-vector \p VectorScale (2 or 4).
+static Value *createWhileLO(IRBuilder<> &Builder, unsigned ElementSizeInBits,
+ Value *Start, Value *End, unsigned VectorScale) {
+ if (Start->getType()->getIntegerBitWidth() < 64) {
+ Start = Builder.CreateZExt(Start, Builder.getInt64Ty());
+ End = Builder.CreateZExt(End, Builder.getInt64Ty());
+ }
+
+ Intrinsic::ID WhileLO = getWhileLOIntrinsic(ElementSizeInBits);
+ return Builder.CreateIntrinsic(WhileLO,
+ {Start, End, Builder.getInt32(VectorScale)},
+ /*FMFSource=*/{}, "pn.mask");
+}
+
+class AArch64PredicateAsCounterLoopRewrites : public LoopPass {
+public:
+ static char ID;
+
+ AArch64PredicateAsCounterLoopRewrites() : LoopPass(ID) {}
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetPassConfig>();
+ // Require loop simplify to ensure loops have a preheader.
+ AU.addRequiredID(LoopSimplifyID);
+ AU.addPreservedID(LoopSimplifyID);
+ AU.setPreservesCFG();
+ }
+
+ bool runOnLoop(Loop *L, LPPassManager &) override;
+
+private:
+ std::optional<MaskRewriteCandidate> matchMaskPhi(Loop &L, PHINode &Phi) const;
+ bool rewriteCandidate(const MaskRewriteCandidate &C, Loop &L) const;
+};
+
+} // end anonymous namespace
+
+char AArch64PredicateAsCounterLoopRewrites::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AArch64PredicateAsCounterLoopRewrites, DEBUG_TYPE,
+ "AArch64 Predicate As Counter Loop Rewrites", false,
+ false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_END(AArch64PredicateAsCounterLoopRewrites, DEBUG_TYPE,
+ "AArch64 Predicate As Counter Loop Rewrites", false, false)
+
+Pass *llvm::createAArch64PredicateAsCounterLoopRewritesPass() {
+ return new AArch64PredicateAsCounterLoopRewrites();
+}
+
+bool AArch64PredicateAsCounterLoopRewrites::runOnLoop(Loop *L,
+ LPPassManager &) {
+ if (skipLoop(L)) {
+ logLoopBailout(*L, "skipLoop requested the loop to be skipped");
+ return false;
+ }
+
+ Function &F = *L->getHeader()->getParent();
+ auto &TPC = getAnalysis<TargetPassConfig>();
+ const AArch64Subtarget *ST =
+ TPC.getTM<AArch64TargetMachine>().getSubtargetImpl(F);
+ if (!ST->hasSVE2p1() && !(ST->hasSME2() && ST->isStreaming())) {
+ logLoopBailout(*L, "neither SVE2.1 nor SME2 is available");
+ return false;
+ }
+
+ if (!L->getLoopPreheader()) {
+ logLoopBailout(*L, "loop has no preheader");
+ return false;
+ }
+ if (!L->getLoopLatch()) {
+ logLoopBailout(*L, "loop has no latch");
+ return false;
+ }
+
+ bool Changed = false;
+ BasicBlock *Header = L->getHeader();
+ for (PHINode &Phi : make_early_inc_range(Header->phis())) {
+ if (std::optional<MaskRewriteCandidate> Candidate = matchMaskPhi(*L, Phi))
+ Changed |= rewriteCandidate(*Candidate, *L);
+ }
+
+ if (Changed)
+ ++LoopsRewritten;
+
+ return Changed;
+}
+
+static IntrinsicInst *getGetActiveLaneMask(Value *V) {
+ auto *II = dyn_cast<IntrinsicInst>(V);
+ return II && II->getIntrinsicID() == Intrinsic::get_active_lane_mask
+ ? II
+ : nullptr;
+}
+
+std::optional<MaskRewriteCandidate>
+AArch64PredicateAsCounterLoopRewrites::matchMaskPhi(Loop &L,
+ PHINode &Phi) const {
+ auto *PhiTy = dyn_cast<ScalableVectorType>(Phi.getType());
+ if (!PhiTy || !PhiTy->getElementType()->isIntegerTy(1))
+ return std::nullopt;
+
+ if (Phi.getNumIncomingValues() != 2) {
+ logMatchFailure(Phi, Twine("phi has ") + Twine(Phi.getNumIncomingValues()) +
+ " incoming values; expected 2");
+ return std::nullopt;
+ }
+
+ Value *StartValue = Phi.getIncomingValueForBlock(L.getLoopPreheader());
+ Value *NextValue = Phi.getIncomingValueForBlock(L.getLoopLatch());
+ IntrinsicInst *StartMask = getGetActiveLaneMask(StartValue);
+ IntrinsicInst *NextMask = getGetActiveLaneMask(NextValue);
+ if (!StartMask) {
+ logMatchFailure(Phi,
+ "preheader incoming value is not get_active_lane_mask");
+ return std::nullopt;
+ }
+ if (!NextMask) {
+ logMatchFailure(Phi, "latch incoming value is not get_active_lane_mask");
+ return std::nullopt;
+ }
+
+ unsigned WideMaskElements = PhiTy->getMinNumElements();
+ if (!isPowerOf2_32(WideMaskElements)) {
+ logMatchFailure(Phi,
+ Twine("wide mask element count is not a power of 2: ") +
+ Twine(WideMaskElements));
+ return std::nullopt;
+ }
+
+ if (StartMask->getArgOperand(0)->getType()->getIntegerBitWidth() > 64) {
+ logMatchFailure(Phi, "start mask induction operand is wider than i64");
+ return std::nullopt;
+ }
+ if (NextMask->getArgOperand(0)->getType()->getIntegerBitWidth() > 64) {
+ logMatchFailure(Phi, "next mask induction operand is wider than i64");
+ return std::nullopt;
+ }
+
+ unsigned PreferredMaskElementSizeInBits =
+ getLargestMaskedMemAccessSizeInBits(L, Phi);
+
+ if (!is_contained({8u, 16u, 32u, 64u}, PreferredMaskElementSizeInBits)) {
+ logMatchFailure(Phi, Twine("unsupported element size in bits: ") +
+ Twine(PreferredMaskElementSizeInBits));
+ return std::nullopt;
+ }
+
+ unsigned SVEMaskElements =
+ AArch64::SVEBitsPerBlock / PreferredMaskElementSizeInBits;
+ if (WideMaskElements <= SVEMaskElements) {
+ logMatchFailure(Phi, Twine("wide mask element count ") +
+ Twine(WideMaskElements) +
+ " is not wider than the legal mask width " +
+ Twine(SVEMaskElements));
+ return std::nullopt;
+ }
+
+ unsigned VectorScale = WideMaskElements / SVEMaskElements;
+ if (VectorScale != 2 && VectorScale != 4) {
+ logMatchFailure(Phi, Twine("unsupported predicate-as-counter scale: ") +
+ Twine(VectorScale));
+ return std::nullopt;
+ }
+
+ return MaskRewriteCandidate{&Phi, StartMask, NextMask, VectorScale,
+ PreferredMaskElementSizeInBits};
+}
+
+bool AArch64PredicateAsCounterLoopRewrites::rewriteCandidate(
+ const MaskRewriteCandidate &C, Loop &L) const {
+ IRBuilder<> Builder(C.StartMask);
+ Value *NewStart =
+ createWhileLO(Builder, C.ElementSizeInBits, C.StartMask->getArgOperand(0),
+ C.StartMask->getArgOperand(1), C.VectorScale);
+ Builder.SetInsertPoint(C.NextMask);
+ Value *NewNext =
+ createWhileLO(Builder, C.ElementSizeInBits, C.NextMask->getArgOperand(0),
+ C.NextMask->getArgOperand(1), C.VectorScale);
+
+ Builder.SetInsertPoint(C.MaskPhi);
+ auto *NewPhi =
+ Builder.CreatePHI(NewStart->getType(), 2, C.MaskPhi->getName() + ".pn");
+ NewPhi->addIncoming(NewStart, L.getLoopPreheader());
+ NewPhi->addIncoming(NewNext, L.getLoopLatch());
+
+ auto RewriteUses = [&](Instruction *OldMask, Value *Count,
+ function_ref<bool(Use &U)> Predicate = nullptr) {
+ SmallVector<Use *, 8> UsesToRewrite;
+ for (Use &U : OldMask->uses()) {
+ if (!Predicate || Predicate(U))
+ UsesToRewrite.push_back(&U);
+ }
+
+ Value *WideMask = nullptr;
+ for (Use *U : UsesToRewrite) {
+ if (!WideMask) {
+ BasicBlock::iterator InsertPt = OldMask->getIterator();
+ if (isa<PHINode>(OldMask))
+ InsertPt = OldMask->getParent()->getFirstNonPHIIt();
+
+ Builder.SetInsertPoint(InsertPt);
+ Builder.SetCurrentDebugLocation(OldMask->getDebugLoc());
+ WideMask = buildWideMask(Builder, C, Count);
+ }
+
+ U->set(WideMask);
+ }
+ };
+
+ // For the start/next mask, we only replace non-phi in-loop users. Due to CSE,
+ // these masks could be used by other loops, and replacing them with
+ // predicate-as-counter masks could prevent matching those loops.
+ auto IsNonPhiUseInLoop = [&](Use &U) {
+ auto *UseInst = cast<Instruction>(U.getUser());
+ return !isa<PHINode>(UseInst) && L.contains(UseInst);
+ };
+
+ RewriteUses(C.MaskPhi, NewPhi);
+ RewriteUses(C.StartMask, NewStart, IsNonPhiUseInLoop);
+ RewriteUses(C.NextMask, NewNext, IsNonPhiUseInLoop);
+
+ RecursivelyDeleteTriviallyDeadInstructions(C.MaskPhi);
+ return true;
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 1f1474d38624d..9e783b0a23e67 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -231,6 +231,12 @@ static cl::opt<bool> EnableSVEShuffleOpt(
"instructions like tbl or the bottom/top variants"),
cl::init(true), cl::Hidden);
+static cl::opt<bool> EnablePredicateAsCounterLoopRewrites(
+ "aarch64-enable-predicate-as-counter-loop-rewrites",
+ cl::desc("Enable rewriting loops with wide loop-carried masks to use "
+ "predicate-as-counter"),
+ cl::init(false), cl::Hidden);
+
extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
LLVMInitializeAArch64Target() {
// Register the target.
@@ -258,6 +264,7 @@ LLVMInitializeAArch64Target() {
initializeAArch64PTrueCoalescingLegacyPass(PR);
initializeAArch64SIMDInstrOptLegacyPass(PR);
initializeAArch64O0PreLegalizerCombinerLegacyPass(PR);
+ initializeAArch64PredicateAsCounterLoopRewritesPass(PR);
initializeAArch64PreLegalizerCombinerLegacyPass(PR);
initializeAArch64PointerAuthLegacyPass(PR);
initializeAArch64PostCoalescerLegacyPass(PR);
@@ -660,6 +667,10 @@ void AArch64PassConfig::addIRPasses() {
// ourselves.
addPass(createAtomicExpandLegacyPass());
+ if (getOptLevel() >= CodeGenOptLevel::Default &&
+ EnablePredicateAsCounterLoopRewrites)
+ addPass(createAArch64PredicateAsCounterLoopRewritesPass());
+
// Cmpxchg instructions are often used with a subsequent comparison to
// determine whether it succeeded. We can exploit existing control-flow in
// ldrex/strex loops to simplify this, but it needs tidying up.
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 12a2214f8e58e..5c4740baf79de 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -79,6 +79,7 @@ add_llvm_target(AArch64CodeGen
AArch64PromoteConstant.cpp
AArch64PTrueCoalescing.cpp
AArch64PBQPRegAlloc.cpp
+ AArch64PredicateAsCounterLoopRewrites.cpp
AArch64RegisterInfo.cpp
AArch64SMEAttributes.cpp
AArch64SLSHardening.cpp
diff --git a/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
new file mode 100644
index 0000000000000..b7640444f34c4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/predicate-as-counter-loop-rewrites.ll
@@ -0,0 +1,675 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-enable-predicate-as-counter-loop-rewrites -mattr=+sve2p1 -aarch64-enable-subreg-liveness-tracking < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Tests the baseline i64 masked load/store rewrite to predicate-as-counter with vlx4.
+define void @rewrite_masked_load_store_i64_vlx4(ptr %x, i64 %n) #0 {
+; CHECK-LABEL: rewrite_masked_load_store_i64_vlx4:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
+; CHECK-NEXT: .LBB0_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
+; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
+; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
+; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
+; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: tbnz w9, #0, .LBB0_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr = getelementptr i64, ptr %x, i64 %base
+ %v = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %ptr, i32 8, <vscale x 8 x i1> %update.mask, <vscale x 8 x i64> poison)
+ %add = add <vscale x 8 x i64> %v, splat (i64 1)
+ call void @llvm.masked.store.nxv8i64(<vscale x 8 x i64> %add, ptr %ptr, i32 8, <vscale x 8 x i1> %update.mask)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests rewriting a loop whose header has multiple outside predecessors before LoopSimplify creates a preheader.
+define void @rewrite_masked_load_store_i64_multi_pred_header(ptr %x, i64 %n, i1 %c) #0 {
+; CHECK-LABEL: rewrite_masked_load_store_i64_multi_pred_header:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
+; CHECK-NEXT: .LBB1_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
+; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
+; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
+; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
+; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: tbnz w9, #0, .LBB1_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br i1 %c, label %left, label %right
+
+left:
+ br label %loop
+
+right:
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %left ], [ 0, %right ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %left ], [ %start.mask, %right ], [ %new.mask, %loop ]
+ %ptr = getelementptr i64, ptr %x, i64 %base
+ %v = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %ptr, i32 8, <vscale x 8 x i1> %update.mask, <vscale x 8 x i64> poison)
+ %add = add <vscale x 8 x i64> %v, splat (i64 1)
+ call void @llvm.masked.store.nxv8i64(<vscale x 8 x i64> %add, ptr %ptr, i32 8, <vscale x 8 x i1> %update.mask)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests that a shared start mask remains available so a later loop can also be rewritten.
+define void @shared_start_mask_between_loops(ptr %x, ptr %y, i64 %n) #0 {
+; CHECK-LABEL: shared_start_mask_between_loops:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: whilelo pn8.d, xzr, x2, vlx4
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: mov p9.b, p8.b
+; CHECK-NEXT: mov x9, x8
+; CHECK-NEXT: .LBB2_1: // %loop1
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p2.d, p3.d }, pn9[1]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn9[0]
+; CHECK-NEXT: whilelo pn9.d, x9, x2, vlx4
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
+; CHECK-NEXT: pext { p4.d, p5.d }, pn9[0]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
+; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-NEXT: pext { p5.d, p6.d }, pn9[1]
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: inch x9
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
+; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
+; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: fmov w10, s0
+; CHECK-NEXT: tbnz w10, #0, .LBB2_1
+; CHECK-NEXT: .LBB2_2: // %loop2
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
+; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4
+; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x1, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x1, #2, mul vl]
+; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x1, #1, mul vl]
+; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x1]
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: st1d { z0.d }, p3, [x1, #3, mul vl]
+; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
+; CHECK-NEXT: st1d { z1.d }, p2, [x1, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p1, [x1, #1, mul vl]
+; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
+; CHECK-NEXT: st1d { z3.d }, p0, [x1]
+; CHECK-NEXT: incb x1, all, mul #4
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: tbnz w9, #0, .LBB2_2
+; CHECK-NEXT: // %bb.3: // %exit
+; CHECK-NEXT: ret
+entry:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop1
+
+loop1:
+ %base1 = phi i64 [ 0, %entry ], [ %next.base1, %loop1 ]
+ %mask1 = phi <vscale x 8 x i1> [ %start.mask, %entry ], [ %new.mask1, %loop1 ]
+ %ptr1 = getelementptr i64, ptr %x, i64 %base1
+ %v1 = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %ptr1, i32 8, <vscale x 8 x i1> %mask1, <vscale x 8 x i64> poison)
+ %add1 = add <vscale x 8 x i64> %v1, splat (i64 1)
+ call void @llvm.masked.store.nxv8i64(<vscale x 8 x i64> %add1, ptr %ptr1, i32 8, <vscale x 8 x i1> %mask1)
+ %next.base1 = add i64 %base1, %step
+ %new.mask1 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base1, i64 %n)
+ %more1 = extractelement <vscale x 8 x i1> %new.mask1, i64 0
+ br i1 %more1, label %loop1, label %between
+
+between:
+ br label %loop2
+
+loop2:
+ %base2 = phi i64 [ 0, %between ], [ %next.base2, %loop2 ]
+ %mask2 = phi <vscale x 8 x i1> [ %start.mask, %between ], [ %new.mask2, %loop2 ]
+ %ptr2 = getelementptr ptr, ptr %y, i64 %base2
+ %v2 = call <vscale x 8 x ptr> @llvm.masked.load.nxv8p0.p0(ptr %ptr2, i32 8, <vscale x 8 x i1> %mask2, <vscale x 8 x ptr> poison)
+ call void @llvm.masked.store.nxv8p0.p0(<vscale x 8 x ptr> %v2, ptr %ptr2, i32 8, <vscale x 8 x i1> %mask2)
+ %next.base2 = add i64 %base2, %step
+ %new.mask2 = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base2, i64 %n)
+ %more2 = extractelement <vscale x 8 x i1> %new.mask2, i64 0
+ br i1 %more2, label %loop2, label %exit
+
+exit:
+ ret void
+}
+
+; Tests the i8 masked load/store rewrite when the induction operands are i32 and must be extended.
+define void @rewrite_masked_load_store_i8_i32_induction(ptr %x, i32 %n) #0 {
+; CHECK-LABEL: rewrite_masked_load_store_i8_i32_induction:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov w8, w1
+; CHECK-NEXT: rdvl x9, #2
+; CHECK-NEXT: whilelo pn8.b, xzr, x8, vlx2
+; CHECK-NEXT: .LBB3_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov x10, x9
+; CHECK-NEXT: pext { p0.b, p1.b }, pn8[0]
+; CHECK-NEXT: mov w12, w9
+; CHECK-NEXT: decb x10, all, mul #2
+; CHECK-NEXT: whilelo pn8.b, x12, x8, vlx2
+; CHECK-NEXT: incb x9, all, mul #2
+; CHECK-NEXT: pext { p2.b, p3.b }, pn8[0]
+; CHECK-NEXT: mov z2.b, p2/z, #1 // =0x1
+; CHECK-NEXT: sxtw x10, w10
+; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, x10]
+; CHECK-NEXT: add x11, x0, x10
+; CHECK-NEXT: ld1b { z1.b }, p1/z, [x11, #1, mul vl]
+; CHECK-NEXT: add z0.b, z0.b, #1 // =0x1
+; CHECK-NEXT: add z1.b, z1.b, #1 // =0x1
+; CHECK-NEXT: st1b { z0.b }, p0, [x0, x10]
+; CHECK-NEXT: fmov w10, s2
+; CHECK-NEXT: st1b { z1.b }, p1, [x11, #1, mul vl]
+; CHECK-NEXT: tbnz w10, #0, .LBB3_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i32(i32 0, i32 %n)
+ %vscale = call i32 @llvm.vscale.i32()
+ %step = shl i32 %vscale, 5
+ br label %loop
+
+loop:
+ %base = phi i32 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 32 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr = getelementptr i8, ptr %x, i32 %base
+ %v = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8(ptr %ptr, i32 1, <vscale x 32 x i1> %update.mask, <vscale x 32 x i8> poison)
+ %add = add <vscale x 32 x i8> %v, splat (i8 1)
+ call void @llvm.masked.store.nxv32i8(<vscale x 32 x i8> %add, ptr %ptr, i32 1, <vscale x 32 x i1> %update.mask)
+ %next.base = add i32 %base, %step
+ %new.mask = call <vscale x 32 x i1> @llvm.get.active.lane.mask.nxv32i1.i32(i32 %next.base, i32 %n)
+ %more = extractelement <vscale x 32 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests the i32 masked load/store rewrite to predicate-as-counter with vlx2.
+define void @rewrite_masked_load_store_i32_vlx2(ptr %x, i64 %n) #0 {
+; CHECK-LABEL: rewrite_masked_load_store_i32_vlx2:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2
+; CHECK-NEXT: .LBB4_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0]
+; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h
+; CHECK-NEXT: add z0.s, z0.s, #1 // =0x1
+; CHECK-NEXT: add z1.s, z1.s, #1 // =0x1
+; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1
+; CHECK-NEXT: st1w { z0.s }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: fmov w9, s2
+; CHECK-NEXT: st1w { z1.s }, p0, [x0]
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: tbnz w9, #0, .LBB4_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr = getelementptr i32, ptr %x, i64 %base
+ %v = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32(ptr %ptr, i32 4, <vscale x 8 x i1> %update.mask, <vscale x 8 x i32> poison)
+ %add = add <vscale x 8 x i32> %v, splat (i32 1)
+ call void @llvm.masked.store.nxv8i32(<vscale x 8 x i32> %add, ptr %ptr, i32 4, <vscale x 8 x i1> %update.mask)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests choosing an i64 predicate-as-counter rewrite while materializing a wide mask for mismatched i32 users.
+define void @mixed_data_vector_types(ptr %x, ptr %y, i64 %n) #0 {
+; CHECK-LABEL: mixed_data_vector_types:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: whilelo pn8.d, xzr, x2, vlx4
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: .LBB5_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
+; CHECK-NEXT: whilelo pn8.d, x8, x2, vlx4
+; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
+; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: uzp1 p6.s, p0.s, p1.s
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
+; CHECK-NEXT: uzp1 p5.s, p2.s, p3.s
+; CHECK-NEXT: ld1w { z5.s }, p6/z, [x1]
+; CHECK-NEXT: mov z6.h, p4/z, #1 // =0x1
+; CHECK-NEXT: ld1w { z4.s }, p5/z, [x1, #1, mul vl]
+; CHECK-NEXT: incb x1, all, mul #2
+; CHECK-NEXT: fmov w9, s6
+; CHECK-NEXT: // fake_use: $z3
+; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: // fake_use: $z0
+; CHECK-NEXT: // fake_use: $z5
+; CHECK-NEXT: // fake_use: $z4
+; CHECK-NEXT: tbnz w9, #0, .LBB5_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr64 = getelementptr i64, ptr %x, i64 %base
+ %ptr32 = getelementptr i32, ptr %y, i64 %base
+ %v64 = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %ptr64, i32 8, <vscale x 8 x i1> %update.mask, <vscale x 8 x i64> poison)
+ %v32 = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32(ptr %ptr32, i32 4, <vscale x 8 x i1> %update.mask, <vscale x 8 x i32> poison)
+ call void (...) @llvm.fake.use(<vscale x 8 x i64> %v64)
+ call void (...) @llvm.fake.use(<vscale x 8 x i32> %v32)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests preferring the most common masked memory access size when selecting the predicate-as-counter element size.
+define void @prefer_more_common_masked_access_size(ptr %x16, ptr %y32, i64 %n) #0 {
+; CHECK-LABEL: prefer_more_common_masked_access_size:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: whilelo pn8.s, xzr, x2, vlx4
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: .LBB6_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: pext { p2.s, p3.s }, pn8[1]
+; CHECK-NEXT: whilelo pn8.s, x8, x2, vlx4
+; CHECK-NEXT: pext { p5.s, p6.s }, pn8[0]
+; CHECK-NEXT: uzp1 p4.h, p2.h, p3.h
+; CHECK-NEXT: incb x8
+; CHECK-NEXT: uzp1 p5.h, p5.h, p6.h
+; CHECK-NEXT: pext { p6.s, p7.s }, pn8[1]
+; CHECK-NEXT: uzp1 p6.h, p6.h, p7.h
+; CHECK-NEXT: st1h { z0.h }, p4, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p7.h, p0.h, p1.h
+; CHECK-NEXT: uzp1 p4.b, p5.b, p6.b
+; CHECK-NEXT: st1h { z0.h }, p7, [x0]
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: ld1w { z1.s }, p3/z, [x1, #3, mul vl]
+; CHECK-NEXT: ld1w { z2.s }, p2/z, [x1, #2, mul vl]
+; CHECK-NEXT: ld1w { z3.s }, p1/z, [x1, #1, mul vl]
+; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1]
+; CHECK-NEXT: mov z5.b, p4/z, #1 // =0x1
+; CHECK-NEXT: incb x1, all, mul #4
+; CHECK-NEXT: fmov w9, s5
+; CHECK-NEXT: // fake_use: $z4
+; CHECK-NEXT: // fake_use: $z3
+; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: tbnz w9, #0, .LBB6_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 4
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 16 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr16 = getelementptr i16, ptr %x16, i64 %base
+ %ptr32 = getelementptr i32, ptr %y32, i64 %base
+ call void @llvm.masked.store.nxv16i16(<vscale x 16 x i16> zeroinitializer, ptr %ptr16, i32 2, <vscale x 16 x i1> %update.mask)
+ call void @llvm.masked.store.nxv16i16(<vscale x 16 x i16> zeroinitializer, ptr %ptr16, i32 2, <vscale x 16 x i1> %update.mask)
+ %v32 = call <vscale x 16 x i32> @llvm.masked.load.nxv16i32(ptr %ptr32, i32 4, <vscale x 16 x i1> %update.mask, <vscale x 16 x i32> poison)
+ call void (...) @llvm.fake.use(<vscale x 16 x i32> %v32)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 16 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests breaking equal access-size counts in favor of the larger access size.
+define void @prefer_larger_access_size_on_tie(ptr %x16, ptr %y32, i64 %n) #0 {
+; CHECK-LABEL: prefer_larger_access_size_on_tie:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: whilelo pn8.s, xzr, x2, vlx4
+; CHECK-NEXT: rdvl x8, #1
+; CHECK-NEXT: .LBB7_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: pext { p2.s, p3.s }, pn8[1]
+; CHECK-NEXT: whilelo pn8.s, x8, x2, vlx4
+; CHECK-NEXT: pext { p5.s, p6.s }, pn8[0]
+; CHECK-NEXT: uzp1 p4.h, p2.h, p3.h
+; CHECK-NEXT: incb x8
+; CHECK-NEXT: uzp1 p5.h, p5.h, p6.h
+; CHECK-NEXT: pext { p6.s, p7.s }, pn8[1]
+; CHECK-NEXT: uzp1 p6.h, p6.h, p7.h
+; CHECK-NEXT: st1h { z0.h }, p4, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p7.h, p0.h, p1.h
+; CHECK-NEXT: uzp1 p4.b, p5.b, p6.b
+; CHECK-NEXT: st1h { z0.h }, p7, [x0]
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: ld1w { z1.s }, p3/z, [x1, #3, mul vl]
+; CHECK-NEXT: ld1w { z2.s }, p2/z, [x1, #2, mul vl]
+; CHECK-NEXT: ld1w { z3.s }, p1/z, [x1, #1, mul vl]
+; CHECK-NEXT: ld1w { z4.s }, p0/z, [x1]
+; CHECK-NEXT: mov z5.b, p4/z, #1 // =0x1
+; CHECK-NEXT: incb x1, all, mul #4
+; CHECK-NEXT: fmov w9, s5
+; CHECK-NEXT: // fake_use: $z4
+; CHECK-NEXT: // fake_use: $z3
+; CHECK-NEXT: // fake_use: $z2
+; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: tbnz w9, #0, .LBB7_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 4
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 16 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr16 = getelementptr i16, ptr %x16, i64 %base
+ %ptr32 = getelementptr i32, ptr %y32, i64 %base
+ call void @llvm.masked.store.nxv16i16(<vscale x 16 x i16> zeroinitializer, ptr %ptr16, i32 2, <vscale x 16 x i1> %update.mask)
+ %v32 = call <vscale x 16 x i32> @llvm.masked.load.nxv16i32(ptr %ptr32, i32 4, <vscale x 16 x i1> %update.mask, <vscale x 16 x i32> poison)
+ call void (...) @llvm.fake.use(<vscale x 16 x i32> %v32)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 16 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Tests rewriting extractelement when the extracted lane is within the first legal predicate section.
+define void @rewrite_extractelement_within_first_section_i32(ptr %x, i64 %n) #0 {
+; CHECK-LABEL: rewrite_extractelement_within_first_section_i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2
+; CHECK-NEXT: .LBB8_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h
+; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1
+; CHECK-NEXT: umov w9, v0.h[3]
+; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: // fake_use: $z0
+; CHECK-NEXT: tbnz w9, #0, .LBB8_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr = getelementptr i32, ptr %x, i64 %base
+ %v = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32(ptr %ptr, i32 4, <vscale x 8 x i1> %update.mask, <vscale x 8 x i32> poison)
+ call void (...) @llvm.fake.use(<vscale x 8 x i32> %v)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 3
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Negative test: masked loads with a non-poison passthru value are not rewritten to pn loads.
+define void @masked_load_passthru_non_poison(ptr %x, i64 %n) #0 {
+; CHECK-LABEL: masked_load_passthru_non_poison:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: whilelo pn8.d, xzr, x1, vlx4
+; CHECK-NEXT: .LBB9_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p2.d, p3.d }, pn8[1]
+; CHECK-NEXT: pext { p0.d, p1.d }, pn8[0]
+; CHECK-NEXT: whilelo pn8.d, x8, x1, vlx4
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0, #3, mul vl]
+; CHECK-NEXT: pext { p4.d, p5.d }, pn8[0]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #2, mul vl]
+; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s
+; CHECK-NEXT: pext { p5.d, p6.d }, pn8[1]
+; CHECK-NEXT: ld1d { z2.d }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s
+; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0]
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: add z0.d, z0.d, #1 // =0x1
+; CHECK-NEXT: add z1.d, z1.d, #1 // =0x1
+; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h
+; CHECK-NEXT: add z2.d, z2.d, #1 // =0x1
+; CHECK-NEXT: add z3.d, z3.d, #1 // =0x1
+; CHECK-NEXT: st1d { z0.d }, p3, [x0, #3, mul vl]
+; CHECK-NEXT: mov z0.h, p4/z, #1 // =0x1
+; CHECK-NEXT: st1d { z1.d }, p2, [x0, #2, mul vl]
+; CHECK-NEXT: st1d { z2.d }, p1, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z3.d }, p0, [x0]
+; CHECK-NEXT: incb x0, all, mul #4
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: tbnz w9, #0, .LBB9_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr = getelementptr i64, ptr %x, i64 %base
+ %v = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64(ptr %ptr, i32 8, <vscale x 8 x i1> %update.mask, <vscale x 8 x i64> zeroinitializer)
+ %add = add <vscale x 8 x i64> %v, splat (i64 1)
+ call void @llvm.masked.store.nxv8i64(<vscale x 8 x i64> %add, ptr %ptr, i32 8, <vscale x 8 x i1> %update.mask)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 0
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Negative test: extractelement is not rewritten when the extracted lane is outside the first predicate section.
+define void @extractelement_outside_first_section_i32(ptr %x, i64 %n) #0 {
+; CHECK-LABEL: extractelement_outside_first_section_i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: cnth x8
+; CHECK-NEXT: whilelo pn8.s, xzr, x1, vlx2
+; CHECK-NEXT: .LBB10_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: pext { p0.s, p1.s }, pn8[0]
+; CHECK-NEXT: whilelo pn8.s, x8, x1, vlx2
+; CHECK-NEXT: inch x8
+; CHECK-NEXT: pext { p2.s, p3.s }, pn8[0]
+; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT: uzp1 p2.h, p2.h, p3.h
+; CHECK-NEXT: mov z0.h, p2/z, #1 // =0x1
+; CHECK-NEXT: umov w9, v0.h[6]
+; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]
+; CHECK-NEXT: incb x0, all, mul #2
+; CHECK-NEXT: // fake_use: $z1
+; CHECK-NEXT: // fake_use: $z0
+; CHECK-NEXT: tbnz w9, #0, .LBB10_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %preheader
+
+preheader:
+ %start.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 %n)
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl i64 %vscale, 3
+ br label %loop
+
+loop:
+ %base = phi i64 [ 0, %preheader ], [ %next.base, %loop ]
+ %update.mask = phi <vscale x 8 x i1> [ %start.mask, %preheader ], [ %new.mask, %loop ]
+ %ptr = getelementptr i32, ptr %x, i64 %base
+ %v = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32(ptr %ptr, i32 4, <vscale x 8 x i1> %update.mask, <vscale x 8 x i32> poison)
+ call void (...) @llvm.fake.use(<vscale x 8 x i32> %v)
+ %next.base = add i64 %base, %step
+ %new.mask = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %next.base, i64 %n)
+ %more = extractelement <vscale x 8 x i1> %new.mask, i64 6
+ br i1 %more, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve2p1" }
>From cc67eac964112fe043aa3527c2d63d2dd55f56ee Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu at google.com>
Date: Wed, 7 Oct 2026 01:29:48 -0700
Subject: [PATCH 16/52] [clang] Replace PointerUnion::dyn_cast with
llvm::dyn_cast (NFC) (#229682)
PointerUnion::dyn_cast has been soft-deprecated in favor of
llvm::dyn_cast and llvm::dyn_cast_if_present. This patch replaces the
former with llvm::dyn_cast where the operand is guaranteed to be
nonnull.
Note that AccessPath::Base is always initialized to a nonnull pointer.
Assisted-by: Antigravity
---
.../clang/Analysis/Analyses/LifetimeSafety/Loans.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/clang/include/clang/Analysis/Analyses/LifetimeSafety/Loans.h b/clang/include/clang/Analysis/Analyses/LifetimeSafety/Loans.h
index 784968789d40a..00c931e4f9f66 100644
--- a/clang/include/clang/Analysis/Analyses/LifetimeSafety/Loans.h
+++ b/clang/include/clang/Analysis/Analyses/LifetimeSafety/Loans.h
@@ -138,19 +138,19 @@ class AccessPath {
}
const clang::ValueDecl *getAsValueDecl() const {
- return Base.dyn_cast<const clang::ValueDecl *>();
+ return dyn_cast<const clang::ValueDecl *>(Base);
}
const clang::MaterializeTemporaryExpr *getAsMaterializeTemporaryExpr() const {
- return Base.dyn_cast<const clang::MaterializeTemporaryExpr *>();
+ return dyn_cast<const clang::MaterializeTemporaryExpr *>(Base);
}
const PlaceholderBase *getAsPlaceholderBase() const {
- return Base.dyn_cast<const PlaceholderBase *>();
+ return dyn_cast<const PlaceholderBase *>(Base);
}
const clang::CXXNewExpr *getAsNewAllocation() const {
- return Base.dyn_cast<const clang::CXXNewExpr *>();
+ return dyn_cast<const clang::CXXNewExpr *>(Base);
}
bool operator==(const AccessPath &RHS) const {
>From 6afc47d555fb21b780175233f777f1ecff87e603 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 10:40:45 +0200
Subject: [PATCH 17/52] CodeGen: Remove TRI arguments from TargetInstrInfo
hooks (#228164)
Continue with cleanups enabled by #158224.. TRI can now always directly
be referenced from TargetInstrInfo
Co-authored-by: Claude Opus 5 <noreply at anthropic.com>
---
llvm/include/llvm/CodeGen/MachineScheduler.h | 7 +--
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 37 ++++++-------
llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 4 +-
llvm/lib/CodeGen/BreakFalseDeps.cpp | 8 +--
llvm/lib/CodeGen/ExpandPostRAPseudos.cpp | 2 +-
llvm/lib/CodeGen/ImplicitNullChecks.cpp | 4 +-
llvm/lib/CodeGen/MIRPrinter.cpp | 2 +-
llvm/lib/CodeGen/MachinePipeliner.cpp | 15 ++----
llvm/lib/CodeGen/MachineScheduler.cpp | 32 +++++------
llvm/lib/CodeGen/MachineSink.cpp | 2 +-
llvm/lib/CodeGen/ModuloSchedule.cpp | 3 +-
llvm/lib/CodeGen/TargetInstrInfo.cpp | 20 +++----
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 12 ++---
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 7 ++-
.../AArch64/AArch64StorePairSuppress.cpp | 5 +-
.../Target/AArch64/AArch64TargetMachine.cpp | 4 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 24 ++++-----
.../lib/Target/AMDGPU/SIInsertHardClauses.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 11 ++--
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 3 +-
llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp | 2 +-
llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp | 34 ++++++------
llvm/lib/Target/ARM/ARMBaseInstrInfo.h | 15 +++---
.../Target/Hexagon/HexagonGenMemAbsolute.cpp | 4 +-
llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp | 3 +-
llvm/lib/Target/Hexagon/HexagonInstrInfo.h | 3 +-
.../Target/Hexagon/HexagonTargetMachine.cpp | 2 +-
llvm/lib/Target/Lanai/LanaiInstrInfo.cpp | 6 +--
llvm/lib/Target/Lanai/LanaiInstrInfo.h | 3 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 5 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.h | 3 +-
llvm/lib/Target/PowerPC/PPCTargetMachine.cpp | 4 +-
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 14 ++---
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 11 ++--
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 8 +--
llvm/lib/Target/X86/X86InstrInfo.cpp | 53 +++++++++----------
llvm/lib/Target/X86/X86InstrInfo.h | 22 ++++----
.../Target/RISCV/RISCVInstrInfoTest.cpp | 11 ++--
38 files changed, 176 insertions(+), 232 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 9f82b09f83714..45a7d159e2654 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -1414,19 +1414,16 @@ class LLVM_ABI PostGenericScheduler : public GenericSchedulerBase {
/// reduce reordering due to store clustering.
LLVM_ABI std::unique_ptr<ScheduleDAGMutation>
createLoadClusterDAGMutation(const TargetInstrInfo *TII,
- const TargetRegisterInfo *TRI,
bool ReorderWhileClustering = false);
/// If ReorderWhileClustering is set to true, no attempt will be made to
/// reduce reordering due to store clustering.
LLVM_ABI std::unique_ptr<ScheduleDAGMutation>
createStoreClusterDAGMutation(const TargetInstrInfo *TII,
- const TargetRegisterInfo *TRI,
bool ReorderWhileClustering = false);
LLVM_ABI std::unique_ptr<ScheduleDAGMutation>
-createCopyConstrainDAGMutation(const TargetInstrInfo *TII,
- const TargetRegisterInfo *TRI);
+createCopyConstrainDAGMutation(const TargetInstrInfo *TII);
/// Create the standard converging machine scheduler. This will be used as the
/// default scheduler if the target does not set a default.
@@ -1440,7 +1437,7 @@ ScheduleDAGMILive *createSchedLive(MachineSchedContext *C) {
// FIXME: extend the mutation API to allow earlier mutations to instantiate
// data and pass it to later mutations. Have a single mutation that gathers
// the interesting nodes in one pass.
- DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII));
return DAG;
}
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 0f36bece2c12b..3191b9e50a86d 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1301,7 +1301,7 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// This function defines the logic to lower COPY instruction to
/// target specific instruction(s).
- void lowerCopy(MachineInstr *MI, const TargetRegisterInfo *TRI) const;
+ void lowerCopy(MachineInstr *MI) const;
/// Return true when there is potentially a faster code sequence
/// for an instruction chain ending in \p Root. All potential patterns are
@@ -1597,8 +1597,7 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// abstraction that supports negative offsets.
bool getMemOperandWithOffset(const MachineInstr &MI,
const MachineOperand *&BaseOp, int64_t &Offset,
- bool &OffsetIsScalable,
- const TargetRegisterInfo *TRI) const;
+ bool &OffsetIsScalable) const;
/// Get zero or more base operands and the byte offset of an instruction that
/// reads/writes memory. Note that there may be zero base operands if the
@@ -1611,8 +1610,7 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// abstraction that supports negative offsets.
virtual bool getMemOperandsWithOffsetWidth(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
return false;
}
@@ -1630,8 +1628,7 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// struct ExtAddrMode which contains all relevant information to make up the
/// address.
virtual std::optional<ExtAddrMode>
- getAddrModeFromMemoryOp(const MachineInstr &MemI,
- const TargetRegisterInfo *TRI) const {
+ getAddrModeFromMemoryOp(const MachineInstr &MemI) const {
return std::nullopt;
}
@@ -1661,8 +1658,7 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// function can return true even if becomes zero. Specifically cases such as
/// NullValueReg = shl NullValueReg, 63.
virtual bool preservesZeroValueInReg(const MachineInstr *MI,
- const Register NullValueReg,
- const TargetRegisterInfo *TRI) const {
+ const Register NullValueReg) const {
return false;
}
@@ -1673,9 +1669,9 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// Returns true if the two given memory operations should be scheduled
/// adjacent. Note that you have to add:
- /// DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ /// DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
/// or
- /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
/// to TargetMachine::createMachineScheduler() to have an effect.
///
/// \p BaseOps1 and \p BaseOps2 are memory operands of two memory operations.
@@ -1729,9 +1725,9 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
}
// Returns a MIRPrinter comment for this machine operand.
- virtual std::string
- createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op,
- unsigned OpIdx, const TargetRegisterInfo *TRI) const;
+ virtual std::string createMIROperandComment(const MachineInstr &MI,
+ const MachineOperand &Op,
+ unsigned OpIdx) const;
/// Returns true if the instruction is a
/// terminator instruction that has not been predicated.
@@ -2038,9 +2034,8 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// 3. Calling breakPartialRegDependency() with the same arguments. This
/// allows the target to insert a dependency breaking instruction.
///
- virtual unsigned
- getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const {
+ virtual unsigned getPartialRegUpdateClearance(const MachineInstr &MI,
+ unsigned OpNum) const {
// The default implementation returns 0 for no partial register dependency.
return 0;
}
@@ -2059,8 +2054,8 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// This hook works similarly to getPartialRegUpdateClearance, except that it
/// does not take an operand index. Instead sets \p OpNum to the index of the
/// unused register.
- virtual unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const {
+ virtual unsigned getUndefRegClearance(const MachineInstr &MI,
+ unsigned OpNum) const {
// The default implementation returns 0 for no undef register dependency.
return 0;
}
@@ -2082,8 +2077,8 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
/// An <imp-kill> operand should be added to MI if an instruction was
/// inserted. This ties the instructions together in the post-ra scheduler.
///
- virtual void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const {}
+ virtual void breakPartialRegDependency(MachineInstr &MI,
+ unsigned OpNum) const {}
/// Create machine specific model for scheduling.
virtual DFAPacketizer *
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 99c66e5ff684f..12fb2829e1ed6 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1017,9 +1017,7 @@ void DwarfDebug::constructCallSiteEntryDIEs(const DISubprogram &SP,
if (PhysRegCalleeOperand) {
bool Scalable = false;
const MachineOperand *BaseOp = nullptr;
- const TargetRegisterInfo &TRI =
- *Asm->MF->getSubtarget().getRegisterInfo();
- if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, Scalable, &TRI)) {
+ if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, Scalable)) {
if (BaseOp && BaseOp->isReg() && !Scalable)
CallTarget = MachineLocation(BaseOp->getReg(), /*Indirect*/ true);
}
diff --git a/llvm/lib/CodeGen/BreakFalseDeps.cpp b/llvm/lib/CodeGen/BreakFalseDeps.cpp
index 180ebc18db06c..c9e71eb3f8fbc 100644
--- a/llvm/lib/CodeGen/BreakFalseDeps.cpp
+++ b/llvm/lib/CodeGen/BreakFalseDeps.cpp
@@ -213,7 +213,7 @@ void BreakFalseDeps::processDefs(MachineInstr *MI) {
if (!MO.isReg() || !MO.getReg() || !MO.isUse() || !MO.isUndef())
continue;
- unsigned Pref = TII->getUndefRegClearance(*MI, i, TRI);
+ unsigned Pref = TII->getUndefRegClearance(*MI, i);
if (Pref) {
bool HadTrueDependency = pickBestRegisterForUndef(MI, i, Pref);
// We don't need to bother trying to break a dependency if this
@@ -238,9 +238,9 @@ void BreakFalseDeps::processDefs(MachineInstr *MI) {
if (MO.isUse())
continue;
// Check clearance before partial register updates.
- unsigned Pref = TII->getPartialRegUpdateClearance(*MI, i, TRI);
+ unsigned Pref = TII->getPartialRegUpdateClearance(*MI, i);
if (Pref && shouldBreakDependence(MI, i, Pref)) {
- TII->breakPartialRegDependency(*MI, i, TRI);
+ TII->breakPartialRegDependency(*MI, i);
Changed = true;
}
}
@@ -270,7 +270,7 @@ void BreakFalseDeps::processUndefReads(MachineBasicBlock *MBB) {
if (UndefMI == &I) {
if (!LiveRegSet.contains(UndefMI->getOperand(OpIdx).getReg())) {
- TII->breakPartialRegDependency(*UndefMI, OpIdx, TRI);
+ TII->breakPartialRegDependency(*UndefMI, OpIdx);
Changed = true;
}
diff --git a/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp b/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp
index 9c819beff55af..b2dcf31f15082 100644
--- a/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp
+++ b/llvm/lib/CodeGen/ExpandPostRAPseudos.cpp
@@ -146,7 +146,7 @@ bool ExpandPostRA::run(MachineFunction &MF) {
MadeChange |= LowerSubregToReg(&MI);
break;
case TargetOpcode::COPY:
- TII->lowerCopy(&MI, TRI);
+ TII->lowerCopy(&MI);
MadeChange = true;
break;
case TargetOpcode::DBG_VALUE:
diff --git a/llvm/lib/CodeGen/ImplicitNullChecks.cpp b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
index a1e256bcd7b17..cde25e771daac 100644
--- a/llvm/lib/CodeGen/ImplicitNullChecks.cpp
+++ b/llvm/lib/CodeGen/ImplicitNullChecks.cpp
@@ -380,7 +380,7 @@ ImplicitNullChecksImpl::isSuitableMemoryOp(const MachineInstr &MI,
if (!MI.mayLoadOrStore() || MI.isPredicable())
return SR_Unsuitable;
- auto AM = TII->getAddrModeFromMemoryOp(MI, TRI);
+ auto AM = TII->getAddrModeFromMemoryOp(MI);
if (!AM || AM->Form != ExtAddrMode::Formula::Basic)
return SR_Unsuitable;
auto AddrMode = *AM;
@@ -700,7 +700,7 @@ bool ImplicitNullChecksImpl::analyzeBlockForNullChecks(
// If MI re-defines the PointerReg in a way that changes the value of
// PointerReg if it was null, then we cannot move further.
- if (!TII->preservesZeroValueInReg(&MI, PointerReg, TRI))
+ if (!TII->preservesZeroValueInReg(&MI, PointerReg))
return false;
InstsSeenSoFar.push_back(&MI);
}
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 7ac0c40dfd93f..def751e6df6bb 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -989,7 +989,7 @@ static void printMIOperand(raw_ostream &OS, MFPrintState &State,
const MachineRegisterInfo &MRI, bool PrintDef) {
LLT TypeToPrint = MI.getTypeToPrint(OpIdx, PrintedTypes, MRI);
const MachineOperand &Op = MI.getOperand(OpIdx);
- std::string MOComment = TII->createMIROperandComment(MI, Op, OpIdx, TRI);
+ std::string MOComment = TII->createMIROperandComment(MI, Op, OpIdx);
switch (Op.getType()) {
case MachineOperand::MO_Immediate:
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 3d7c1f1f8a9e5..1546f191889c1 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -3050,9 +3050,6 @@ static bool findLoopIncrementValue(const MachineInstr &MI,
const TargetInstrInfo *TII =
LoopBB->getParent()->getSubtarget().getInstrInfo();
- const TargetRegisterInfo *TRI =
- LoopBB->getParent()->getSubtarget().getRegisterInfo();
-
MachineInstr *Phi = nullptr;
MachineInstr *Increment = nullptr;
@@ -3091,8 +3088,8 @@ static bool findLoopIncrementValue(const MachineInstr &MI,
const MachineOperand *BaseOp;
int64_t Offset;
bool OffsetIsScalable;
- if (TII->getMemOperandWithOffset(*Def, BaseOp, Offset, OffsetIsScalable,
- TRI)) {
+ if (TII->getMemOperandWithOffset(*Def, BaseOp, Offset,
+ OffsetIsScalable)) {
// Pre/post increment instruction
CurReg = BaseOp->getReg();
} else {
@@ -3119,11 +3116,10 @@ static bool findLoopIncrementValue(const MachineInstr &MI,
/// Return true if we can compute the amount the instruction changes
/// during each iteration. Set Delta to the amount of the change.
bool SwingSchedulerDAG::computeDelta(const MachineInstr &MI, int &Delta) const {
- const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
const MachineOperand *BaseOp;
int64_t Offset;
bool OffsetIsScalable;
- if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
+ if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable))
return false;
// FIXME: This algorithm assumes instructions have fixed-size offsets.
@@ -3264,11 +3260,10 @@ bool SwingSchedulerDAG::mayOverlapInLaterIter(
const MachineOperand *BaseOpB, *BaseOpO;
int64_t OffsetB, OffsetO;
bool OffsetBIsScalable, OffsetOIsScalable;
- const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
if (!TII->getMemOperandWithOffset(*BaseMI, BaseOpB, OffsetB,
- OffsetBIsScalable, TRI) ||
+ OffsetBIsScalable) ||
!TII->getMemOperandWithOffset(*OtherMI, BaseOpO, OffsetO,
- OffsetOIsScalable, TRI))
+ OffsetOIsScalable))
return true;
if (OffsetBIsScalable || OffsetOIsScalable)
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 19d293ecd58e3..df0886097997a 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -2214,15 +2214,13 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation {
};
const TargetInstrInfo *TII;
- const TargetRegisterInfo *TRI;
bool IsLoad;
bool ReorderWhileClustering;
public:
- BaseMemOpClusterMutation(const TargetInstrInfo *tii,
- const TargetRegisterInfo *tri, bool IsLoad,
+ BaseMemOpClusterMutation(const TargetInstrInfo *tii, bool IsLoad,
bool ReorderWhileClustering)
- : TII(tii), TRI(tri), IsLoad(IsLoad),
+ : TII(tii), IsLoad(IsLoad),
ReorderWhileClustering(ReorderWhileClustering) {}
void apply(ScheduleDAGInstrs *DAGInstrs) override;
@@ -2238,36 +2236,31 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation {
class StoreClusterMutation : public BaseMemOpClusterMutation {
public:
- StoreClusterMutation(const TargetInstrInfo *tii,
- const TargetRegisterInfo *tri,
- bool ReorderWhileClustering)
- : BaseMemOpClusterMutation(tii, tri, false, ReorderWhileClustering) {}
+ StoreClusterMutation(const TargetInstrInfo *tii, bool ReorderWhileClustering)
+ : BaseMemOpClusterMutation(tii, false, ReorderWhileClustering) {}
};
class LoadClusterMutation : public BaseMemOpClusterMutation {
public:
- LoadClusterMutation(const TargetInstrInfo *tii, const TargetRegisterInfo *tri,
- bool ReorderWhileClustering)
- : BaseMemOpClusterMutation(tii, tri, true, ReorderWhileClustering) {}
+ LoadClusterMutation(const TargetInstrInfo *tii, bool ReorderWhileClustering)
+ : BaseMemOpClusterMutation(tii, true, ReorderWhileClustering) {}
};
} // end anonymous namespace
std::unique_ptr<ScheduleDAGMutation>
llvm::createLoadClusterDAGMutation(const TargetInstrInfo *TII,
- const TargetRegisterInfo *TRI,
bool ReorderWhileClustering) {
return EnableMemOpCluster ? std::make_unique<LoadClusterMutation>(
- TII, TRI, ReorderWhileClustering)
+ TII, ReorderWhileClustering)
: nullptr;
}
std::unique_ptr<ScheduleDAGMutation>
llvm::createStoreClusterDAGMutation(const TargetInstrInfo *TII,
- const TargetRegisterInfo *TRI,
bool ReorderWhileClustering) {
return EnableMemOpCluster ? std::make_unique<StoreClusterMutation>(
- TII, TRI, ReorderWhileClustering)
+ TII, ReorderWhileClustering)
: nullptr;
}
@@ -2397,7 +2390,7 @@ void BaseMemOpClusterMutation::collectMemOpRecords(
bool OffsetIsScalable;
LocationSize Width = LocationSize::precise(0);
if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
- OffsetIsScalable, Width, TRI)) {
+ OffsetIsScalable, Width)) {
if (!Width.hasValue())
continue;
@@ -2488,7 +2481,7 @@ class CopyConstrain : public ScheduleDAGMutation {
SlotIndex RegionEndIdx;
public:
- CopyConstrain(const TargetInstrInfo *, const TargetRegisterInfo *) {}
+ CopyConstrain(const TargetInstrInfo *) {}
void apply(ScheduleDAGInstrs *DAGInstrs) override;
@@ -2499,9 +2492,8 @@ class CopyConstrain : public ScheduleDAGMutation {
} // end anonymous namespace
std::unique_ptr<ScheduleDAGMutation>
-llvm::createCopyConstrainDAGMutation(const TargetInstrInfo *TII,
- const TargetRegisterInfo *TRI) {
- return std::make_unique<CopyConstrain>(TII, TRI);
+llvm::createCopyConstrainDAGMutation(const TargetInstrInfo *TII) {
+ return std::make_unique<CopyConstrain>(TII);
}
/// constrainLocalCopy handles two possibilities:
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 3bfa604ccbd7b..c3caac2d97154 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -1540,7 +1540,7 @@ static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI,
const MachineOperand *BaseOp;
int64_t Offset;
bool OffsetIsScalable;
- if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
+ if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable))
return false;
if (!BaseOp->isReg())
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index a86ca6ff7e1e8..edae24742ec9f 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -933,11 +933,10 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
/// Return true if we can compute the amount the instruction changes
/// during each iteration. Set Delta to the amount of the change.
bool ModuloScheduleExpander::computeDelta(MachineInstr &MI, unsigned &Delta) {
- const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
const MachineOperand *BaseOp;
int64_t Offset;
bool OffsetIsScalable;
- if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
+ if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable))
return false;
// FIXME: This algorithm assumes instructions have fixed-size offsets.
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index e5bb73dc4b581..0b87900194b89 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -888,8 +888,7 @@ static void transferImplicitOperands(MachineInstr *MI,
}
}
-void TargetInstrInfo::lowerCopy(
- MachineInstr *MI, const TargetRegisterInfo * /*Remove me*/) const {
+void TargetInstrInfo::lowerCopy(MachineInstr *MI) const {
if (MI->allDefsAreDead()) {
MI->setDesc(get(TargetOpcode::KILL));
return;
@@ -1749,13 +1748,14 @@ CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
}
// Default implementation of getMemOperandWithOffset.
-bool TargetInstrInfo::getMemOperandWithOffset(
- const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset,
- bool &OffsetIsScalable, const TargetRegisterInfo * /*RemoveMe*/) const {
+bool TargetInstrInfo::getMemOperandWithOffset(const MachineInstr &MI,
+ const MachineOperand *&BaseOp,
+ int64_t &Offset,
+ bool &OffsetIsScalable) const {
SmallVector<const MachineOperand *, 4> BaseOps;
LocationSize Width = LocationSize::precise(0);
if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable,
- Width, &TRI) ||
+ Width) ||
BaseOps.size() != 1)
return false;
BaseOp = BaseOps.front();
@@ -1917,7 +1917,7 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI,
return std::nullopt;
const MachineOperand *BaseOp;
- if (!getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, &TRI))
+ if (!getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable))
return std::nullopt;
// FIXME: Scalable offsets are not yet handled in the offset code below.
@@ -2054,9 +2054,9 @@ bool TargetInstrInfo::getInsertSubregInputs(
}
// Returns a MIRPrinter comment for this machine operand.
-std::string TargetInstrInfo::createMIROperandComment(
- const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
- const TargetRegisterInfo * /*RemoveMe*/) const {
+std::string TargetInstrInfo::createMIROperandComment(const MachineInstr &MI,
+ const MachineOperand &Op,
+ unsigned OpIdx) const {
if (!MI.isInlineAsm())
return "";
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 485ba3ffd82d3..c1f2515134c2e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3683,15 +3683,14 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
if (!LdSt.mayLoadOrStore())
return false;
const MachineOperand *BaseOp;
TypeSize WidthN(0, false);
if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
- WidthN, TRI))
+ WidthN, &RI))
return false;
// The maximum vscale is 16 under AArch64, return the maximal extent for the
// vector.
@@ -3701,12 +3700,11 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
}
std::optional<ExtAddrMode>
-AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
- const TargetRegisterInfo *TRI) const {
+AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI) const {
const MachineOperand *Base; // Filled with the base operand of MI.
int64_t Offset; // Filled with the offset of MI.
bool OffsetIsScalable;
- if (!getMemOperandWithOffset(MemI, Base, Offset, OffsetIsScalable, TRI))
+ if (!getMemOperandWithOffset(MemI, Base, Offset, OffsetIsScalable))
return std::nullopt;
if (!Base->isReg())
@@ -10716,7 +10714,7 @@ AArch64InstrInfo::getOutliningCandidateInfo(
// Does it allow us to offset the base operand and is the base the
// register SP?
- if (!getMemOperandWithOffset(MI, Base, Offset, OffsetIsScalable, &TRI) ||
+ if (!getMemOperandWithOffset(MI, Base, Offset, OffsetIsScalable) ||
!Base->isReg() || Base->getReg() != AArch64::SP)
return false;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 6a8346395eea1..4654fce39b02e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -314,8 +314,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
static void suppressLdStPair(MachineInstr &MI);
std::optional<ExtAddrMode>
- getAddrModeFromMemoryOp(const MachineInstr &MemI,
- const TargetRegisterInfo *TRI) const override;
+ getAddrModeFromMemoryOp(const MachineInstr &MemI) const override;
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
const MachineInstr &AddrI,
@@ -326,8 +325,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const override;
+ int64_t &Offset, bool &OffsetIsScalable,
+ LocationSize &Width) const override;
/// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.
/// This is true for some SVE instructions like ldr/str that have a
diff --git a/llvm/lib/Target/AArch64/AArch64StorePairSuppress.cpp b/llvm/lib/Target/AArch64/AArch64StorePairSuppress.cpp
index dd775da97112d..f339390b7561a 100644
--- a/llvm/lib/Target/AArch64/AArch64StorePairSuppress.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StorePairSuppress.cpp
@@ -30,7 +30,6 @@ using namespace llvm;
namespace {
class AArch64StorePairSuppress : public MachineFunctionPass {
const AArch64InstrInfo *TII;
- const TargetRegisterInfo *TRI;
const MachineRegisterInfo *MRI;
TargetSchedModel SchedModel;
MachineTraceMetrics *Traces;
@@ -134,7 +133,6 @@ bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
return false;
TII = ST.getInstrInfo();
- TRI = ST.getRegisterInfo();
MRI = &MF.getRegInfo();
SchedModel.init(&ST);
Traces = &getAnalysis<MachineTraceMetricsWrapperPass>().getMTM();
@@ -160,8 +158,7 @@ bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {
const MachineOperand *BaseOp;
int64_t Offset;
bool OffsetIsScalable;
- if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,
- TRI) &&
+ if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable) &&
BaseOp->isReg()) {
Register BaseReg = BaseOp->getReg();
if (PrevBaseReg == BaseReg) {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 9e783b0a23e67..a0eb10770fed5 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -542,8 +542,8 @@ ScheduleDAGInstrs *
AArch64TargetMachine::createMachineScheduler(MachineSchedContext *C) const {
const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
ScheduleDAGMILive *DAG = createSchedLive(C);
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
if (ST.hasFusion())
DAG->addMutation(createAArch64MacroFusionDAGMutation());
if (ST.hasSME() && ST.isStreaming())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index fab805fa180f1..c866722fbc3e1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -789,9 +789,9 @@ createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
ScheduleDAGMILive *DAG =
new GCNScheduleDAGMILive(C, std::make_unique<GCNMaxOccupancySchedStrategy>(C));
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
if (ST.shouldClusterStores())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));
DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
@@ -813,9 +813,9 @@ createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C) {
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
C, std::make_unique<GCNMaxMemoryClauseSchedStrategy>(C));
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
if (ST.shouldClusterStores())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
DAG->addMutation(createAMDGPUBarrierLatencyDAGMutation(C->MF));
DAG->addMutation(createAMDGPUHazardLatencyDAGMutation(C->MF));
@@ -827,9 +827,9 @@ createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
auto *DAG = new GCNIterativeScheduler(
C, GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
if (ST.shouldClusterStores())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));
return DAG;
}
@@ -845,9 +845,9 @@ static ScheduleDAGInstrs *
createIterativeILPMachineScheduler(MachineSchedContext *C) {
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
auto *DAG = new GCNIterativeScheduler(C, GCNIterativeScheduler::SCHEDULE_ILP);
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
if (ST.shouldClusterStores())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::Initial));
return DAG;
@@ -968,9 +968,9 @@ llvm::ScheduleDAGInstrs *
AMDGPUTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
ScheduleDAGMILive *DAG = createSchedLive(C);
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
if (ST.shouldClusterStores())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
return DAG;
}
@@ -1468,9 +1468,9 @@ GCNTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
new GCNPostScheduleDAGMILive(C, std::make_unique<PostGenericScheduler>(C),
/*RemoveKillFlags=*/true);
const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
- DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createLoadClusterDAGMutation(DAG->TII));
if (ST.shouldClusterStores())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
DAG->addMutation(createIGroupLPDAGMutation(AMDGPU::SchedulingPhase::PostRA));
if ((EnableVOPD.getNumOccurrences() ||
getOptLevel() >= CodeGenOptLevel::Less) &&
diff --git a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
index b25403fb95b9f..7c33ed771a53a 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
@@ -220,7 +220,6 @@ class SIInsertHardClauses {
return false;
const SIInstrInfo *SII = ST->getInstrInfo();
- const TargetRegisterInfo *TRI = ST->getRegisterInfo();
bool Changed = false;
for (auto &MBB : MF) {
@@ -247,7 +246,7 @@ class SIInsertHardClauses {
SmallVector<const MachineOperand *, 4> BaseOps;
if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2,
- Dummy3, TRI)) {
+ Dummy3)) {
// We failed to get the base operands, so we'll never clause this
// instruction with any other, so pretend it's illegal.
Type = HARDCLAUSE_ILLEGAL;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 78ba4d6e01e2e..dcc0e98eb18f2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -501,8 +501,7 @@ static bool isStride64(unsigned Opc) {
bool SIInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -550,11 +549,11 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
unsigned EltSize;
if (LdSt.mayLoad())
- EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
+ EltSize = RI.getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
else {
assert(LdSt.mayStore());
int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
- EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
+ EltSize = RI.getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
}
if (isStride64(Opc))
@@ -4268,9 +4267,9 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
LocationSize Dummy1 = LocationSize::precise(0);
bool Offset0IsScalable, Offset1IsScalable;
if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
- Dummy0, &RI) ||
+ Dummy0) ||
!getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,
- Dummy1, &RI))
+ Dummy1))
return false;
if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 12b0af0c94dc7..70b1f237f3c7e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -298,8 +298,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const final;
+ bool &OffsetIsScalable, LocationSize &Width) const final;
bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
int64_t Offset1, bool OffsetIsScalable1,
diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
index c802a301418e2..c96b39c81c5a6 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp
@@ -1903,7 +1903,7 @@ void SIScheduleDAGMI::schedule()
IsLowLatencySU[i] = 1;
bool OffsetIsScalable;
if (SITII->getMemOperandWithOffset(*SU->getInstr(), BaseLatOp, OffLatReg,
- OffsetIsScalable, TRI))
+ OffsetIsScalable))
LowLatencyOffset[i] = OffLatReg;
} else if (SITII->isHighLatencyDef(SU->getInstr()->getOpcode()))
IsHighLatencySU[i] = 1;
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 404afe6a8c9a9..92e8c0e994e79 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -399,13 +399,13 @@ bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
return PIdx != -1 && MI.getOperand(PIdx).getImm() != ARMCC::AL;
}
-std::string ARMBaseInstrInfo::createMIROperandComment(
- const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
- const TargetRegisterInfo *TRI) const {
+std::string ARMBaseInstrInfo::createMIROperandComment(const MachineInstr &MI,
+ const MachineOperand &Op,
+ unsigned OpIdx) const {
// First, let's see if there is a generic comment for this operand
std::string GenericComment =
- TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
+ TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx);
if (!GenericComment.empty())
return GenericComment;
@@ -5184,15 +5184,13 @@ void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
// VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
//
// FCONSTD can be used as a dependency-breaking instruction.
-unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
- const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const {
+unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(const MachineInstr &MI,
+ unsigned OpNum) const {
auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
if (!PartialUpdateClearance)
return 0;
- assert(TRI && "Need TRI instance");
-
+ const ARMBaseRegisterInfo &TRI = getRegisterInfo();
const MachineOperand &MO = MI.getOperand(OpNum);
if (MO.readsReg())
return 0;
@@ -5209,7 +5207,7 @@ unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
case ARM::VMOVv2i32:
case ARM::VMOVv2f32:
case ARM::VMOVv1i64:
- UseOp = MI.findRegisterUseOperandIdx(Reg, TRI, false);
+ UseOp = MI.findRegisterUseOperandIdx(Reg, &TRI, false);
break;
// Explicitly reads the dependency.
@@ -5233,8 +5231,8 @@ unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
} else if (ARM::SPRRegClass.contains(Reg)) {
// Physical register: MI must define the full D-reg.
MCRegister DReg =
- TRI->getMatchingSuperReg(Reg, ARM::ssub_0, &ARM::DPRRegClass);
- if (!DReg || !MI.definesRegister(DReg, TRI))
+ TRI.getMatchingSuperReg(Reg, ARM::ssub_0, &ARM::DPRRegClass);
+ if (!DReg || !MI.definesRegister(DReg, &TRI))
return 0;
}
@@ -5245,11 +5243,11 @@ unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
// Break a partial register dependency after getPartialRegUpdateClearance
// returned non-zero.
-void ARMBaseInstrInfo::breakPartialRegDependency(
- MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
+void ARMBaseInstrInfo::breakPartialRegDependency(MachineInstr &MI,
+ unsigned OpNum) const {
assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
- assert(TRI && "Need TRI instance");
+ const ARMBaseRegisterInfo &TRI = getRegisterInfo();
const MachineOperand &MO = MI.getOperand(OpNum);
Register Reg = MO.getReg();
assert(Reg.isPhysical() && "Can't break virtual register dependencies.");
@@ -5258,11 +5256,11 @@ void ARMBaseInstrInfo::breakPartialRegDependency(
// If MI defines an S-reg, find the corresponding D super-register.
if (ARM::SPRRegClass.contains(Reg)) {
DReg = ARM::D0 + (Reg - ARM::S0) / 2;
- assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
+ assert(TRI.isSuperRegister(Reg, DReg) && "Register enums broken");
}
assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
- assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
+ assert(MI.definesRegister(DReg, &TRI) && "MI doesn't clobber full D-reg");
// FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
// the full D-register by loading the same value to both lanes. The
@@ -5275,7 +5273,7 @@ void ARMBaseInstrInfo::breakPartialRegDependency(
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::FCONSTD), DReg)
.addImm(96)
.add(predOps(ARMCC::AL));
- MI.addRegisterKilled(DReg, TRI, true);
+ MI.addRegisterKilled(DReg, &TRI, true);
}
bool ARMBaseInstrInfo::hasNOP() const {
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index ed06343716a7f..9bf4e7cef6ddd 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -171,10 +171,9 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
bool isPredicated(const MachineInstr &MI) const override;
// MIR printer helper function to annotate Operands with a comment.
- std::string
- createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op,
- unsigned OpIdx,
- const TargetRegisterInfo *TRI) const override;
+ std::string createMIROperandComment(const MachineInstr &MI,
+ const MachineOperand &Op,
+ unsigned OpIdx) const override;
ARMCC::CondCodes getPredicate(const MachineInstr &MI) const {
int PIdx = MI.findFirstPredOperandIdx();
@@ -348,11 +347,9 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo {
getExecutionDomain(const MachineInstr &MI) const override;
void setExecutionDomain(MachineInstr &MI, unsigned Domain) const override;
- unsigned
- getPartialRegUpdateClearance(const MachineInstr &, unsigned,
- const TargetRegisterInfo *) const override;
- void breakPartialRegDependency(MachineInstr &, unsigned,
- const TargetRegisterInfo *TRI) const override;
+ unsigned getPartialRegUpdateClearance(const MachineInstr &,
+ unsigned) const override;
+ void breakPartialRegDependency(MachineInstr &, unsigned) const override;
/// Get the number of addresses by LDM or VLDM or zero for unknown.
unsigned getNumLDMAddresses(const MachineInstr &MI) const;
diff --git a/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp b/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
index 3fd34c6daad94..d635700c3f051 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
@@ -36,7 +36,6 @@ namespace {
class HexagonGenMemAbsolute : public MachineFunctionPass {
const HexagonInstrInfo *TII = nullptr;
MachineRegisterInfo *MRI = nullptr;
- const TargetRegisterInfo *TRI = nullptr;
public:
static char ID;
@@ -72,7 +71,6 @@ bool HexagonGenMemAbsolute::runOnMachineFunction(MachineFunction &Fn) {
TII = Fn.getSubtarget<HexagonSubtarget>().getInstrInfo();
MRI = &Fn.getRegInfo();
- TRI = Fn.getRegInfo().getTargetRegisterInfo();
MachineDominatorTree &MDT =
getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
@@ -121,7 +119,7 @@ bool HexagonGenMemAbsolute::runOnMachineFunction(MachineFunction &Fn) {
const MachineOperand *BaseOp = nullptr;
int64_t Offset;
bool Scalable;
- TII->getMemOperandWithOffset(*NextMI, BaseOp, Offset, Scalable, TRI);
+ TII->getMemOperandWithOffset(*NextMI, BaseOp, Offset, Scalable);
// Ensure BaseOp is non-null and register type.
if (!BaseOp || !BaseOp->isReg())
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 97f3d9aee87ee..e78bf6b726663 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -3115,8 +3115,7 @@ bool HexagonInstrInfo::addLatencyToSchedule(const MachineInstr &MI1,
/// Get the base register and byte offset of a load/store instr.
bool HexagonInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
OffsetIsScalable = false;
const MachineOperand *BaseOp = getBaseAndOffset(LdSt, Offset, Width);
if (!BaseOp || !BaseOp->isReg())
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 30b2c91d44442..5e5c93b2b4ccb 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -222,8 +222,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const override;
+ bool &OffsetIsScalable, LocationSize &Width) const override;
/// Reverses the branch condition of the specified condition list,
/// returning false on success and true if it cannot be reversed.
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
index 84e3390e4f5ae..42418227e7c13 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -188,7 +188,7 @@ static ScheduleDAGInstrs *createVLIWMachineSched(MachineSchedContext *C) {
DAG->addMutation(std::make_unique<HexagonSubtarget::UsrOverflowMutation>());
DAG->addMutation(std::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
DAG->addMutation(std::make_unique<HexagonSubtarget::CallMutation>());
- DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII));
return DAG;
}
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index 18bddf9c2fc35..33773023f6664 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -778,8 +778,7 @@ bool LanaiInstrInfo::getMemOperandWithOffsetWidth(
bool LanaiInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
switch (LdSt.getOpcode()) {
default:
return false;
@@ -794,7 +793,8 @@ bool LanaiInstrInfo::getMemOperandsWithOffsetWidth(
case Lanai::LDBz_RI:
const MachineOperand *BaseOp;
OffsetIsScalable = false;
- if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
+ if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width,
+ &getRegisterInfo()))
return false;
BaseOps.push_back(BaseOp);
return true;
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
index f9b9d91b5b6b7..2ac81edf49fad 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
@@ -73,8 +73,7 @@ class LanaiInstrInfo : public LanaiGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const override;
+ bool &OffsetIsScalable, LocationSize &Width) const override;
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index bf7f3df6035d8..e8935ceae4fb3 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -2891,11 +2891,10 @@ bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
const MachineOperand *BaseOp;
OffsetIsScalable = false;
- if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
+ if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, &RI))
return false;
BaseOps.push_back(BaseOp);
return true;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 542e0b0701506..d86cf04facaff 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -679,8 +679,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const override;
+ bool &OffsetIsScalable, LocationSize &Width) const override;
/// Returns true if the two given memory operations should be scheduled
/// adjacent.
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index b67961aa22c30..c63eba5f124f8 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -266,7 +266,7 @@ static ScheduleDAGInstrs *createPPCMachineScheduler(MachineSchedContext *C) {
: createSchedLive<GenericScheduler>(C);
// add DAG Mutations here.
if (ST.hasStoreFusion())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
if (ST.hasFusion())
DAG->addMutation(createPowerPCMacroFusionDAGMutation());
@@ -281,7 +281,7 @@ createPPCPostMachineScheduler(MachineSchedContext *C) {
: createSchedPostRA<PostGenericScheduler>(C);
// add DAG Mutations here.
if (ST.hasStoreFusion())
- DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
+ DAG->addMutation(createStoreClusterDAGMutation(DAG->TII));
if (ST.hasFusion())
DAG->addMutation(createPowerPCMacroFusionDAGMutation());
return DAG;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 908de0c374b3d..23d566581a1ae 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -3561,8 +3561,7 @@ bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -3597,7 +3596,8 @@ bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
}
const MachineOperand *BaseOp;
OffsetIsScalable = false;
- if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
+ if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width,
+ &getRegisterInfo()))
return false;
BaseOps.push_back(BaseOp);
return true;
@@ -4144,12 +4144,12 @@ std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
}
// MIR printer helper function to annotate Operands with a comment.
-std::string RISCVInstrInfo::createMIROperandComment(
- const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
- const TargetRegisterInfo *TRI) const {
+std::string RISCVInstrInfo::createMIROperandComment(const MachineInstr &MI,
+ const MachineOperand &Op,
+ unsigned OpIdx) const {
// Print a generic comment for this operand if there is one.
std::string GenericComment =
- TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
+ TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx);
if (!GenericComment.empty())
return GenericComment;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 8e47b0e0aff0b..7cfb7f70521b5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -204,8 +204,8 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
bool getMemOperandsWithOffsetWidth(
const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const override;
+ int64_t &Offset, bool &OffsetIsScalable,
+ LocationSize &Width) const override;
bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
int64_t Offset1, bool OffsetIsScalable1,
@@ -283,10 +283,9 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
LiveIntervals *LIS) const override;
// MIR printer helper function to annotate Operands with a comment.
- std::string
- createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op,
- unsigned OpIdx,
- const TargetRegisterInfo *TRI) const override;
+ std::string createMIROperandComment(const MachineInstr &MI,
+ const MachineOperand &Op,
+ unsigned OpIdx) const override;
/// Generate code to multiply the value in DestReg by Amt - handles all
/// the common optimizations for this idiom, and supports fallback for
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 685bb91583cf5..c8a5483d8dc26 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -290,11 +290,11 @@ RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
if (ST.enableMISchedLoadClustering())
DAG->addMutation(createLoadClusterDAGMutation(
- DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+ DAG->TII, /*ReorderWhileClustering=*/true));
if (ST.enableMISchedStoreClustering())
DAG->addMutation(createStoreClusterDAGMutation(
- DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+ DAG->TII, /*ReorderWhileClustering=*/true));
if (!DisableVectorMaskMutation && ST.hasVInstructions())
DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
@@ -314,11 +314,11 @@ RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
if (ST.enablePostMISchedLoadClustering())
DAG->addMutation(createLoadClusterDAGMutation(
- DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+ DAG->TII, /*ReorderWhileClustering=*/true));
if (ST.enablePostMISchedStoreClustering())
DAG->addMutation(createStoreClusterDAGMutation(
- DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+ DAG->TII, /*ReorderWhileClustering=*/true));
return DAG;
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 307369010cc8e..9c83d216b93fb 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4506,8 +4506,7 @@ static unsigned getLoadStoreRegOpcode(Register Reg,
}
std::optional<ExtAddrMode>
-X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
- const TargetRegisterInfo *TRI) const {
+X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI) const {
int MemRefBegin = X86II::getMemoryOperandIdx(MemI.getDesc());
if (MemRefBegin < 0)
return std::nullopt;
@@ -4531,7 +4530,7 @@ X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
bool X86InstrInfo::verifyInstruction(const MachineInstr &MI,
StringRef &ErrInfo) const {
- std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr);
+ std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI);
if (!AMOrNone)
return true;
@@ -4597,10 +4596,9 @@ bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
return true;
}
-bool X86InstrInfo::preservesZeroValueInReg(
- const MachineInstr *MI, const Register NullValueReg,
- const TargetRegisterInfo *TRI) const {
- if (!MI->modifiesRegister(NullValueReg, TRI))
+bool X86InstrInfo::preservesZeroValueInReg(const MachineInstr *MI,
+ const Register NullValueReg) const {
+ if (!MI->modifiesRegister(NullValueReg, &RI))
return true;
switch (MI->getOpcode()) {
// Shift right/left of a null unto itself is still a null, i.e. rax = shl rax
@@ -4617,7 +4615,7 @@ bool X86InstrInfo::preservesZeroValueInReg(
// null value.
case X86::MOV32rr:
return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) {
- return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
+ return RI.isSubRegisterEq(NullValueReg, MO.getReg());
});
default:
return false;
@@ -4627,8 +4625,7 @@ bool X86InstrInfo::preservesZeroValueInReg(
bool X86InstrInfo::getMemOperandsWithOffsetWidth(
const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps,
- int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const {
+ int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
int MemRefBegin = X86II::getMemoryOperandIdx(MemOp.getDesc());
if (MemRefBegin < 0)
return false;
@@ -6970,9 +6967,8 @@ static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget,
/// Inform the BreakFalseDeps pass how many idle
/// instructions we would like before a partial register update.
-unsigned X86InstrInfo::getPartialRegUpdateClearance(
- const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const {
+unsigned X86InstrInfo::getPartialRegUpdateClearance(const MachineInstr &MI,
+ unsigned OpNum) const {
if (OpNum != 0)
return 0;
@@ -7001,7 +6997,7 @@ unsigned X86InstrInfo::getPartialRegUpdateClearance(
if (Reg.isVirtual())
ReadsReg = (MO.readsReg() || MI.readsVirtualRegister(Reg));
else
- ReadsReg = MI.readsRegister(Reg, TRI);
+ ReadsReg = MI.readsRegister(Reg, &RI);
if (ReadsReg != HasNDDPartialWrite)
return 0;
@@ -7357,9 +7353,8 @@ static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum,
///
/// Like getPartialRegUpdateClearance, this makes a strong assumption that the
/// high bits that are passed-through are not live.
-unsigned
-X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const {
+unsigned X86InstrInfo::getUndefRegClearance(const MachineInstr &MI,
+ unsigned OpNum) const {
const MachineOperand &MO = MI.getOperand(OpNum);
if (MO.getReg().isPhysical() && hasUndefRegUpdate(MI.getOpcode(), OpNum))
return Subtarget.getCLOpts().undef_reg_clearance;
@@ -7367,11 +7362,11 @@ X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned OpNum,
return 0;
}
-void X86InstrInfo::breakPartialRegDependency(
- MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
+void X86InstrInfo::breakPartialRegDependency(MachineInstr &MI,
+ unsigned OpNum) const {
Register Reg = MI.getOperand(OpNum).getReg();
// If MI kills this register, the false dependence is already broken.
- if (MI.killsRegister(Reg, TRI))
+ if (MI.killsRegister(Reg, &RI))
return;
if (X86::VR128RegClass.contains(Reg)) {
@@ -7381,16 +7376,16 @@ void X86InstrInfo::breakPartialRegDependency(
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(Opc), Reg)
.addReg(Reg, RegState::Undef)
.addReg(Reg, RegState::Undef);
- MI.addRegisterKilled(Reg, TRI, true);
+ MI.addRegisterKilled(Reg, &RI, true);
} else if (X86::VR256RegClass.contains(Reg)) {
// Use vxorps to clear the full ymm register.
// It wants to read and write the xmm sub-register.
- Register XReg = TRI->getSubReg(Reg, X86::sub_xmm);
+ Register XReg = RI.getSubReg(Reg, X86::sub_xmm);
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg)
.addReg(XReg, RegState::Undef)
.addReg(XReg, RegState::Undef)
.addReg(Reg, RegState::ImplicitDefine);
- MI.addRegisterKilled(Reg, TRI, true);
+ MI.addRegisterKilled(Reg, &RI, true);
} else if (X86::VR128XRegClass.contains(Reg)) {
// Only handle VLX targets.
if (!Subtarget.hasVLX())
@@ -7399,7 +7394,7 @@ void X86InstrInfo::breakPartialRegDependency(
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), Reg)
.addReg(Reg, RegState::Undef)
.addReg(Reg, RegState::Undef);
- MI.addRegisterKilled(Reg, TRI, true);
+ MI.addRegisterKilled(Reg, &RI, true);
} else if (X86::VR256XRegClass.contains(Reg) ||
X86::VR512RegClass.contains(Reg)) {
// Only handle VLX targets.
@@ -7407,26 +7402,26 @@ void X86InstrInfo::breakPartialRegDependency(
return;
// Use vpxord to clear the full ymm/zmm register.
// It wants to read and write the xmm sub-register.
- Register XReg = TRI->getSubReg(Reg, X86::sub_xmm);
+ Register XReg = RI.getSubReg(Reg, X86::sub_xmm);
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), XReg)
.addReg(XReg, RegState::Undef)
.addReg(XReg, RegState::Undef)
.addReg(Reg, RegState::ImplicitDefine);
- MI.addRegisterKilled(Reg, TRI, true);
+ MI.addRegisterKilled(Reg, &RI, true);
} else if (X86::GR64RegClass.contains(Reg)) {
// Using XOR32rr because it has shorter encoding and zeros up the upper bits
// as well.
- Register XReg = TRI->getSubReg(Reg, X86::sub_32bit);
+ Register XReg = RI.getSubReg(Reg, X86::sub_32bit);
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg)
.addReg(XReg, RegState::Undef)
.addReg(XReg, RegState::Undef)
.addReg(Reg, RegState::ImplicitDefine);
- MI.addRegisterKilled(Reg, TRI, true);
+ MI.addRegisterKilled(Reg, &RI, true);
} else if (X86::GR32RegClass.contains(Reg)) {
BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg)
.addReg(Reg, RegState::Undef)
.addReg(Reg, RegState::Undef);
- MI.addRegisterKilled(Reg, TRI, true);
+ MI.addRegisterKilled(Reg, &RI, true);
} else if ((X86::GR16RegClass.contains(Reg) ||
X86::GR8RegClass.contains(Reg)) &&
X86II::hasNewDataDest(MI.getDesc().TSFlags)) {
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index fe26940f85357..a01c4cbdb3223 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -444,21 +444,18 @@ class X86InstrInfo final : public X86GenInstrInfo {
int getJumpTableIndex(const MachineInstr &MI) const override;
std::optional<ExtAddrMode>
- getAddrModeFromMemoryOp(const MachineInstr &MemI,
- const TargetRegisterInfo *TRI) const override;
+ getAddrModeFromMemoryOp(const MachineInstr &MemI) const override;
bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
int64_t &ImmVal) const override;
bool preservesZeroValueInReg(const MachineInstr *MI,
- const Register NullValueReg,
- const TargetRegisterInfo *TRI) const override;
+ const Register NullValueReg) const override;
bool getMemOperandsWithOffsetWidth(
const MachineInstr &LdSt,
SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
- bool &OffsetIsScalable, LocationSize &Width,
- const TargetRegisterInfo *TRI) const override;
+ bool &OffsetIsScalable, LocationSize &Width) const override;
bool analyzeBranchPredicate(MachineBasicBlock &MBB,
TargetInstrInfo::MachineBranchPredicate &MBP,
bool AllowModify = false) const override;
@@ -584,13 +581,12 @@ class X86InstrInfo final : public X86GenInstrInfo {
bool setExecutionDomainCustom(MachineInstr &MI, unsigned Domain) const;
- unsigned
- getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const override;
- unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const override;
- void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum,
- const TargetRegisterInfo *TRI) const override;
+ unsigned getPartialRegUpdateClearance(const MachineInstr &MI,
+ unsigned OpNum) const override;
+ unsigned getUndefRegClearance(const MachineInstr &MI,
+ unsigned OpNum) const override;
+ void breakPartialRegDependency(MachineInstr &MI,
+ unsigned OpNum) const override;
MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
unsigned OpNum,
diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
index 3a70ceace00ec..a220cf939a38c 100644
--- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
+++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
@@ -208,7 +208,6 @@ TEST_P(RISCVInstrInfoTest, IsCopyInstrImpl) {
TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
const RISCVInstrInfo *TII = ST->getInstrInfo();
- const TargetRegisterInfo *TRI = ST->getRegisterInfo();
DebugLoc DL;
SmallVector<const MachineOperand *> BaseOps;
@@ -224,7 +223,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
.addMemOperand(MMO)
.getInstr();
bool Res = TII->getMemOperandsWithOffsetWidth(*MI, BaseOps, Offset,
- OffsetIsScalable, Width, TRI);
+ OffsetIsScalable, Width);
ASSERT_TRUE(Res);
ASSERT_EQ(BaseOps.size(), 1u);
ASSERT_TRUE(BaseOps.front()->isReg());
@@ -242,7 +241,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
.addImm(36)
.addMemOperand(MMO);
Res = TII->getMemOperandsWithOffsetWidth(*MI, BaseOps, Offset,
- OffsetIsScalable, Width, TRI);
+ OffsetIsScalable, Width);
ASSERT_TRUE(Res);
ASSERT_EQ(BaseOps.size(), 1u);
ASSERT_TRUE(BaseOps.front()->isReg());
@@ -258,7 +257,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
.addReg(RISCV::X3)
.addMemOperand(MMO);
Res = TII->getMemOperandsWithOffsetWidth(*MI, BaseOps, Offset,
- OffsetIsScalable, Width, TRI);
+ OffsetIsScalable, Width);
ASSERT_FALSE(Res); // Vector loads/stored are not handled for now.
BaseOps.clear();
@@ -266,7 +265,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
.addReg(RISCV::X5)
.addImm(16);
Res = TII->getMemOperandsWithOffsetWidth(*MI, BaseOps, Offset,
- OffsetIsScalable, Width, TRI);
+ OffsetIsScalable, Width);
BaseOps.clear();
MMO = MF->getMachineMemOperand(MachinePointerInfo(),
@@ -277,7 +276,7 @@ TEST_P(RISCVInstrInfoTest, GetMemOperandsWithOffsetWidth) {
.addImm(4)
.addMemOperand(MMO);
Res = TII->getMemOperandsWithOffsetWidth(*MI, BaseOps, Offset,
- OffsetIsScalable, Width, TRI);
+ OffsetIsScalable, Width);
ASSERT_TRUE(Res);
ASSERT_EQ(BaseOps.size(), 1u);
ASSERT_TRUE(BaseOps.front()->isFI());
>From 7e751154a0e5e05e0d527733a4968109d2f5aea2 Mon Sep 17 00:00:00 2001
From: Yair Ben Avraham <yairba at protonmail.com>
Date: Wed, 7 Oct 2026 11:43:06 +0300
Subject: [PATCH 18/52] [CIR][AArch64] Handle constrained Neon FMA and sqrt
(#218307)
Propagate expression FP options through AArch64 builtin emission and
attach the active FP environment to CIR `cir.fma` and `cir.sqrt`
operations. This allows strict FP operations to lower to constrained
LLVM intrinsics while preserving unconstrained behavior.
Co-locate classic Clang, CIR, and CIR-to-LLVM coverage under
command-line strict mode. For FP16 FMA and sqrt operations, also test a
pragma overriding a command-line maytrap setting.
Place this coverage in focused `-constrained.c` files alongside the
corresponding Neon test families. Move the matching classic constrained
checks from the legacy files and remove their superseded blocks, while
retaining unrelated legacy tests. Existing unconstrained coverage
remains in the regular Neon tests.
Track FMA operands through ABI conversions and lane selection, verify
the constrained results are returned, and check that lowering does not
fall back to generic CIR calls or unconstrained LLVM FMA and sqrt
intrinsics.
Assisted-by: Codex
Follow-up to #213800
---
.../lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp | 21 +-
.../AArch64/neon-intrinsics-constrained.c | 40 ---
.../CodeGen/AArch64/neon-misc-constrained.c | 22 --
.../neon-scalar-x-indexed-elem-constrained.c | 67 +----
.../AArch64/neon/fullfp16-constrained.c | 33 +++
.../fused-multiple-fullfp16-constrained.c | 155 ++++++++++
.../AArch64/neon/fused-multiply-constrained.c | 91 ++++++
.../AArch64/neon/intrinsics-constrained.c | 56 ++++
.../v8.2a-fp16-intrinsics-constrained.c | 16 -
.../v8.2a-neon-intrinsics-constrained.c | 280 +-----------------
10 files changed, 359 insertions(+), 422 deletions(-)
create mode 100644 clang/test/CodeGen/AArch64/neon/fullfp16-constrained.c
create mode 100644 clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16-constrained.c
create mode 100644 clang/test/CodeGen/AArch64/neon/fused-multiply-constrained.c
create mode 100644 clang/test/CodeGen/AArch64/neon/intrinsics-constrained.c
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
index 70253221c7b7d..af0c3c6b1c2d8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinAArch64.cpp
@@ -204,12 +204,26 @@ emitNeonCallToOp(CIRGenModule &cgm, CIRGenBuilderTy &builder,
builder.getStringAttr(intrinsicName.value()),
funcResTy, args)
.getResult();
- } else {
+ }
+ if constexpr (std::is_same_v<Operation, cir::FMAOp>) {
+ assert(args.size() == 3 && "fma expects three operands");
+ return Operation::create(builder, loc, funcResTy, args[0], args[1], args[2],
+ builder.getConstrainedFPAttr())
+ .getResult();
+ }
+ if constexpr (std::is_same_v<Operation, cir::SqrtOp>) {
+ assert(args.size() == 1 && "sqrt expects one operand");
+ return Operation::create(builder, loc, funcResTy, args[0],
+ builder.getConstrainedFPAttr())
+ .getResult();
+ }
+ if constexpr (!std::is_same_v<Operation, cir::LLVMIntrinsicCallOp> &&
+ !std::is_same_v<Operation, cir::FMAOp> &&
+ !std::is_same_v<Operation, cir::SqrtOp>)
return Operation::create(
builder, loc, mlir::TypeRange{funcResTy}, args,
cir::getDefaultProperties<Operation>(builder.getContext()))
.getResult();
- }
}
// TODO(cir): Remove `cgm` from the list of arguments once all NYI(s) are gone.
@@ -2709,6 +2723,8 @@ CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
// evaluation.
assert(!cir::MissingFeatures::msvcBuiltins());
+ CIRGenFPOptionsRAII fpOptsRAII(*this, expr);
+
// Some intrinsics are equivalent - if they are use the base intrinsic ID.
auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) {
return p.first == builtinID;
@@ -3630,7 +3646,6 @@ CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr,
}
case NEON::BI__builtin_neon_vsqrt_v:
case NEON::BI__builtin_neon_vsqrtq_v:
- assert(!cir::MissingFeatures::emitConstrainedFPCall());
return emitNeonCallToOp<cir::SqrtOp>(cgm, builder, {ty}, ops, std::nullopt,
ty, loc);
case NEON::BI__builtin_neon_vrbit_v:
diff --git a/clang/test/CodeGen/AArch64/neon-intrinsics-constrained.c b/clang/test/CodeGen/AArch64/neon-intrinsics-constrained.c
index 50a2a629ea55b..09f2aec4176b4 100644
--- a/clang/test/CodeGen/AArch64/neon-intrinsics-constrained.c
+++ b/clang/test/CodeGen/AArch64/neon-intrinsics-constrained.c
@@ -1418,46 +1418,6 @@ float64x1_t test_vmls_f64(float64x1_t a, float64x1_t b, float64x1_t c) {
return vmls_f64(a, b, c);
}
-// UNCONSTRAINED-LABEL: define dso_local <1 x double> @test_vfma_f64(
-// UNCONSTRAINED-SAME: <1 x double> noundef [[A:%.*]], <1 x double> noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <1 x double> [[A]] to i64
-// UNCONSTRAINED-NEXT: [[__P0_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP0]], i64 0
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <1 x double> [[B]] to i64
-// UNCONSTRAINED-NEXT: [[__P1_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i64 0
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <1 x double> [[C]] to i64
-// UNCONSTRAINED-NEXT: [[__P2_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP2]], i64 0
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <1 x i64> [[__P0_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <1 x i64> [[__P1_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <1 x i64> [[__P2_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <1 x double>
-// UNCONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <1 x double>
-// UNCONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <1 x double>
-// UNCONSTRAINED-NEXT: [[TMP9:%.*]] = call <1 x double> @llvm.fma.v1f64(<1 x double> [[TMP7]], <1 x double> [[TMP8]], <1 x double> [[TMP6]])
-// UNCONSTRAINED-NEXT: ret <1 x double> [[TMP9]]
-//
-// CONSTRAINED-LABEL: define dso_local <1 x double> @test_vfma_f64(
-// CONSTRAINED-SAME: <1 x double> noundef [[A:%.*]], <1 x double> noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <1 x double> [[A]] to i64
-// CONSTRAINED-NEXT: [[__P0_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP0]], i64 0
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <1 x double> [[B]] to i64
-// CONSTRAINED-NEXT: [[__P1_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i64 0
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <1 x double> [[C]] to i64
-// CONSTRAINED-NEXT: [[__P2_ADDR_I_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP2]], i64 0
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <1 x i64> [[__P0_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <1 x i64> [[__P1_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <1 x i64> [[__P2_ADDR_I_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <1 x double>
-// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <1 x double>
-// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <1 x double>
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <1 x double> @llvm.experimental.constrained.fma.v1f64(<1 x double> [[TMP7]], <1 x double> [[TMP8]], <1 x double> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR3]]
-// CONSTRAINED-NEXT: ret <1 x double> [[TMP9]]
-//
-float64x1_t test_vfma_f64(float64x1_t a, float64x1_t b, float64x1_t c) {
- return vfma_f64(a, b, c);
-}
-
// UNCONSTRAINED-LABEL: define dso_local <1 x double> @test_vfms_f64(
// UNCONSTRAINED-SAME: <1 x double> noundef [[A:%.*]], <1 x double> noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/neon-misc-constrained.c b/clang/test/CodeGen/AArch64/neon-misc-constrained.c
index 49208892e3035..f1f638cb272c6 100644
--- a/clang/test/CodeGen/AArch64/neon-misc-constrained.c
+++ b/clang/test/CodeGen/AArch64/neon-misc-constrained.c
@@ -82,28 +82,6 @@ float32x4_t test_vsqrtq_f32(float32x4_t a) {
}
-// UNCONSTRAINED-LABEL: define dso_local <2 x double> @test_vsqrtq_f64(
-// UNCONSTRAINED-SAME: <2 x double> noundef [[A:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <2 x double> [[A]] to <2 x i64>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x double>
-// UNCONSTRAINED-NEXT: [[VSQRT_I:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP2]])
-// UNCONSTRAINED-NEXT: ret <2 x double> [[VSQRT_I]]
-//
-// CONSTRAINED-LABEL: define dso_local <2 x double> @test_vsqrtq_f64(
-// CONSTRAINED-SAME: <2 x double> noundef [[A:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <2 x double> [[A]] to <2 x i64>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[TMP0]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[TMP1]] to <2 x double>
-// CONSTRAINED-NEXT: [[VSQRT_I:%.*]] = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(<2 x double> [[TMP2]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <2 x double> [[VSQRT_I]]
-//
-float64x2_t test_vsqrtq_f64(float64x2_t a) {
- return vsqrtq_f64(a);
-}
-
// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vcvt_f16_f32(
// UNCONSTRAINED-SAME: <4 x float> noundef [[A:%.*]]) #[[ATTR0]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/neon-scalar-x-indexed-elem-constrained.c b/clang/test/CodeGen/AArch64/neon-scalar-x-indexed-elem-constrained.c
index 944929ccb5f42..046bca2420a37 100644
--- a/clang/test/CodeGen/AArch64/neon-scalar-x-indexed-elem-constrained.c
+++ b/clang/test/CodeGen/AArch64/neon-scalar-x-indexed-elem-constrained.c
@@ -13,36 +13,18 @@
#include <arm_neon.h>
-// UNCONSTRAINED-LABEL: define dso_local float @test_vfmas_lane_f32(
-// UNCONSTRAINED-SAME: float noundef [[A:%.*]], float noundef [[B:%.*]], <2 x float> noundef [[C:%.*]]) #[[ATTR0:[0-9]+]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <2 x float> [[C]], i32 1
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = call float @llvm.fma.f32(float [[B]], float [[EXTRACT]], float [[A]])
-// UNCONSTRAINED-NEXT: ret float [[TMP0]]
-//
-// CONSTRAINED-LABEL: define dso_local float @test_vfmas_lane_f32(
-// CONSTRAINED-SAME: float noundef [[A:%.*]], float noundef [[B:%.*]], <2 x float> noundef [[C:%.*]]) #[[ATTR0:[0-9]+]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <2 x float> [[C]], i32 1
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = call float @llvm.experimental.constrained.fma.f32(float [[B]], float [[EXTRACT]], float [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2:[0-9]+]]
-// CONSTRAINED-NEXT: ret float [[TMP0]]
-//
-float32_t test_vfmas_lane_f32(float32_t a, float32_t b, float32x2_t c) {
- return vfmas_lane_f32(a, b, c, 1);
-}
-
// UNCONSTRAINED-LABEL: define dso_local double @test_vfmad_lane_f64(
-// UNCONSTRAINED-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0]] {
+// UNCONSTRAINED-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0:[0-9]+]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
// UNCONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <1 x double> [[C]], i32 0
// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = call double @llvm.fma.f64(double [[B]], double [[EXTRACT]], double [[A]])
// UNCONSTRAINED-NEXT: ret double [[TMP0]]
//
// CONSTRAINED-LABEL: define dso_local double @test_vfmad_lane_f64(
-// CONSTRAINED-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0]] {
+// CONSTRAINED-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]], <1 x double> noundef [[C:%.*]]) #[[ATTR0:[0-9]+]] {
// CONSTRAINED-NEXT: [[ENTRY:.*:]]
// CONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <1 x double> [[C]], i32 0
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = call double @llvm.experimental.constrained.fma.f64(double [[B]], double [[EXTRACT]], double [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
+// CONSTRAINED-NEXT: [[TMP0:%.*]] = call double @llvm.experimental.constrained.fma.f64(double [[B]], double [[EXTRACT]], double [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2:[0-9]+]]
// CONSTRAINED-NEXT: ret double [[TMP0]]
//
float64_t test_vfmad_lane_f64(float64_t a, float64_t b, float64x1_t c) {
@@ -173,48 +155,6 @@ float64x1_t test_vfms_lane_f64(float64x1_t a, float64x1_t b, float64x1_t v) {
return vfms_lane_f64(a, b, v, 0);
}
-// UNCONSTRAINED-LABEL: define dso_local <1 x double> @test_vfma_laneq_f64(
-// UNCONSTRAINED-SAME: <1 x double> noundef [[A:%.*]], <1 x double> noundef [[B:%.*]], <2 x double> noundef [[V:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <1 x double> [[A]] to i64
-// UNCONSTRAINED-NEXT: [[__S0_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP0]], i64 0
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <1 x double> [[B]] to i64
-// UNCONSTRAINED-NEXT: [[__S1_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i64 0
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <2 x double> [[V]] to <2 x i64>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <1 x i64> [[__S0_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <1 x i64> [[__S1_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to double
-// UNCONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to double
-// UNCONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <2 x double>
-// UNCONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <2 x double> [[TMP8]], i32 0
-// UNCONSTRAINED-NEXT: [[TMP9:%.*]] = call double @llvm.fma.f64(double [[TMP7]], double [[EXTRACT]], double [[TMP6]])
-// UNCONSTRAINED-NEXT: [[TMP10:%.*]] = bitcast double [[TMP9]] to <1 x double>
-// UNCONSTRAINED-NEXT: ret <1 x double> [[TMP10]]
-//
-// CONSTRAINED-LABEL: define dso_local <1 x double> @test_vfma_laneq_f64(
-// CONSTRAINED-SAME: <1 x double> noundef [[A:%.*]], <1 x double> noundef [[B:%.*]], <2 x double> noundef [[V:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <1 x double> [[A]] to i64
-// CONSTRAINED-NEXT: [[__S0_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP0]], i64 0
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <1 x double> [[B]] to i64
-// CONSTRAINED-NEXT: [[__S1_SROA_0_0_VEC_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i64 0
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <2 x double> [[V]] to <2 x i64>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <1 x i64> [[__S0_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <1 x i64> [[__S1_SROA_0_0_VEC_INSERT]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <2 x i64> [[TMP2]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to double
-// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to double
-// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <2 x double>
-// CONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <2 x double> [[TMP8]], i32 0
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call double @llvm.experimental.constrained.fma.f64(double [[TMP7]], double [[EXTRACT]], double [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: [[TMP10:%.*]] = bitcast double [[TMP9]] to <1 x double>
-// CONSTRAINED-NEXT: ret <1 x double> [[TMP10]]
-//
-float64x1_t test_vfma_laneq_f64(float64x1_t a, float64x1_t b, float64x2_t v) {
- return vfma_laneq_f64(a, b, v, 0);
-}
-
// UNCONSTRAINED-LABEL: define dso_local <1 x double> @test_vfms_laneq_f64(
// UNCONSTRAINED-SAME: <1 x double> noundef [[A:%.*]], <1 x double> noundef [[B:%.*]], <2 x double> noundef [[V:%.*]]) #[[ATTR0]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
@@ -258,4 +198,3 @@ float64x1_t test_vfma_laneq_f64(float64x1_t a, float64x1_t b, float64x2_t v) {
float64x1_t test_vfms_laneq_f64(float64x1_t a, float64x1_t b, float64x2_t v) {
return vfms_laneq_f64(a, b, v, 0);
}
-
diff --git a/clang/test/CodeGen/AArch64/neon/fullfp16-constrained.c b/clang/test/CodeGen/AArch64/neon/fullfp16-constrained.c
new file mode 100644
index 0000000000000..1a668bd657e0d
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/neon/fullfp16-constrained.c
@@ -0,0 +1,33 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1_cg_arm64_neon -target-feature +fullfp16 -ffp-exception-behavior=strict -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,simplifycfg | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=' @llvm.fma.' --implicit-check-not=' @llvm.sqrt.'
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=maytrap -DEXCEPT=1 -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,simplifycfg | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=fpexcept.maytrap --implicit-check-not=' @llvm.fma.' --implicit-check-not=' @llvm.sqrt.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=maytrap -DEXCEPT=1 -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefixes=ALL,CIR --implicit-check-not='except_mode = maytrap' --implicit-check-not='cir.call_llvm_intrinsic "fma"' --implicit-check-not='cir.call_llvm_intrinsic "sqrt"' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,simplifycfg | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=' @llvm.fma.' --implicit-check-not=' @llvm.sqrt.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefixes=ALL,CIR --implicit-check-not='cir.call_llvm_intrinsic "fma"' --implicit-check-not='cir.call_llvm_intrinsic "sqrt"' %}
+
+#if EXCEPT
+#pragma float_control(except, on)
+#endif
+
+#include <arm_fp16.h>
+
+// ALL-LABEL: @test_vsqrth_f16(
+float16_t test_vsqrth_f16(float16_t a) {
+// CIR: cir.sqrt %{{.*}} : !cir.f16 fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: half {{.*}} [[A:%.*]]) {{.*}} {
+// LLVM: [[SQRT:%.*]] = call half @llvm.experimental.constrained.sqrt.f16(half [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret half [[SQRT]]
+ return vsqrth_f16(a);
+}
+
+// ALL-LABEL: @test_vfmah_f16(
+float16_t test_vfmah_f16(float16_t a, float16_t b, float16_t c) {
+// CIR: cir.fma %{{.*}}, %{{.*}}, %{{.*}} : !cir.f16 fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: half {{.*}} [[A:%.*]], half {{.*}} [[B:%.*]], half {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[FMA:%.*]] = call half @llvm.experimental.constrained.fma.f16(half [[B]], half [[C]], half [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret half [[FMA]]
+ return vfmah_f16(a, b, c);
+}
diff --git a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16-constrained.c b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16-constrained.c
new file mode 100644
index 0000000000000..a56cb34014f15
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16-constrained.c
@@ -0,0 +1,155 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1_cg_arm64_neon -target-feature +fullfp16 -ffp-exception-behavior=maytrap -DEXCEPT=1 -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=fpexcept.maytrap --implicit-check-not=' @llvm.fma.'
+// RUN: %clang_cc1_cg_arm64_neon -target-feature +fullfp16 -ffp-exception-behavior=strict -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=' @llvm.fma.'
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=maytrap -DEXCEPT=1 -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=fpexcept.maytrap --implicit-check-not=' @llvm.fma.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=maytrap -DEXCEPT=1 -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefixes=ALL,CIR --implicit-check-not='except_mode = maytrap' --implicit-check-not='cir.call_llvm_intrinsic "fma"' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=' @llvm.fma.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefixes=ALL,CIR --implicit-check-not='cir.call_llvm_intrinsic "fma"' %}
+
+#if EXCEPT
+#pragma float_control(except, on)
+#endif
+
+#include <arm_neon.h>
+
+// LLVM-LABEL: @test_vfma_f16(
+// CIR-LABEL: @vfma_f16(
+float16x4_t test_vfma_f16(float16x4_t a, float16x4_t b, float16x4_t c) {
+// CIR: cir.fma %{{.*}}, %{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
+// LLVM: [[B_I:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
+// LLVM: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half>
+// LLVM: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half>
+// LLVM: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half>
+// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[B_CAST]], <4 x half> [[C_CAST]], <4 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <4 x half> [[FMA]]
+ return vfma_f16(a, b, c);
+}
+
+// LLVM-LABEL: @test_vfmaq_f16(
+// CIR-LABEL: @vfmaq_f16(
+float16x8_t test_vfmaq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
+// CIR: cir.fma %{{.*}}, %{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
+// LLVM: [[B_I:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
+// LLVM: [[C_I:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half>
+// LLVM: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half>
+// LLVM: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half>
+// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.experimental.constrained.fma.v8f16(<8 x half> [[B_CAST]], <8 x half> [[C_CAST]], <8 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <8 x half> [[FMA]]
+ return vfmaq_f16(a, b, c);
+}
+
+// ALL-LABEL: @test_vfma_lane_f16(
+float16x4_t test_vfma_lane_f16(float16x4_t a, float16x4_t b,
+ float16x4_t c) {
+// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<4 x !cir.f16>
+// CIR: cir.fma %{{.*}}, [[LANE]], %{{.*}} : !cir.vector<4 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
+// LLVM: [[B_I:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
+// LLVM: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8>
+// LLVM-DAG: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half>
+// LLVM-DAG: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half>
+// LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half>
+// LLVM-DAG: [[LANE:%.*]] = shufflevector <4 x half> [[C_CAST]], <4 x half> {{.*}}, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[B_CAST]], <4 x half> [[LANE]], <4 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <4 x half> [[FMA]]
+ return vfma_lane_f16(a, b, c, 3);
+}
+
+// ALL-LABEL: @test_vfmaq_lane_f16(
+float16x8_t test_vfmaq_lane_f16(float16x8_t a, float16x8_t b,
+ float16x4_t c) {
+// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<8 x !cir.f16>
+// CIR: cir.fma %{{.*}}, [[LANE]], %{{.*}} : !cir.vector<8 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
+// LLVM: [[B_I:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
+// LLVM: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8>
+// LLVM-DAG: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half>
+// LLVM-DAG: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half>
+// LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half>
+// LLVM-DAG: [[LANE:%.*]] = shufflevector <4 x half> [[C_CAST]], <4 x half> {{.*}}, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.experimental.constrained.fma.v8f16(<8 x half> [[B_CAST]], <8 x half> [[LANE]], <8 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <8 x half> [[FMA]]
+ return vfmaq_lane_f16(a, b, c, 3);
+}
+
+// ALL-LABEL: @test_vfma_laneq_f16(
+float16x4_t test_vfma_laneq_f16(float16x4_t a, float16x4_t b,
+ float16x8_t c) {
+// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<4 x !cir.f16>
+// CIR: cir.fma [[LANE]], %{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
+// LLVM: [[B_I:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
+// LLVM: [[C_I:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half>
+// LLVM: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half>
+// LLVM: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half>
+// LLVM: [[LANE:%.*]] = shufflevector <8 x half> [[C_CAST]], <8 x half> {{.*}}, <4 x i32> <i32 7, i32 7, i32 7, i32 7>
+// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[LANE]], <4 x half> [[B_CAST]], <4 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <4 x half> [[FMA]]
+ return vfma_laneq_f16(a, b, c, 7);
+}
+
+// ALL-LABEL: @test_vfmaq_laneq_f16(
+float16x8_t test_vfmaq_laneq_f16(float16x8_t a, float16x8_t b,
+ float16x8_t c) {
+// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<8 x !cir.f16>
+// CIR: cir.fma [[LANE]], %{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
+// LLVM: [[B_I:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
+// LLVM: [[C_I:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half>
+// LLVM: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half>
+// LLVM: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half>
+// LLVM: [[LANE:%.*]] = shufflevector <8 x half> [[C_CAST]], <8 x half> {{.*}}, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
+// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.experimental.constrained.fma.v8f16(<8 x half> [[LANE]], <8 x half> [[B_CAST]], <8 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <8 x half> [[FMA]]
+ return vfmaq_laneq_f16(a, b, c, 7);
+}
+
+// ALL-LABEL: @test_vfmah_lane_f16(
+float16_t test_vfmah_lane_f16(float16_t a, float16_t b, float16x4_t c) {
+// CIR: [[INDEX:%.*]] = cir.const #cir.int<3> : !u64i
+// CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}{{\[}}[[INDEX]] : !u64i] : !cir.vector<4 x !cir.f16>
+// CIR: cir.fma %{{.*}}, [[LANE]], %{{.*}} : !cir.f16 fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: half {{.*}} [[A:%.*]], half {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[LANE:%.*]] = extractelement <4 x half> [[C]], i{{32|64}} 3
+// LLVM: [[FMA:%.*]] = call half @llvm.experimental.constrained.fma.f16(half [[B]], half [[LANE]], half [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret half [[FMA]]
+ return vfmah_lane_f16(a, b, c, 3);
+}
diff --git a/clang/test/CodeGen/AArch64/neon/fused-multiply-constrained.c b/clang/test/CodeGen/AArch64/neon/fused-multiply-constrained.c
new file mode 100644
index 0000000000000..52ee3f2062851
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/neon/fused-multiply-constrained.c
@@ -0,0 +1,91 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1_cg_arm64_neon -ffp-exception-behavior=strict -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefixes=ALL,LLVM
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefixes=ALL,LLVM --implicit-check-not=' @llvm.fma.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefixes=ALL,CIR --implicit-check-not='cir.call_llvm_intrinsic "fma"' %}
+
+#include <arm_neon.h>
+
+// LLVM-LABEL: @test_vfma_f64(
+// CIR-LABEL: @vfma_f64(
+float64x1_t test_vfma_f64(float64x1_t a, float64x1_t b, float64x1_t c) {
+// CIR: cir.fma %{{.*}}, %{{.*}}, %{{.*}} : !cir.vector<1 x !cir.double> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <1 x double> {{.*}} [[A:%.*]], <1 x double> {{.*}} [[B:%.*]], <1 x double> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <1 x double> [[A]] to i64
+// LLVM: [[A_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[A_I]], i64 0
+// LLVM: [[B_I:%.*]] = bitcast <1 x double> [[B]] to i64
+// LLVM: [[B_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[B_I]], i64 0
+// LLVM: [[C_I:%.*]] = bitcast <1 x double> [[C]] to i64
+// LLVM: [[C_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[C_I]], i64 0
+// LLVM: [[A_BYTES:%.*]] = bitcast <1 x i64> [[A_INSERT]] to <8 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <1 x i64> [[B_INSERT]] to <8 x i8>
+// LLVM: [[C_BYTES:%.*]] = bitcast <1 x i64> [[C_INSERT]] to <8 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <1 x double>
+// LLVM: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <1 x double>
+// LLVM: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <1 x double>
+// LLVM: [[FMA:%.*]] = call <1 x double> @llvm.experimental.constrained.fma.v1f64(<1 x double> [[B_CAST]], <1 x double> [[C_CAST]], <1 x double> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <1 x double> [[FMA]]
+ return vfma_f64(a, b, c);
+}
+
+// ALL-LABEL: @test_vfma_laneq_f64(
+float64x1_t test_vfma_laneq_f64(float64x1_t a, float64x1_t b,
+ float64x2_t v) {
+// CIR: [[INDEX:%.*]] = cir.const #cir.int<0> : !u64i
+// CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}{{\[}}[[INDEX]] : !u64i] : !cir.vector<2 x !cir.double>
+// CIR: cir.fma %{{.*}}, [[LANE]], %{{.*}} : !cir.double fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <1 x double> {{.*}} [[A:%.*]], <1 x double> {{.*}} [[B:%.*]], <2 x double> {{.*}} [[V:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <1 x double> [[A]] to i64
+// LLVM: [[A_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[A_I]], i64 0
+// LLVM: [[B_I:%.*]] = bitcast <1 x double> [[B]] to i64
+// LLVM: [[B_INSERT:%.*]] = insertelement <1 x i64> undef, i64 [[B_I]], i64 0
+// LLVM: [[V_I:%.*]] = bitcast <2 x double> [[V]] to <2 x i64>
+// LLVM: [[A_BYTES:%.*]] = bitcast <1 x i64> [[A_INSERT]] to <8 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <1 x i64> [[B_INSERT]] to <8 x i8>
+// LLVM: [[V_BYTES:%.*]] = bitcast <2 x i64> [[V_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to double
+// LLVM: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to double
+// LLVM: [[V_CAST:%.*]] = bitcast <16 x i8> [[V_BYTES]] to <2 x double>
+// LLVM: [[LANE:%.*]] = extractelement <2 x double> [[V_CAST]], i{{32|64}} 0
+// LLVM: [[FMA:%.*]] = call double @llvm.experimental.constrained.fma.f64(double [[B_CAST]], double [[LANE]], double [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: [[RESULT:%.*]] = bitcast double [[FMA]] to <1 x double>
+// LLVM: ret <1 x double> [[RESULT]]
+ return vfma_laneq_f64(a, b, v, 0);
+}
+
+// ALL-LABEL: @test_vfmaq_laneq_f64(
+float64x2_t test_vfmaq_laneq_f64(float64x2_t a, float64x2_t b,
+ float64x2_t v) {
+// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<2 x !cir.double>) [#cir.int<1> : !s32i, #cir.int<1> : !s32i] : !cir.vector<2 x !cir.double>
+// CIR: cir.fma [[LANE]], %{{.*}}, %{{.*}} : !cir.vector<2 x !cir.double> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <2 x double> {{.*}} [[A:%.*]], <2 x double> {{.*}} [[B:%.*]], <2 x double> {{.*}} [[V:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <2 x double> [[A]] to <2 x i64>
+// LLVM: [[B_I:%.*]] = bitcast <2 x double> [[B]] to <2 x i64>
+// LLVM: [[V_I:%.*]] = bitcast <2 x double> [[V]] to <2 x i64>
+// LLVM: [[A_BYTES:%.*]] = bitcast <2 x i64> [[A_I]] to <16 x i8>
+// LLVM: [[B_BYTES:%.*]] = bitcast <2 x i64> [[B_I]] to <16 x i8>
+// LLVM: [[V_BYTES:%.*]] = bitcast <2 x i64> [[V_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <2 x double>
+// LLVM: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <2 x double>
+// LLVM: [[V_CAST:%.*]] = bitcast <16 x i8> [[V_BYTES]] to <2 x double>
+// LLVM: [[LANE:%.*]] = shufflevector <2 x double> [[V_CAST]], <2 x double> {{.*}}, <2 x i32> <i32 1, i32 1>
+// LLVM: [[FMA:%.*]] = call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> [[LANE]], <2 x double> [[B_CAST]], <2 x double> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <2 x double> [[FMA]]
+ return vfmaq_laneq_f64(a, b, v, 1);
+}
+
+// ALL-LABEL: @test_vfmas_lane_f32(
+float32_t test_vfmas_lane_f32(float32_t a, float32_t b, float32x2_t c) {
+// CIR: [[INDEX:%.*]] = cir.const #cir.int<1> : !u64i
+// CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}{{\[}}[[INDEX]] : !u64i] : !cir.vector<2 x !cir.float>
+// CIR: cir.fma %{{.*}}, [[LANE]], %{{.*}} : !cir.float fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: float {{.*}} [[A:%.*]], float {{.*}} [[B:%.*]], <2 x float> {{.*}} [[C:%.*]]) {{.*}} {
+// LLVM: [[LANE:%.*]] = extractelement <2 x float> [[C]], i{{32|64}} 1
+// LLVM: [[FMA:%.*]] = call float @llvm.experimental.constrained.fma.f32(float [[B]], float [[LANE]], float [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret float [[FMA]]
+ return vfmas_lane_f32(a, b, c, 1);
+}
diff --git a/clang/test/CodeGen/AArch64/neon/intrinsics-constrained.c b/clang/test/CodeGen/AArch64/neon/intrinsics-constrained.c
new file mode 100644
index 0000000000000..ef915a0ccea1e
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/neon/intrinsics-constrained.c
@@ -0,0 +1,56 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1_cg_arm64_neon -target-feature +fullfp16 -ffp-exception-behavior=maytrap -DEXCEPT=1 -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefix=LLVM --implicit-check-not=fpexcept.maytrap --implicit-check-not=' @llvm.sqrt.'
+// RUN: %clang_cc1_cg_arm64_neon -target-feature +fullfp16 -ffp-exception-behavior=strict -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefix=LLVM --implicit-check-not=' @llvm.sqrt.'
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=maytrap -DEXCEPT=1 -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefix=LLVM --implicit-check-not=fpexcept.maytrap --implicit-check-not=' @llvm.sqrt.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=maytrap -DEXCEPT=1 -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefix=CIR --implicit-check-not='except_mode = maytrap' --implicit-check-not='cir.call_llvm_intrinsic "sqrt"' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-llvm %s -disable-O0-optnone | opt -S -passes=mem2reg,sroa | FileCheck %s --check-prefix=LLVM --implicit-check-not=' @llvm.sqrt.' %}
+// RUN: %if cir-enabled %{%clang_cc1_cg_arm64_neon -target-feature +fullfp16 -fexperimental-strict-floating-point -ffp-exception-behavior=strict -fclangir -emit-cir %s -disable-O0-optnone | FileCheck %s --check-prefix=CIR --implicit-check-not='cir.call_llvm_intrinsic "sqrt"' %}
+
+#if EXCEPT
+#pragma float_control(except, on)
+#endif
+
+#include <arm_neon.h>
+
+// LLVM-LABEL: @test_vsqrt_f16(
+// CIR-LABEL: @vsqrt_f16(
+float16x4_t test_vsqrt_f16(float16x4_t a) {
+// CIR: cir.sqrt %{{.*}} : !cir.vector<4 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half>
+// LLVM: [[SQRT:%.*]] = call <4 x half> @llvm.experimental.constrained.sqrt.v4f16(<4 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <4 x half> [[SQRT]]
+ return vsqrt_f16(a);
+}
+
+// LLVM-LABEL: @test_vsqrtq_f16(
+// CIR-LABEL: @vsqrtq_f16(
+float16x8_t test_vsqrtq_f16(float16x8_t a) {
+// CIR: cir.sqrt %{{.*}} : !cir.vector<8 x !cir.f16> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
+// LLVM: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half>
+// LLVM: [[SQRT:%.*]] = call <8 x half> @llvm.experimental.constrained.sqrt.v8f16(<8 x half> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <8 x half> [[SQRT]]
+ return vsqrtq_f16(a);
+}
+
+// LLVM-LABEL: @test_vsqrtq_f64(
+// CIR-LABEL: @vsqrtq_f64(
+float64x2_t test_vsqrtq_f64(float64x2_t a) {
+// CIR: cir.sqrt %{{.*}} : !cir.vector<2 x !cir.double> fenv<dynamic_rounding_mode = tonearest, except_mode = unknown, strict_except = true>
+
+// LLVM-SAME: <2 x double> {{.*}} [[A:%.*]]) {{.*}} {
+// LLVM: [[A_I:%.*]] = bitcast <2 x double> [[A]] to <2 x i64>
+// LLVM: [[A_BYTES:%.*]] = bitcast <2 x i64> [[A_I]] to <16 x i8>
+// LLVM: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <2 x double>
+// LLVM: [[SQRT:%.*]] = call <2 x double> @llvm.experimental.constrained.sqrt.v2f64(<2 x double> [[A_CAST]], metadata !"round.tonearest", metadata !"fpexcept.strict")
+// LLVM: ret <2 x double> [[SQRT]]
+ return vsqrtq_f64(a);
+}
diff --git a/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics-constrained.c b/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics-constrained.c
index c9d5071c4eb83..4006171e40296 100644
--- a/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics-constrained.c
+++ b/clang/test/CodeGen/AArch64/v8.2a-fp16-intrinsics-constrained.c
@@ -200,14 +200,6 @@ float16_t test_vrndxh_f16(float16_t a) {
return vrndxh_f16(a);
}
-// COMMON-LABEL: test_vsqrth_f16
-// UNCONSTRAINED: [[SQR:%.*]] = call half @llvm.sqrt.f16(half %a)
-// CONSTRAINED: [[SQR:%.*]] = call half @llvm.experimental.constrained.sqrt.f16(half %a, metadata !"round.tonearest", metadata !"fpexcept.strict")
-// COMMONIR: ret half [[SQR]]
-float16_t test_vsqrth_f16(float16_t a) {
- return vsqrth_f16(a);
-}
-
// COMMON-LABEL: test_vaddh_f16
// UNCONSTRAINED: [[ADD:%.*]] = fadd half %a, %b
// CONSTRAINED: [[ADD:%.*]] = call half @llvm.experimental.constrained.fadd.f16(half %a, half %b, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -285,14 +277,6 @@ float16_t test_vsubh_f16(float16_t a, float16_t b) {
return vsubh_f16(a, b);
}
-// COMMON-LABEL: test_vfmah_f16
-// UNCONSTRAINED: [[FMA:%.*]] = call half @llvm.fma.f16(half %b, half %c, half %a)
-// CONSTRAINED: [[FMA:%.*]] = call half @llvm.experimental.constrained.fma.f16(half %b, half %c, half %a, metadata !"round.tonearest", metadata !"fpexcept.strict")
-// COMMONIR: ret half [[FMA]]
-float16_t test_vfmah_f16(float16_t a, float16_t b, float16_t c) {
- return vfmah_f16(a, b, c);
-}
-
// COMMON-LABEL: test_vfmsh_f16
// COMMONIR: [[SUB:%.*]] = fneg half %b
// UNCONSTRAINED: [[ADD:%.*]] = call half @llvm.fma.f16(half [[SUB]], half %c, half %a)
diff --git a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.c b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.c
index dac8b931ff210..a30bfa8352646 100644
--- a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.c
+++ b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics-constrained.c
@@ -21,120 +21,8 @@
#include <arm_neon.h>
-// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vsqrt_f16(
-// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[VSQRT_I:%.*]] = call <4 x half> @llvm.sqrt.v4f16(<4 x half> [[TMP2]])
-// UNCONSTRAINED-NEXT: ret <4 x half> [[VSQRT_I]]
-//
-// CONSTRAINED-LABEL: define dso_local <4 x half> @test_vsqrt_f16(
-// CONSTRAINED-SAME: <4 x half> noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CONSTRAINED-NEXT: [[VSQRT_I:%.*]] = call <4 x half> @llvm.experimental.constrained.sqrt.v4f16(<4 x half> [[TMP2]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2:[0-9]+]]
-// CONSTRAINED-NEXT: ret <4 x half> [[VSQRT_I]]
-//
-float16x4_t test_vsqrt_f16(float16x4_t a) {
- return vsqrt_f16(a);
-}
-
-// UNCONSTRAINED-LABEL: define dso_local <8 x half> @test_vsqrtq_f16(
-// UNCONSTRAINED-SAME: <8 x half> noundef [[A:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[VSQRT_I:%.*]] = call <8 x half> @llvm.sqrt.v8f16(<8 x half> [[TMP2]])
-// UNCONSTRAINED-NEXT: ret <8 x half> [[VSQRT_I]]
-//
-// CONSTRAINED-LABEL: define dso_local <8 x half> @test_vsqrtq_f16(
-// CONSTRAINED-SAME: <8 x half> noundef [[A:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CONSTRAINED-NEXT: [[VSQRT_I:%.*]] = call <8 x half> @llvm.experimental.constrained.sqrt.v8f16(<8 x half> [[TMP2]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <8 x half> [[VSQRT_I]]
-//
-float16x8_t test_vsqrtq_f16(float16x8_t a) {
- return vsqrtq_f16(a);
-}
-
-// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_f16(
-// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]])
-// UNCONSTRAINED-NEXT: ret <4 x half> [[TMP9]]
-//
-// CONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_f16(
-// CONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
-// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <4 x half> [[TMP9]]
-//
-float16x4_t test_vfma_f16(float16x4_t a, float16x4_t b, float16x4_t c) {
- return vfma_f16(a, b, c);
-}
-
-// UNCONSTRAINED-LABEL: define dso_local <8 x half> @test_vfmaq_f16(
-// UNCONSTRAINED-SAME: <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[TMP7]], <8 x half> [[TMP8]], <8 x half> [[TMP6]])
-// UNCONSTRAINED-NEXT: ret <8 x half> [[TMP9]]
-//
-// CONSTRAINED-LABEL: define dso_local <8 x half> @test_vfmaq_f16(
-// CONSTRAINED-SAME: <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
-// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.experimental.constrained.fma.v8f16(<8 x half> [[TMP7]], <8 x half> [[TMP8]], <8 x half> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <8 x half> [[TMP9]]
-//
-float16x8_t test_vfmaq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
- return vfmaq_f16(a, b, c);
-}
-
// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vfms_f16(
-// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
+// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0:[0-9]+]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
// UNCONSTRAINED-NEXT: [[FNEG_I:%.*]] = fneg <4 x half> [[B]]
// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
@@ -150,7 +38,7 @@ float16x8_t test_vfmaq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
// UNCONSTRAINED-NEXT: ret <4 x half> [[TMP9]]
//
// CONSTRAINED-LABEL: define dso_local <4 x half> @test_vfms_f16(
-// CONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
+// CONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0:[0-9]+]] {
// CONSTRAINED-NEXT: [[ENTRY:.*:]]
// CONSTRAINED-NEXT: [[FNEG_I:%.*]] = fneg <4 x half> [[B]]
// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
@@ -162,7 +50,7 @@ float16x8_t test_vfmaq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
+// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2:[0-9]+]]
// CONSTRAINED-NEXT: ret <4 x half> [[TMP9]]
//
float16x4_t test_vfms_f16(float16x4_t a, float16x4_t b, float16x4_t c) {
@@ -205,150 +93,6 @@ float16x8_t test_vfmsq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
return vfmsq_f16(a, b, c);
}
-// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_lane_f16(
-// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP6]], <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-// UNCONSTRAINED-NEXT: [[FMLA:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[FMLA1:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[FMLA2:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[FMLA]], <4 x half> [[LANE]], <4 x half> [[FMLA1]])
-// UNCONSTRAINED-NEXT: ret <4 x half> [[FMLA2]]
-//
-// CONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_lane_f16(
-// CONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// CONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP6]], <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-// CONSTRAINED-NEXT: [[FMLA:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
-// CONSTRAINED-NEXT: [[FMLA1:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CONSTRAINED-NEXT: [[FMLA2:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[FMLA]], <4 x half> [[LANE]], <4 x half> [[FMLA1]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <4 x half> [[FMLA2]]
-//
-float16x4_t test_vfma_lane_f16(float16x4_t a, float16x4_t b, float16x4_t c) {
- return vfma_lane_f16(a, b, c, 3);
-}
-
-// UNCONSTRAINED-LABEL: define dso_local <8 x half> @test_vfmaq_lane_f16(
-// UNCONSTRAINED-SAME: <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP6]], <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-// UNCONSTRAINED-NEXT: [[FMLA:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[FMLA1:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[FMLA2:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[FMLA]], <8 x half> [[LANE]], <8 x half> [[FMLA1]])
-// UNCONSTRAINED-NEXT: ret <8 x half> [[FMLA2]]
-//
-// CONSTRAINED-LABEL: define dso_local <8 x half> @test_vfmaq_lane_f16(
-// CONSTRAINED-SAME: <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half>
-// CONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP6]], <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-// CONSTRAINED-NEXT: [[FMLA:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
-// CONSTRAINED-NEXT: [[FMLA1:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CONSTRAINED-NEXT: [[FMLA2:%.*]] = call <8 x half> @llvm.experimental.constrained.fma.v8f16(<8 x half> [[FMLA]], <8 x half> [[LANE]], <8 x half> [[FMLA1]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <8 x half> [[FMLA2]]
-//
-float16x8_t test_vfmaq_lane_f16(float16x8_t a, float16x8_t b, float16x4_t c) {
- return vfmaq_lane_f16(a, b, c, 3);
-}
-
-// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_laneq_f16(
-// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
-// UNCONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <8 x half> [[TMP8]], <8 x half> [[TMP8]], <4 x i32> <i32 7, i32 7, i32 7, i32 7>
-// UNCONSTRAINED-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[LANE]], <4 x half> [[TMP7]], <4 x half> [[TMP6]])
-// UNCONSTRAINED-NEXT: ret <4 x half> [[TMP9]]
-//
-// CONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_laneq_f16(
-// CONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half>
-// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half>
-// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
-// CONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <8 x half> [[TMP8]], <8 x half> [[TMP8]], <4 x i32> <i32 7, i32 7, i32 7, i32 7>
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> [[LANE]], <4 x half> [[TMP7]], <4 x half> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <4 x half> [[TMP9]]
-//
-float16x4_t test_vfma_laneq_f16(float16x4_t a, float16x4_t b, float16x8_t c) {
- return vfma_laneq_f16(a, b, c, 7);
-}
-
-// UNCONSTRAINED-LABEL: define dso_local <8 x half> @test_vfmaq_laneq_f16(
-// UNCONSTRAINED-SAME: <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
-// UNCONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
-// UNCONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
-// UNCONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <8 x half> [[TMP8]], <8 x half> [[TMP8]], <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
-// UNCONSTRAINED-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[LANE]], <8 x half> [[TMP7]], <8 x half> [[TMP6]])
-// UNCONSTRAINED-NEXT: ret <8 x half> [[TMP9]]
-//
-// CONSTRAINED-LABEL: define dso_local <8 x half> @test_vfmaq_laneq_f16(
-// CONSTRAINED-SAME: <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16>
-// CONSTRAINED-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
-// CONSTRAINED-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
-// CONSTRAINED-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
-// CONSTRAINED-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
-// CONSTRAINED-NEXT: [[LANE:%.*]] = shufflevector <8 x half> [[TMP8]], <8 x half> [[TMP8]], <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
-// CONSTRAINED-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.experimental.constrained.fma.v8f16(<8 x half> [[LANE]], <8 x half> [[TMP7]], <8 x half> [[TMP6]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret <8 x half> [[TMP9]]
-//
-float16x8_t test_vfmaq_laneq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
- return vfmaq_laneq_f16(a, b, c, 7);
-}
-
// UNCONSTRAINED-LABEL: define dso_local <4 x half> @test_vfma_n_f16(
// UNCONSTRAINED-SAME: <4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], half noundef [[C:%.*]]) #[[ATTR0]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
@@ -441,24 +185,6 @@ float16x8_t test_vfmaq_n_f16(float16x8_t a, float16x8_t b, float16_t c) {
return vfmaq_n_f16(a, b, c);
}
-// UNCONSTRAINED-LABEL: define dso_local half @test_vfmah_lane_f16(
-// UNCONSTRAINED-SAME: half noundef [[A:%.*]], half noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
-// UNCONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <4 x half> [[C]], i32 3
-// UNCONSTRAINED-NEXT: [[TMP0:%.*]] = call half @llvm.fma.f16(half [[B]], half [[EXTRACT]], half [[A]])
-// UNCONSTRAINED-NEXT: ret half [[TMP0]]
-//
-// CONSTRAINED-LABEL: define dso_local half @test_vfmah_lane_f16(
-// CONSTRAINED-SAME: half noundef [[A:%.*]], half noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] {
-// CONSTRAINED-NEXT: [[ENTRY:.*:]]
-// CONSTRAINED-NEXT: [[EXTRACT:%.*]] = extractelement <4 x half> [[C]], i32 3
-// CONSTRAINED-NEXT: [[TMP0:%.*]] = call half @llvm.experimental.constrained.fma.f16(half [[B]], half [[EXTRACT]], half [[A]], metadata !"round.tonearest", metadata !"fpexcept.strict") #[[ATTR2]]
-// CONSTRAINED-NEXT: ret half [[TMP0]]
-//
-float16_t test_vfmah_lane_f16(float16_t a, float16_t b, float16x4_t c) {
- return vfmah_lane_f16(a, b, c, 3);
-}
-
// UNCONSTRAINED-LABEL: define dso_local half @test_vfmah_laneq_f16(
// UNCONSTRAINED-SAME: half noundef [[A:%.*]], half noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] {
// UNCONSTRAINED-NEXT: [[ENTRY:.*:]]
>From 91ba026f7d015d8c37a31735c5c5b5f2bbcc542a Mon Sep 17 00:00:00 2001
From: Lorcan Purcell <1w3r5yviper2077 at gmail.com>
Date: Wed, 7 Oct 2026 10:01:01 +0100
Subject: [PATCH 19/52] [libc++][NFC] Fix include guards in __concepts/
(#229398)
Rename the include guards in `__concepts/primary_template.h` and
`__concepts/referenceable.h` to match their file paths, like the other
headers in `__concepts/`.
Both headers were moved from `__type_traits/` (in #197646 and #199821
respectively) but kept their old guards,
`_LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H` and
`_LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H`.
Fixes #229284
---
libcxx/include/__concepts/primary_template.h | 6 +++---
libcxx/include/__concepts/referenceable.h | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/libcxx/include/__concepts/primary_template.h b/libcxx/include/__concepts/primary_template.h
index ab22b6dc74eed..8db7743de9bac 100644
--- a/libcxx/include/__concepts/primary_template.h
+++ b/libcxx/include/__concepts/primary_template.h
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
-#define _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+#ifndef _LIBCPP___CONCEPTS_PRIMARY_TEMPLATE_H
+#define _LIBCPP___CONCEPTS_PRIMARY_TEMPLATE_H
#include <__config>
#include <__type_traits/is_same.h>
@@ -30,4 +30,4 @@ _LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP_STD_VER >= 20
-#endif // _LIBCPP___TYPE_TRAITS_IS_PRIMARY_TEMPLATE_H
+#endif // _LIBCPP___CONCEPTS_PRIMARY_TEMPLATE_H
diff --git a/libcxx/include/__concepts/referenceable.h b/libcxx/include/__concepts/referenceable.h
index 471d1157a9b87..ae7977a7382f8 100644
--- a/libcxx/include/__concepts/referenceable.h
+++ b/libcxx/include/__concepts/referenceable.h
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-#ifndef _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
-#define _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
+#ifndef _LIBCPP___CONCEPTS_REFERENCEABLE_H
+#define _LIBCPP___CONCEPTS_REFERENCEABLE_H
#include <__config>
#include <__type_traits/void_t.h>
@@ -27,4 +27,4 @@ _LIBCPP_END_NAMESPACE_STD
#endif
-#endif // _LIBCPP___TYPE_TRAITS_IS_REFERENCEABLE_H
+#endif // _LIBCPP___CONCEPTS_REFERENCEABLE_H
>From 3378a49e6e495007ba4db9039717295e3e95844e Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard at gmail.com>
Date: Wed, 7 Oct 2026 11:10:16 +0200
Subject: [PATCH 20/52] AMDGPU: Fix selection of atomicrmw usub_sat for LDS and
flat (#229471)
Since #141068, `atomicrmw usub_sat` is treated as legal on every address
space as soon as the subtarget has `global_atomic_csub` (gfx10.3+).
However, `ds_sub_clamp_u32` only exists on gfx12+, and
`flat_atomic_sub_clamp_u32`, also gfx12+, was never given a selection
pattern. Both SelectionDAG and GlobalISel then fail where LLVM 21
emitted a cmpxchg loop:
```llvm
define i32 @lds(ptr addrspace(3) %p, i32 %v) {
%r = atomicrmw usub_sat ptr addrspace(3) %p, i32 %v syncscope("agent") monotonic, align 4
ret i32 %r
}
```
```
$ llc -mtriple=amdgpu11.00-amd-amdhsa usub.ll
LLVM ERROR: Cannot select: t5: i32,ch = AtomicLoadUSubSat<(load store syncscope("agent") monotonic (s32) on %ir.p, addrspace 3)> ...
```
This affects LDS on gfx10.3/11, flat on gfx12 (and private on gfx1250,
which is lowered to flat), and flat on gfx10.3/11 when the fine-grained
memory fallback doesn't kick in (e.g. with
`!amdgpu.no.fine.grained.memory`).
This PR expands `usub_sat` to cmpxchg when the LDS or flat form is
unavailable, using new subtarget features
(`atomic-ds-cond-sub-clamp-insts`, `atomic-cond-sub-clamp-flat-insts`)
from which the existing `HasAtomicDsCondSubClampInsts` /
`HasAtomicCondSubClampFlatInsts` predicates are now generated. It also
adds the missing flat pattern, mirroring the one for `usub_cond`.
`usub_cond` itself was not affected, as all of its forms were introduced
together in gfx12.
We ran into this with Julia's AMDGPU.jl, where `UnsafeAtomics.sub_sat!`
emits `atomicrmw usub_sat`.
Fixes #229442
Assisted-by: Opus 5.5, Sol 6
---
llvm/lib/Target/AMDGPU/AMDGPU.td | 19 +-
llvm/lib/Target/AMDGPU/FLATInstructions.td | 2 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 11 +-
.../test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll | 1008 +++++++++++++++++
4 files changed, 1030 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 8021a707247d0..73464a89544cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1072,6 +1072,11 @@ defm AtomicDsPkAdd16Insts : AMDGPUSubtargetFeature<"atomic-ds-pk-add-16-insts",
"ds_pk_add_rtn_f16 instructions"
>;
+defm AtomicDsCondSubClampInsts : AMDGPUSubtargetFeature<"atomic-ds-cond-sub-clamp-insts",
+ "Has ds_cond_sub_u32, ds_cond_sub_rtn_u32, ds_sub_clamp_u32 and "
+ "ds_sub_clamp_rtn_u32 instructions"
+>;
+
defm LdsAtomicAddF64 : AMDGPUSubtargetFeature<"lds-atomic-add-f64",
"Has ds_add_f64 and ds_add_rtn_f64 instructions"
>;
@@ -1080,6 +1085,10 @@ defm AtomicFlatPkAdd16Insts : AMDGPUSubtargetFeature<"atomic-flat-pk-add-16-inst
"Has flat_atomic_pk_add_f16 and flat_atomic_pk_add_bf16 instructions"
>;
+defm AtomicCondSubClampFlatInsts : AMDGPUSubtargetFeature<"atomic-cond-sub-clamp-flat-insts",
+ "Has flat_atomic_cond_sub_u32 and flat_atomic_sub_clamp_u32 instructions"
+>;
+
defm AtomicFaddRtnInsts : AMDGPUSubtargetFeature<"atomic-fadd-rtn-insts",
"Has buffer_atomic_add_f32 and global_atomic_add_f32 instructions that "
"return original value",
@@ -1883,6 +1892,7 @@ def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12",
FeatureUnalignedDSAccess, FeatureTrue16BitInsts,
FeatureDefaultComponentBroadcast, FeatureMaxHardClauseLength32,
FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF32FlatInsts,
+ FeatureAtomicDsCondSubClampInsts, FeatureAtomicCondSubClampFlatInsts,
FeatureIEEEMinimumMaximumInsts, FeatureSALUMinimumMaximumInsts,
FeatureMinimum3Maximum3F32, FeatureMinimum3Maximum3F16,
FeatureAgentScopeFineGrainedRemoteMemoryAtomics, FeatureFlatOffsetBits24,
@@ -1913,6 +1923,7 @@ def FeatureGFX13 : GCNSubtargetFeatureGeneration<"GFX13",
FeatureUnalignedDSAccess, FeatureTrue16BitInsts,
FeatureDefaultComponentBroadcast, FeatureMaxHardClauseLength32,
FeatureAtomicFMinFMaxF32GlobalInsts, FeatureAtomicFMinFMaxF32FlatInsts,
+ FeatureAtomicDsCondSubClampInsts, FeatureAtomicCondSubClampFlatInsts,
FeatureIEEEMinimumMaximumInsts, FeatureSALUMinimumMaximumInsts,
FeatureMinimum3Maximum3F32, FeatureMinimum3Maximum3F16,
FeatureAgentScopeFineGrainedRemoteMemoryAtomics, FeatureFlatOffsetBits24,
@@ -3132,10 +3143,6 @@ def isGFX13Plus :
Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX13">,
AssemblerPredicate<(all_of FeatureGFX13Insts)>;
-def HasAtomicCondSubClampFlatInsts :
- Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12">,
- AssemblerPredicate<(all_of FeatureGFX12Insts)>;
-
def HasD16LoadStore : Predicate<"Subtarget->hasD16LoadStore()">,
AssemblerPredicate<(all_of FeatureGFX9Insts)>;
@@ -3277,10 +3284,6 @@ def HasCvtFP8ByteSelNotE5M3 : Predicate<"Subtarget->hasCvtFP8ByteSel() && !Subta
def HasFmacLegacy32 : Predicate<"Subtarget->hasFmaLegacy32Insts() && Subtarget->getGeneration() < AMDGPUSubtarget::GFX12">,
AssemblerPredicate<(all_of FeatureFmaLegacy32Insts, (not FeatureGFX12Insts))>;
-def HasAtomicDsCondSubClampInsts :
- Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12">,
- AssemblerPredicate<(all_of FeatureGFX12Insts)>;
-
def HasAtomicBufferGlobalPkAddF16NoRtnInsts
: Predicate<"Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() || Subtarget->hasAtomicBufferGlobalPkAddF16Insts()">,
AssemblerPredicate<(any_of FeatureAtomicBufferGlobalPkAddF16NoRtnInsts, FeatureAtomicBufferGlobalPkAddF16Insts)>;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index f241f6677d22b..ca2fa2995d10d 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -2187,6 +2187,8 @@ let SubtargetPredicate = HasAtomicCondSubClampFlatInsts in {
defm : FlatAtomicRtnPat<"FLAT_ATOMIC_COND_SUB_U32", "atomic_load_usub_cond_" #as, i32 >;
defm : FlatAtomicNoRtnPat<"FLAT_ATOMIC_COND_SUB_U32", "atomic_load_usub_cond_"#as, i32>;
+
+ defm : FlatAtomicPat<"FLAT_ATOMIC_CSUB_U32", "atomic_load_usub_sat_"#as, i32>;
}
} // end foreach as
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 2915d7a50074c..1d1eb983245cb 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -21123,8 +21123,15 @@ SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
case AtomicRMWInst::USubSat: {
if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
return AtomicExpansionKind::CmpXChg;
- if (Op == AtomicRMWInst::USubSat && !Subtarget->hasSubClampInsts())
- return AtomicExpansionKind::CmpXChg;
+ if (Op == AtomicRMWInst::USubSat) {
+ // The global and buffer forms predate the LDS and flat ones.
+ if (!Subtarget->hasSubClampInsts() ||
+ (AS == AMDGPUAS::LOCAL_ADDRESS &&
+ !Subtarget->hasAtomicDsCondSubClampInsts()) ||
+ (AS == AMDGPUAS::FLAT_ADDRESS &&
+ !Subtarget->hasAtomicCondSubClampFlatInsts()))
+ return AtomicExpansionKind::CmpXChg;
+ }
if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
auto *IT = dyn_cast<IntegerType>(RMW->getType());
if (!IT || IT->getBitWidth() != 32)
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
index 8e0ae55af891a..33dc6b1c58f92 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
@@ -4072,6 +4072,1014 @@ define i32 @global_atomic_usub_sat__amdgpu_no_fine_grained_memory__amdgpu_no_rem
ret i32 %ret
}
+define i32 @flat_atomic_usub_sat(ptr %ptr, i32 %data) {
+; GFX9-GISEL-LABEL: flat_atomic_usub_sat:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB21_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_atomic_usub_sat:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_gl1_inv
+; GFX10-GISEL-NEXT: buffer_gl0_inv
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB21_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_atomic_usub_sat:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_atomic_usub_sat:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT: flat_atomic_sub_clamp_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: flat_atomic_usub_sat:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-SDAG-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB21_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: flat_atomic_usub_sat:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0
+; GFX10-SDAG-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_gl1_inv
+; GFX10-SDAG-NEXT: buffer_gl0_inv
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-SDAG-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB21_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: flat_atomic_usub_sat:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB21_1
+; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: flat_atomic_usub_sat:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT: flat_atomic_sub_clamp_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+ %ret = atomicrmw usub_sat ptr %ptr, i32 %data syncscope("agent") seq_cst, align 4
+ ret i32 %ret
+}
+
+define void @flat_atomic_usub_sat_nortn(ptr %ptr, i32 %data) {
+; GFX9-GISEL-LABEL: flat_atomic_usub_sat_nortn:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: flat_load_dword v4, v[0:1]
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB22_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_atomic_usub_sat_nortn:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_gl1_inv
+; GFX10-GISEL-NEXT: buffer_gl0_inv
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB22_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_atomic_usub_sat_nortn:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_atomic_usub_sat_nortn:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT: flat_atomic_sub_clamp_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: flat_atomic_usub_sat_nortn:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: flat_load_dword v4, v[0:1]
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-SDAG-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB22_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: flat_atomic_usub_sat_nortn:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0
+; GFX10-SDAG-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_gl1_inv
+; GFX10-SDAG-NEXT: buffer_gl0_inv
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-SDAG-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB22_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: flat_atomic_usub_sat_nortn:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-NEXT: .LBB22_1: ; %atomicrmw.start
+; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB22_1
+; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: flat_atomic_usub_sat_nortn:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT: flat_atomic_sub_clamp_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+ %ret = atomicrmw usub_sat ptr %ptr, i32 %data syncscope("agent") seq_cst, align 4
+ ret void
+}
+
+define i32 @flat_atomic_usub_sat__amdgpu_no_fine_grained_memory(ptr %ptr, i32 %data) {
+; GFX9-GISEL-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB23_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_gl1_inv
+; GFX10-GISEL-NEXT: buffer_gl0_inv
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB23_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT: flat_atomic_sub_clamp_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-SDAG-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB23_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0
+; GFX10-SDAG-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_gl1_inv
+; GFX10-SDAG-NEXT: buffer_gl0_inv
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-SDAG-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB23_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-NEXT: .LBB23_1: ; %atomicrmw.start
+; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB23_1
+; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: flat_atomic_usub_sat__amdgpu_no_fine_grained_memory:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT: flat_atomic_sub_clamp_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+ %ret = atomicrmw usub_sat ptr %ptr, i32 %data syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
+ ret i32 %ret
+}
+
+define void @flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory(ptr %ptr, i32 %data) {
+; GFX9-GISEL-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: flat_load_dword v4, v[0:1]
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB24_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-GISEL-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_gl1_inv
+; GFX10-GISEL-NEXT: buffer_gl0_inv
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB24_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT: flat_atomic_sub_clamp_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: flat_load_dword v4, v[0:1]
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-SDAG-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
+; GFX9-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB24_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0
+; GFX10-SDAG-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX10-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-SDAG-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_gl1_inv
+; GFX10-SDAG-NEXT: buffer_gl0_inv
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-SDAG-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB24_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-NEXT: .LBB24_1: ; %atomicrmw.start
+; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_sub_nc_u32_e64 v3, v4, v2 clamp
+; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB24_1
+; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: flat_atomic_usub_sat_nortn__amdgpu_no_fine_grained_memory:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT: flat_atomic_sub_clamp_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+ %ret = atomicrmw usub_sat ptr %ptr, i32 %data syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define i32 @local_atomic_usub_sat(ptr addrspace(3) %ptr, i32 %data) {
+; GFX9-GISEL-LABEL: local_atomic_usub_sat:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: ds_read_b32 v2, v0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v2, v3, v1 clamp
+; GFX9-GISEL-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB25_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: local_atomic_usub_sat:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: ds_read_b32 v2, v0
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX10-GISEL-NEXT: v_sub_nc_u32_e64 v2, v3, v1 clamp
+; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-GISEL-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_gl0_inv
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB25_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: local_atomic_usub_sat:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: ds_load_b32 v2, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u32_e64 v2, v3, v1 clamp
+; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: local_atomic_usub_sat:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT: ds_sub_clamp_rtn_u32 v0, v0, v1
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: local_atomic_usub_sat:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-SDAG-NEXT: ds_read_b32 v0, v0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-SDAG-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v0, v3, v1 clamp
+; GFX9-SDAG-NEXT: ds_cmpst_rtn_b32 v0, v2, v3, v0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB25_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: local_atomic_usub_sat:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-SDAG-NEXT: ds_read_b32 v0, v0
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0
+; GFX10-SDAG-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-SDAG-NEXT: v_sub_nc_u32_e64 v0, v3, v1 clamp
+; GFX10-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-SDAG-NEXT: ds_cmpst_rtn_b32 v0, v2, v3, v0
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_gl0_inv
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v3
+; GFX10-SDAG-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB25_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: local_atomic_usub_sat:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-NEXT: .LBB25_1: ; %atomicrmw.start
+; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u32_e64 v0, v3, v1 clamp
+; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: ds_cmpstore_rtn_b32 v0, v2, v0, v3
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v3
+; GFX11-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB25_1
+; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: local_atomic_usub_sat:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT: ds_sub_clamp_rtn_u32 v0, v0, v1
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+ %ret = atomicrmw usub_sat ptr addrspace(3) %ptr, i32 %data syncscope("agent") seq_cst, align 4
+ ret i32 %ret
+}
+
+define void @local_atomic_usub_sat_nortn(ptr addrspace(3) %ptr, i32 %data) {
+; GFX9-GISEL-LABEL: local_atomic_usub_sat_nortn:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: ds_read_b32 v2, v0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-GISEL-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v3, v2, v1 clamp
+; GFX9-GISEL-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB26_1
+; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: local_atomic_usub_sat_nortn:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: ds_read_b32 v2, v0
+; GFX10-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX10-GISEL-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_sub_nc_u32_e64 v3, v2, v1 clamp
+; GFX10-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-GISEL-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-GISEL-NEXT: buffer_gl0_inv
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB26_1
+; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: local_atomic_usub_sat_nortn:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: ds_load_b32 v2, v0
+; GFX11-GISEL-NEXT: s_mov_b32 s0, 0
+; GFX11-GISEL-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_sub_nc_u32_e64 v3, v2, v1 clamp
+; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-GISEL-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB26_1
+; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: local_atomic_usub_sat_nortn:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT: ds_sub_clamp_u32 v0, v1
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: local_atomic_usub_sat_nortn:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: ds_read_b32 v2, v0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-SDAG-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v3, v2, v1 clamp
+; GFX9-SDAG-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB26_1
+; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: local_atomic_usub_sat_nortn:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: ds_read_b32 v2, v0
+; GFX10-SDAG-NEXT: s_mov_b32 s4, 0
+; GFX10-SDAG-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_sub_nc_u32_e64 v3, v2, v1 clamp
+; GFX10-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX10-SDAG-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: buffer_gl0_inv
+; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-SDAG-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB26_1
+; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: local_atomic_usub_sat_nortn:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: ds_load_b32 v2, v0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, 0
+; GFX11-SDAG-NEXT: .LBB26_1: ; %atomicrmw.start
+; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_sub_nc_u32_e64 v3, v2, v1 clamp
+; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-SDAG-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB26_1
+; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: local_atomic_usub_sat_nortn:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT: ds_sub_clamp_u32 v0, v1
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
+ %ret = atomicrmw usub_sat ptr addrspace(3) %ptr, i32 %data syncscope("agent") seq_cst, align 4
+ ret void
+}
+
attributes #0 = { nounwind willreturn }
attributes #1 = { argmemonly nounwind }
>From e699478b640736e11855041fda4e2902c0d0bb22 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Wed, 7 Oct 2026 11:23:02 +0200
Subject: [PATCH 21/52] Revert "[IndirectBrExpand] Preserve profile weights"
(#229691)
Reverts llvm/llvm-project#227784
Causes compile-time regression at `-O0` due to the extra passes.
---
llvm/lib/CodeGen/IndirectBrExpandPass.cpp | 127 ++--------------
llvm/test/CodeGen/X86/O0-pipeline.ll | 6 +-
llvm/test/CodeGen/X86/opt-pipeline.ll | 5 +-
llvm/test/Transforms/IndirectBrExpand/pgo.ll | 148 -------------------
llvm/utils/profcheck-xfail.txt | 1 +
5 files changed, 16 insertions(+), 271 deletions(-)
delete mode 100644 llvm/test/Transforms/IndirectBrExpand/pgo.ll
diff --git a/llvm/lib/CodeGen/IndirectBrExpandPass.cpp b/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
index bb922b76d8c0f..e7a22656f17c2 100644
--- a/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
+++ b/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
@@ -25,12 +25,9 @@
///
//===----------------------------------------------------------------------===//
-#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/DomTreeUpdater.h"
-#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/CodeGen/IndirectBrExpand.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -39,11 +36,9 @@
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
-#include "llvm/IR/ProfDataUtils.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/ScaledNumber.h"
#include "llvm/Target/TargetMachine.h"
#include <optional>
@@ -51,10 +46,6 @@ using namespace llvm;
#define DEBUG_TYPE "indirectbr-expand"
-namespace llvm {
-extern cl::opt<bool> ProfcheckDisableMetadataFixes;
-} // namespace llvm
-
namespace {
class IndirectBrExpandLegacyPass : public FunctionPass {
@@ -64,7 +55,6 @@ class IndirectBrExpandLegacyPass : public FunctionPass {
IndirectBrExpandLegacyPass() : FunctionPass(ID) {}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
AU.addPreserved<DominatorTreeWrapperPass>();
}
@@ -73,8 +63,8 @@ class IndirectBrExpandLegacyPass : public FunctionPass {
} // end anonymous namespace
-static bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
- function_ref<BlockFrequencyInfo *()> GetBFI);
+static bool runImpl(Function &F, const TargetLowering *TLI,
+ DomTreeUpdater *DTU);
PreservedAnalyses IndirectBrExpandPass::run(Function &F,
FunctionAnalysisManager &FAM) {
@@ -86,9 +76,7 @@ PreservedAnalyses IndirectBrExpandPass::run(Function &F,
auto *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
- bool Changed = runImpl(F, TLI, DT ? &DTU : nullptr, [&]() {
- return &FAM.getResult<BlockFrequencyAnalysis>(F);
- });
+ bool Changed = runImpl(F, TLI, DT ? &DTU : nullptr);
if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA;
@@ -108,26 +96,13 @@ FunctionPass *llvm::createIndirectBrExpandPass() {
return new IndirectBrExpandLegacyPass();
}
-bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
- function_ref<BlockFrequencyInfo *()> GetBFI) {
+bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU) {
auto &DL = F.getDataLayout();
SmallVector<IndirectBrInst *, 1> IndirectBrs;
- SmallVector<uint64_t, 1> IndirectBrsBlockFrequencies;
- SmallVector<uint64_t, 1> IndirectBrsBranchWeightSums;
- bool SkipProfileUpdates = false;
- BlockFrequencyInfo *BFI = nullptr;
-
- struct IndirectBrSuccessor {
- // The index into the IndirectBrs, IndirectBrsBlockFrequencies, and
- // IndirectBrsBranchWeightSums vectors.
- size_t IndirectBrIndex = 0;
- uint64_t SuccessorBranchWeight = 0;
- };
// Set of all potential successors for indirectbr instructions.
- DenseMap<const BasicBlock *, SmallVector<IndirectBrSuccessor>>
- IndirectBrSuccToIndirectBr;
+ SmallPtrSet<BasicBlock *, 4> IndirectBrSuccs;
// Build a list of indirectbrs that we want to rewrite.
for (BasicBlock &BB : F)
@@ -141,34 +116,7 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
}
IndirectBrs.push_back(IBr);
- const size_t CurrentIndirectBrIndex = IndirectBrs.size() - 1;
- for (const BasicBlock *SuccessorBB : IBr->successors())
- IndirectBrSuccToIndirectBr.insert({SuccessorBB, {}});
-
- if (SkipProfileUpdates)
- continue;
- if (!BFI)
- BFI = GetBFI();
- std::optional<uint64_t> BlockFrequency = BFI->getBlockProfileCount(&BB);
- if (!BlockFrequency.has_value()) {
- SkipProfileUpdates = true;
- continue;
- }
- IndirectBrsBlockFrequencies.push_back(*BlockFrequency);
- SmallVector<uint32_t> IndirectBrBranchWeights;
- bool HasBranchWeights =
- extractBranchWeights(*IBr, IndirectBrBranchWeights);
- if (!HasBranchWeights) {
- SkipProfileUpdates = true;
- continue;
- }
- for (const auto [SuccessorBB, SuccessorBranchWeight] :
- zip_equal(IBr->successors(), IndirectBrBranchWeights))
- IndirectBrSuccToIndirectBr[SuccessorBB].push_back(
- {CurrentIndirectBrIndex, SuccessorBranchWeight});
- IndirectBrsBranchWeightSums.push_back(sum_of(IndirectBrBranchWeights));
- assert(IndirectBrsBranchWeightSums.size() == IndirectBrs.size() &&
- "expected an identical number of blocks in both vectors");
+ IndirectBrSuccs.insert_range(IBr->successors());
}
if (IndirectBrs.empty())
@@ -179,13 +127,11 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
// whose address escapes. We do that here and rewrite all the blockaddress
// constants to just be those integer constants cast to a pointer type.
SmallVector<BasicBlock *, 4> BBs;
- SmallVector<ScaledNumber<uint64_t>, 4> BBWeights;
for (BasicBlock &BB : F) {
// Skip blocks that aren't successors to an indirectbr we're going to
// rewrite.
- auto IndirectBrSuccToIndirectBrIt = IndirectBrSuccToIndirectBr.find(&BB);
- if (IndirectBrSuccToIndirectBrIt == IndirectBrSuccToIndirectBr.end())
+ if (!IndirectBrSuccs.count(&BB))
continue;
auto *BA = BlockAddress::lookup(&BB);
@@ -208,28 +154,6 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
// expressions, for instance, where they are used to pass labels to
// asm-goto. This part of the pass needs a rework.
BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(BBIndexC, BA->getType()));
-
- if (SkipProfileUpdates)
- continue;
- ScaledNumber<uint64_t> BranchWeightSumsProduct(1, 0);
- for (uint64_t BranchWeightSum : IndirectBrsBranchWeightSums)
- BranchWeightSumsProduct *= ScaledNumber<uint64_t>(BranchWeightSum, 0);
- ScaledNumber<uint64_t> BlockWeight(0, 0);
- for (const auto &[IndirectBrIndex, BlockBranchProbability] :
- IndirectBrSuccToIndirectBrIt->second) {
- // If the branch weight sum is zero, skip adding the block weight or
- // otherwise we end up dividing by zero.
- const uint64_t CurrentBranchWeightSum =
- IndirectBrsBranchWeightSums[IndirectBrIndex];
- if (CurrentBranchWeightSum == 0)
- continue;
- BlockWeight += ScaledNumber<uint64_t>(
- IndirectBrsBlockFrequencies[IndirectBrIndex], 0) *
- ScaledNumber<uint64_t>(BlockBranchProbability, 0) *
- (BranchWeightSumsProduct /
- ScaledNumber<uint64_t>(CurrentBranchWeightSum, 0));
- }
- BBWeights.push_back(BlockWeight);
}
if (BBs.empty()) {
@@ -237,7 +161,7 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
// cannot get a valid input and we can replace all of them with unreachable.
SmallVector<DominatorTree::UpdateType, 8> Updates;
if (DTU)
- Updates.reserve(IndirectBrSuccToIndirectBr.size());
+ Updates.reserve(IndirectBrSuccs.size());
for (auto *IBr : IndirectBrs) {
if (DTU) {
for (BasicBlock *SuccBB : IBr->successors())
@@ -247,7 +171,7 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
IBr->eraseFromParent();
}
if (DTU) {
- assert(Updates.size() == IndirectBrSuccToIndirectBr.size() &&
+ assert(Updates.size() == IndirectBrSuccs.size() &&
"Got unexpected update count.");
DTU->applyUpdates(Updates);
}
@@ -282,10 +206,10 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
SwitchBB = IBr->getParent();
SwitchValue = GetSwitchValue(IBr);
if (DTU) {
- Updates.reserve(IndirectBrSuccToIndirectBr.size());
+ Updates.reserve(IndirectBrSuccs.size());
for (BasicBlock *SuccBB : IBr->successors())
Updates.push_back({DominatorTree::Delete, IBr->getParent(), SuccBB});
- assert(Updates.size() == IndirectBrSuccToIndirectBr.size() &&
+ assert(Updates.size() == IndirectBrSuccs.size() &&
"Got unexpected update count.");
}
IBr->eraseFromParent();
@@ -301,8 +225,7 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
// Now replace the indirectbr instructions with direct branches to the
// switch block and fill out the PHI operands.
if (DTU)
- Updates.reserve(IndirectBrs.size() +
- 2 * IndirectBrSuccToIndirectBr.size());
+ Updates.reserve(IndirectBrs.size() + 2 * IndirectBrSuccs.size());
for (auto *IBr : IndirectBrs) {
SwitchPN->addIncoming(GetSwitchValue(IBr), IBr->getParent());
UncondBrInst::Create(SwitchBB, IBr->getIterator());
@@ -335,28 +258,6 @@ bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU,
DTU->applyUpdates(Updates);
}
- if (SkipProfileUpdates || ProfcheckDisableMetadataFixes) {
- setExplicitlyUnknownBranchWeightsIfProfiled(*SI, DEBUG_TYPE);
- return true;
- }
-
- // We need to convert the ScaledNumber weights (which might not be
- // representable in 64 bits) back to normal 64 bit integers so we can apply
- // them as metadata. They might not have the same scale though, so we find the
- // max scale and then scale down any weights that have a scale less than the
- // max scale. This ensures that all the weights have the same scale.
- int16_t MaxScale = 0;
- for (const ScaledNumber<uint64_t> &BBWeight : BBWeights)
- MaxScale = std::max(MaxScale, BBWeight.getScale());
- SmallVector<uint64_t, 4> ExtractedBBWeights;
- ExtractedBBWeights.reserve(BBWeights.size());
- for (ScaledNumber<uint64_t> &BBWeight : BBWeights) {
- int16_t Shift = MaxScale - BBWeight.getScale();
- assert(Shift >= 0 && "expected non-negative shift");
- ExtractedBBWeights.push_back(BBWeight.getDigits() >> Shift);
- }
- setFittedBranchWeights(*SI, ExtractedBBWeights, false);
-
return true;
}
@@ -375,7 +276,5 @@ bool IndirectBrExpandLegacyPass::runOnFunction(Function &F) {
if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy);
- return runImpl(F, TLI, DTU ? &*DTU : nullptr, [&]() {
- return &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
- });
+ return runImpl(F, TLI, DTU ? &*DTU : nullptr);
}
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index e88041b2e6cf7..479b8825ce7b3 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -30,11 +30,7 @@
; CHECK-NEXT: Instrument function entry/exit with calls to e.g. mcount() (post inlining)
; CHECK-NEXT: Scalarize Masked Memory Intrinsics
; CHECK-NEXT: Expand reduction intrinsics
-; CHECK-NEXT: Dominator Tree Construction
-; CHECK-NEXT: Cycle Info Analysis
-; CHECK-NEXT: Lazy Branch Probability Analysis
-; CHECK-NEXT: Lazy Block Frequency Analysis
-; CHECK-NEXT: Expand indirectbr instructions
+; CHECK-NEXT: Expand indirectbr instructions
; CHECK-NEXT: Exception handling preparation
; CHECK-NEXT: Prepare inline asm insts
; CHECK-NEXT: Safe Stack instrumentation pass
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index 58fb03b8f94a5..47d97871274ae 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -66,10 +66,7 @@
; CHECK-NEXT: Expand reduction intrinsics
; CHECK-NEXT: Interleaved Access Pass
; CHECK-NEXT: X86 Partial Reduction
-; CHECK-NEXT: Cycle Info Analysis
-; CHECK-NEXT: Lazy Branch Probability Analysis
-; CHECK-NEXT: Lazy Block Frequency Analysis
-; CHECK-NEXT: Expand indirectbr instructions
+; CHECK-NEXT: Expand indirectbr instructions
; CHECK-NEXT: Natural Loop Information
; CHECK-NEXT: Cycle Info Analysis
; CHECK-NEXT: Post-Dominator Tree Construction
diff --git a/llvm/test/Transforms/IndirectBrExpand/pgo.ll b/llvm/test/Transforms/IndirectBrExpand/pgo.ll
deleted file mode 100644
index 82f9ca445660f..0000000000000
--- a/llvm/test/Transforms/IndirectBrExpand/pgo.ll
+++ /dev/null
@@ -1,148 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 6
-; RUN: opt < %s -passes=indirectbr-expand -S | FileCheck %s
-;
-; REQUIRES: x86-registered-target
-
-target triple = "x86_64-unknown-linux-gnu"
-
- at test1.targets = constant [4 x ptr] [ptr blockaddress(@test1, %bb0),
- ptr blockaddress(@test1, %bb1),
- ptr blockaddress(@test1, %bb2),
- ptr blockaddress(@test1, %bb3)]
-
-;.
-; CHECK: @test1.targets = constant [4 x ptr] [ptr inttoptr (i64 1 to ptr), ptr inttoptr (i64 2 to ptr), ptr inttoptr (i64 3 to ptr), ptr blockaddress(@test1, %bb3)]
-; CHECK: @missing_branch_weights.targets = constant [2 x ptr] [ptr inttoptr (i64 1 to ptr), ptr inttoptr (i64 2 to ptr)]
-;.
-define void @test1(ptr readonly %p, ptr %sink) #0 !prof !0 {
-; CHECK-LABEL: define void @test1(
-; CHECK-SAME: ptr readonly [[P:%.*]], ptr [[SINK:%.*]]) #[[ATTR0:[0-9]+]] !prof [[PROF0:![0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[BR_SELECT_I:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: [[BR_SELECT_CMP:%.*]] = icmp eq i64 [[BR_SELECT_I]], 64
-; CHECK-NEXT: br i1 [[BR_SELECT_CMP]], label %[[BR1:.*]], label %[[BR2:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK: [[BR1]]:
-; CHECK-NEXT: [[I0:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: [[TARGET_I0:%.*]] = getelementptr [4 x ptr], ptr @test1.targets, i64 0, i64 [[I0]]
-; CHECK-NEXT: [[TARGET0:%.*]] = load ptr, ptr [[TARGET_I0]], align 8
-; CHECK-NEXT: [[TARGET0_SWITCH_CAST:%.*]] = ptrtoint ptr [[TARGET0]] to i64
-; CHECK-NEXT: br label %[[SWITCH_BB:.*]]
-; CHECK: [[BR2]]:
-; CHECK-NEXT: [[I_NEXT:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: [[TARGET_I_NEXT:%.*]] = getelementptr [4 x ptr], ptr @test1.targets, i64 0, i64 [[I_NEXT]]
-; CHECK-NEXT: [[TARGET_NEXT:%.*]] = load ptr, ptr [[TARGET_I_NEXT]], align 8
-; CHECK-NEXT: [[TARGET_NEXT_SWITCH_CAST:%.*]] = ptrtoint ptr [[TARGET_NEXT]] to i64
-; CHECK-NEXT: br label %[[SWITCH_BB]]
-; CHECK: [[BB0:.*]]:
-; CHECK-NEXT: store volatile i64 0, ptr [[SINK]], align 8
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[BB1:.*]]:
-; CHECK-NEXT: store volatile i64 1, ptr [[SINK]], align 8
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[BB2:.*]]:
-; CHECK-NEXT: store volatile i64 2, ptr [[SINK]], align 8
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[BB3:.*:]]
-; CHECK-NEXT: store volatile i64 3, ptr [[SINK]], align 8
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-; CHECK: [[SWITCH_BB]]:
-; CHECK-NEXT: [[SWITCH_VALUE_PHI:%.*]] = phi i64 [ [[TARGET0_SWITCH_CAST]], %[[BR1]] ], [ [[TARGET_NEXT_SWITCH_CAST]], %[[BR2]] ]
-; CHECK-NEXT: switch i64 [[SWITCH_VALUE_PHI]], label %[[BB0]] [
-; CHECK-NEXT: i64 2, label %[[BB1]]
-; CHECK-NEXT: i64 3, label %[[BB2]]
-; CHECK-NEXT: ], !prof [[PROF2:![0-9]+]]
-;
-entry:
- %br.select.i = load i64, ptr %p
- %br.select.cmp = icmp eq i64 %br.select.i, 64
- br i1 %br.select.cmp, label %br1, label %br2, !prof !1
-
-br1:
- %i0 = load i64, ptr %p
- %target.i0 = getelementptr [4 x ptr], ptr @test1.targets, i64 0, i64 %i0
- %target0 = load ptr, ptr %target.i0
- ; Only a subset of blocks are viable successors here.
- indirectbr ptr %target0, [label %bb0, label %bb1], !prof !2
-
-br2:
- %i.next = load i64, ptr %p
- %target.i.next = getelementptr [4 x ptr], ptr @test1.targets, i64 0, i64 %i.next
- %target.next = load ptr, ptr %target.i.next
- ; A different subset of blocks are viable successors here.
- indirectbr ptr %target.next, [label %bb1, label %bb2], !prof !3
-
-bb0:
- store volatile i64 0, ptr %sink
- br label %exit
-
-bb1:
- store volatile i64 1, ptr %sink
- br label %exit
-
-bb2:
- store volatile i64 2, ptr %sink
- br label %exit
-
-bb3:
- store volatile i64 3, ptr %sink
- br label %exit
-
-exit:
- ret void
-}
-
- at missing_branch_weights.targets = constant [2 x ptr] [ptr blockaddress(@missing_branch_weights, %bb0),
- ptr blockaddress(@missing_branch_weights, %bb1)]
-
-define void @missing_branch_weights(ptr readonly %p, ptr %sink) #0 !prof !0 {
-; CHECK-LABEL: define void @missing_branch_weights(
-; CHECK-SAME: ptr readonly [[P:%.*]], ptr [[SINK:%.*]]) #[[ATTR0]] !prof [[PROF0]] {
-; CHECK-NEXT: [[I0:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: [[TARGET_I0:%.*]] = getelementptr [4 x ptr], ptr @missing_branch_weights.targets, i64 0, i64 [[I0]]
-; CHECK-NEXT: [[TARGET0:%.*]] = load ptr, ptr [[TARGET_I0]], align 8
-; CHECK-NEXT: [[TARGET0_SWITCH_CAST:%.*]] = ptrtoint ptr [[TARGET0]] to i64
-; CHECK-NEXT: switch i64 [[TARGET0_SWITCH_CAST]], label %[[BB0:.*]] [
-; CHECK-NEXT: i64 2, label %[[BB1:.*]]
-; CHECK-NEXT: ], !prof [[PROF3:![0-9]+]]
-; CHECK: [[BB0]]:
-; CHECK-NEXT: store volatile i64 0, ptr [[SINK]], align 8
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[BB1]]:
-; CHECK-NEXT: store volatile i64 1, ptr [[SINK]], align 8
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
- %i0 = load i64, ptr %p
- %target.i0 = getelementptr [4 x ptr], ptr @missing_branch_weights.targets, i64 0, i64 %i0
- %target0 = load ptr, ptr %target.i0
- indirectbr ptr %target0, [label %bb0, label %bb1]
-
-bb0:
- store volatile i64 0, ptr %sink
- br label %exit
-
-bb1:
- store volatile i64 1, ptr %sink
- br label %exit
-
-exit:
- ret void
-}
-
-attributes #0 = { "target-features"="+retpoline" }
-
-!0 = !{!"function_entry_count", i32 10}
-!1 = !{!"branch_weights", i32 1, i32 2}
-!2 = !{!"branch_weights", i32 3, i32 5}
-!3 = !{!"branch_weights", i32 7, i32 11}
-;.
-; CHECK: attributes #[[ATTR0]] = { "target-features"="+retpoline" }
-;.
-; CHECK: [[PROF0]] = !{!"function_entry_count", i32 10}
-; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 2}
-; CHECK: [[PROF2]] = !{!"branch_weights", i32 162, i32 662, i32 616}
-; CHECK: [[PROF3]] = !{!"unknown", !"indirectbr-expand"}
-;.
diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt
index d1c4432040ca0..f1a81ed741605 100644
--- a/llvm/utils/profcheck-xfail.txt
+++ b/llvm/utils/profcheck-xfail.txt
@@ -25,6 +25,7 @@ Transforms/GVN/pr69301.ll
Transforms/IndVarSimplify/debugloc-rem-subst.ll
Transforms/IndVarSimplify/eliminate-rem.ll
Transforms/IndVarSimplify/preserving-debugloc-rem-div.ll
+Transforms/IndirectBrExpand/basic.ll
Transforms/InstCombine/fcmp-select.ll
Transforms/InstCombine/ffs-1.ll
Transforms/InstCombine/ffs-i16.ll
>From f549451f9dd79fd68388d988858c2d99d73d5f8a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 11:25:28 +0200
Subject: [PATCH 22/52] RuntimeLibcallsEmitter: Add isolated LibcallLibrary
variant (#223764)
This is essentially a hack to not break the common core functions shared
by most targets. Most targets have essentially the same base set of
compiler-rt or libc/libm functions, but a few are so radically different
there is nothing in common (e.g., the GPU targets have a handful of
functions, arm64ec changes every single function). This bit will pull these out of
the library merge-by-name system, and emitted as its own special case. This
allows the single name to be universal across targets without duplicating large
tables in the emitted inc file.
Co-authored-by: Claude (Opus 4.8) <noreply at anthropic.com>
---
llvm/include/llvm/IR/RuntimeLibcallsImpl.td | 7 ++
.../RuntimeLibcallEmitter-library-isolated.td | 74 +++++++++++++++++++
.../TableGen/Basic/RuntimeLibcallsEmitter.cpp | 22 ++++--
3 files changed, 98 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/TableGen/RuntimeLibcallEmitter-library-isolated.td
diff --git a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
index 516cf0515a103..c8255913a9d21 100644
--- a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
+++ b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
@@ -123,6 +123,13 @@ class LibcallLibrary<string name, dag impls,
// Availability guard for the library. Libraries sharing a LibraryName merge
// into one function, each variant under its own Pred.
RuntimeLibcallAvailability Pred = pred;
+
+ // When set, emit this library as its own setup function instead of merging it
+ // with others sharing its LibraryName, so a consumer gets exactly these members
+ // rather than the merged core. For a target providing a strict subset or a
+ // name-decorated variant of a library. LibraryName stays the
+ // isLibraryAvailable() key.
+ bit Isolated = false;
}
// Reference a shared LibcallLibrary from a SystemRuntimeLibrary, dropping the
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-library-isolated.td b/llvm/test/TableGen/RuntimeLibcallEmitter-library-isolated.td
new file mode 100644
index 0000000000000..47b3e13eabf5b
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-library-isolated.td
@@ -0,0 +1,74 @@
+// RUN: llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s | FileCheck %s
+
+// An isolated LibcallLibrary shares a linker name with a merged library but is
+// emitted as its own setAvailableLibFuncs_<defname> containing only its members.
+// Its consumer dispatches to that function, still guarding on the real
+// LibraryName.
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+def MEMCPY : RuntimeLibcall;
+def MEMSET : RuntimeLibcall;
+def MALLOC : RuntimeLibcall;
+
+def memcpy : RuntimeLibcallImpl<MEMCPY>;
+def memset : RuntimeLibcallImpl<MEMSET>;
+def malloc : RuntimeLibcallImpl<MALLOC>;
+
+def IsX86 : LibcallPredicate<[{TT.getArch() == Triple::x86_64}]>;
+def isX86 : RuntimeLibcallAvailability<(all_of IsX86)>;
+def IsSPIRV : LibcallPredicate<[{TT.isSPIRV()}]>;
+def isSPIRV : RuntimeLibcallAvailability<(all_of IsSPIRV)>;
+
+// The generic libc: the full core, always available.
+def Libc : LibcallLibrary<"libc", (add memcpy, memset, malloc)>;
+
+// SPIRV supplies only memset/memcpy from libc; isolated so "libc" does not pull
+// in the full merged core (malloc).
+def SPIRVLibc : LibcallLibrary<"libc", (add memcpy, memset), isSPIRV> {
+ let Isolated = true;
+}
+
+def X86SystemLibrary : SystemRuntimeLibrary<isX86, (add Libc)>;
+def SPIRVSystemLibrary : SystemRuntimeLibrary<isSPIRV, (add SPIRVLibc)>;
+
+// The merged libc function carries the full core.
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_libc(const llvm::Triple &TT, ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, StringRef ABIName, LongDoubleFormat LongDoubleFormat) {
+// CHECK-NEXT: {
+// CHECK-NEXT: static const RTLIB::LibcallImpl LibraryCalls[] = {
+// CHECK-NEXT: RTLIB::impl_malloc, // malloc
+// CHECK-NEXT: RTLIB::impl_memcpy, // memcpy
+// CHECK-NEXT: RTLIB::impl_memset, // memset
+// CHECK-NEXT: };
+// CHECK-EMPTY:
+// CHECK-NEXT: for (const RTLIB::LibcallImpl Impl : LibraryCalls) {
+// CHECK-NEXT: setAvailable(Impl);
+// CHECK-NEXT: }
+// CHECK-EMPTY:
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+
+// The isolated SPIRV libc is its own function with only memset/memcpy, no malloc.
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_SPIRVLibc(const llvm::Triple &TT, ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, StringRef ABIName, LongDoubleFormat LongDoubleFormat) {
+// CHECK-NEXT: if (TT.isSPIRV()) {
+// CHECK-NEXT: static const RTLIB::LibcallImpl LibraryCalls[] = {
+// CHECK-NEXT: RTLIB::impl_memcpy, // memcpy
+// CHECK-NEXT: RTLIB::impl_memset, // memset
+// CHECK-NEXT: };
+// CHECK-EMPTY:
+// CHECK-NEXT: for (const RTLIB::LibcallImpl Impl : LibraryCalls) {
+// CHECK-NEXT: setAvailable(Impl);
+// CHECK-NEXT: }
+// CHECK-EMPTY:
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+
+// The SPIRV target dispatches to the isolated function, but the availability
+// guard still uses the real "libc" linker name.
+
+// CHECK-LABEL: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(
+// CHECK: if (TT.isSPIRV()) {
+// CHECK: if (isLibraryAvailable("libc"))
+// CHECK-NEXT: setAvailableLibFuncs_SPIRVLibc(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat);
diff --git a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
index 980d474ed6ee6..6dfe4b9b0ac50 100644
--- a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
@@ -720,11 +720,18 @@ void RuntimeLibcallEmitter::emitLibraryFunction(
OS << "}\n\n";
}
+// The setAvailableLibFuncs_ suffix (and merge key) for a library: its shared
+// LibraryName normally, or its own def name if isolated (so it does not merge).
+static StringRef libFuncKey(const Record *Lib) {
+ return Lib->getValueAsBit("Isolated") ? Lib->getName()
+ : Lib->getValueAsString("LibraryName");
+}
+
MapVector<StringRef, std::vector<const Record *>>
RuntimeLibcallEmitter::collectLibrariesByName() const {
MapVector<StringRef, std::vector<const Record *>> LibsByName;
for (const Record *Lib : Records.getAllDerivedDefinitions("LibcallLibrary"))
- LibsByName[Lib->getValueAsString("LibraryName")].push_back(Lib);
+ LibsByName[libFuncKey(Lib)].push_back(Lib);
return LibsByName;
}
@@ -842,9 +849,12 @@ void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
// Split the top-level member list into named LibcallLibrary references
// (dispatched to their own setAvailableLibFuncs_<name> under an
// isLibraryAvailable guard) and the remaining bare impl / LibcallImpls
- // members. A LibraryRef also records impls to drop.
+ // members (emitted inline below). A LibraryRef also records impls to drop.
+ // Name is the linker library (the isLibraryAvailable guard); FuncSuffix is
+ // the function suffix, which differs from Name only when isolated.
struct DispatchLib {
StringRef Name;
+ StringRef FuncSuffix;
std::vector<const RuntimeLibcallImpl *> Exclude;
};
const DagInit *MemberDag =
@@ -857,13 +867,15 @@ void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
if (const auto *DI = dyn_cast<DefInit>(Arg)) {
const Record *Def = DI->getDef();
if (Def->isSubClassOf("LibcallLibrary")) {
- DispatchLibs.push_back({Def->getValueAsString("LibraryName"), {}});
+ DispatchLibs.push_back(
+ {Def->getValueAsString("LibraryName"), libFuncKey(Def), {}});
continue;
}
if (Def->isSubClassOf("LibraryRef")) {
const Record *Lib = Def->getValueAsDef("Library");
- DispatchLib DL{Lib->getValueAsString("LibraryName"), {}};
+ DispatchLib DL{
+ Lib->getValueAsString("LibraryName"), libFuncKey(Lib), {}};
for (const Record *ExcludeRec :
Def->getValueAsListOfDefs("Exclude")) {
if (const RuntimeLibcallImpl *Impl =
@@ -961,7 +973,7 @@ void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
for (const DispatchLib &DL : DispatchLibs) {
OS << indent(4) << "if (isLibraryAvailable(\"" << DL.Name << "\"))\n"
<< indent(6) << "setAvailableLibFuncs_";
- emitLibFuncSuffix(OS, DL.Name);
+ emitLibFuncSuffix(OS, DL.FuncSuffix);
OS << "(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat);\n";
}
if (!DispatchLibs.empty())
>From 708fee365beae021be8378300a4fbe6e2d9508d4 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen at arm.com>
Date: Wed, 7 Oct 2026 10:32:48 +0100
Subject: [PATCH 23/52] [AArch64][SME] Allow more inlining when SME attributes
are incompatible. (#223393)
At the moment, 'areInlineCompatible' is very strict as it conservatively
disallows inlining any callee if they use intrinsics and have
incompatible SME attributes. This PR relaxes those constraints by
allowing more intrinsics.
It also updates the Clang diagnostic to match the 'new' behaviour that a
function is no longer inlined despite 'always_inline' when they are not
inline compatible.
This is an alternative approach to #218727. This PR holds on to the
approach of not inlining unless proven safe to do so, as opposed to
relying on the user to know what they're doing (#218727).
This a first step in trying to fix
https://github.com/llvm/llvm-project/issues/217639 in a better way.
---
.../clang/Basic/DiagnosticFrontendKinds.td | 2 +-
.../AArch64/sme-inline-streaming-attrs.c | 6 +-
.../AArch64/AArch64TargetTransformInfo.cpp | 187 ++++++-
.../sme-pstatesm-attrs-low-threshold.ll | 47 --
.../Inline/AArch64/sme-pstatesm-attrs.ll | 512 +++++++++++++++---
.../AArch64/sme-pstatesm-inline-transitive.ll | 47 ++
6 files changed, 666 insertions(+), 135 deletions(-)
delete mode 100644 llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll
create mode 100644 llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 5a9b79f0659bd..232475b6fb5a7 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -324,7 +324,7 @@ def err_function_always_inline_attribute_mismatch : Error<
"always_inline function %1 and its caller %0 have mismatching %2 attributes">;
def warn_function_always_inline_attribute_mismatch : Warning<
"always_inline function %1 and its caller %0 have mismatching %2 attributes, "
- "inlining may change runtime behaviour">, InGroup<AArch64SMEAttributes>;
+ "%1 may not be inlined">, InGroup<AArch64SMEAttributes>;
def err_function_always_inline_new_za : Error<
"always_inline function %0 has new za state">;
def err_function_always_inline_new_zt0
diff --git a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c
index 68102c9ded40c..ba9f24032d274 100644
--- a/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c
+++ b/clang/test/CodeGen/AArch64/sme-inline-streaming-attrs.c
@@ -26,7 +26,7 @@ void caller(void) {
#ifdef TEST_COMPATIBLE
void caller_compatible(void) __arm_streaming_compatible {
- inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, inlining may change runtime behaviour}}
+ inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_compatible' have mismatching streaming attributes, 'inlined_fn' may not be inlined}}
inlined_fn_streaming_compatible();
inlined_fn_streaming(); // expected-error {{always_inline function 'inlined_fn_streaming' and its caller 'caller_compatible' have mismatching streaming attributes}}
inlined_fn_local(); // expected-error {{always_inline function 'inlined_fn_local' and its caller 'caller_compatible' have mismatching streaming attributes}}
@@ -35,7 +35,7 @@ void caller_compatible(void) __arm_streaming_compatible {
#ifdef TEST_STREAMING
void caller_streaming(void) __arm_streaming {
- inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, inlining may change runtime behaviour}}
+ inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_streaming' have mismatching streaming attributes, 'inlined_fn' may not be inlined}}
inlined_fn_streaming_compatible();
inlined_fn_streaming();
inlined_fn_local();
@@ -45,7 +45,7 @@ void caller_streaming(void) __arm_streaming {
#ifdef TEST_LOCALLY
__arm_locally_streaming
void caller_local(void) {
- inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, inlining may change runtime behaviour}}
+ inlined_fn(); // expected-warning {{always_inline function 'inlined_fn' and its caller 'caller_local' have mismatching streaming attributes, 'inlined_fn' may not be inlined}}
inlined_fn_streaming_compatible();
inlined_fn_streaming();
inlined_fn_local();
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 562a8588e33fd..986c21c50cad5 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -11,6 +11,7 @@
#include "AArch64PerfectShuffle.h"
#include "AArch64SMEAttributes.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
+#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/bit.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -230,30 +231,126 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode(
static cl::opt<bool> EnableScalableAutovecInStreamingMode(
"enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden);
-static bool isSMEABIRoutineCall(const CallInst &CI,
- const AArch64TargetLowering &TLI) {
- const auto *F = CI.getCalledFunction();
- return F &&
- SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine();
+// Enum to record why an operation is incompatible in a different streaming
+// mode.
+enum class StreamingIncompatibility : uint8_t {
+ None = 0,
+ VScaleDependent = 1 << 0,
+ HasIncompatibleInstruction = 1 << 1,
+ FullyIncompatible = VScaleDependent | HasIncompatibleInstruction,
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestFlag=*/HasIncompatibleInstruction)
+};
+
+/// Returns an enum describing whether \p I is an instruction that is
+/// incompatible in the mode of the caller.
+static StreamingIncompatibility
+getCompatibilityForChangeToStreamingMode(const Instruction *I) {
+ if (auto *II = dyn_cast<IntrinsicInst>(I)) {
+ unsigned IID = II->getIntrinsicID();
+ switch (IID) {
+ default:
+ // Conservatively disallow target intrinsics in other modes.
+ if (Intrinsic::isTargetIntrinsic(IID))
+ return StreamingIncompatibility::FullyIncompatible;
+ break;
+ case Intrinsic::vscale:
+ return StreamingIncompatibility::VScaleDependent;
+ case Intrinsic::masked_gather:
+ // Instructions that are not available in streaming mode.
+ // Fixed-length operations can still be code-generated by scalarizing.
+ // FIXME: Consider FEAT_FA64.
+ if (I->getOperand(0)->getType()->isScalableTy())
+ return StreamingIncompatibility::FullyIncompatible;
+ break;
+ case Intrinsic::masked_scatter:
+ case Intrinsic::masked_compressstore:
+ case Intrinsic::experimental_vector_histogram_add:
+ // Instructions that are not available in streaming mode.
+ // Fixed-length operations can still be code-generated by scalarizing.
+ // FIXME: Consider FEAT_FA64.
+ if (I->getOperand(0)->getType()->isScalableTy())
+ return StreamingIncompatibility::FullyIncompatible;
+ break;
+ }
+ }
+
+ // Inlining operations on scalable vectors is rejected when
+ // vscale-dependent operations cannot be safely inlined.
+ if (I->getType()->isScalableTy() ||
+ any_of(I->operand_values(),
+ [](const Value *V) { return V->getType()->isScalableTy(); }) ||
+ (isa<GetElementPtrInst>(I) &&
+ cast<GetElementPtrInst>(I)->getSourceElementType()->isScalableTy()) ||
+ (isa<AllocaInst>(I) && cast<AllocaInst>(I)->isScalable()))
+ return StreamingIncompatibility::VScaleDependent;
+
+ return StreamingIncompatibility::None;
}
/// Returns true if the function has explicit operations that can only be
/// lowered using incompatible instructions for the selected mode. This also
/// returns true if the function F may use or modify ZA state.
static bool hasPossibleIncompatibleOps(const Function *F,
- const AArch64TargetLowering &TLI) {
+ const AArch64TargetLowering &TLI,
+ bool ConsiderZA, bool ConsiderZT,
+ bool ConsiderSM) {
+ if (!ConsiderZA && !ConsiderZT && !ConsiderSM)
+ return false;
+
+ bool IsAlwaysInline = F->hasFnAttribute(Attribute::AlwaysInline);
+ // If the interface has vscale-dependent arguments/return value, then the ACLE
+ // describes that the function can only have defined behaviour if vscale is
+ // the same for both streaming/non-streaming mode, meaning that we can assume
+ // vscale is equivalent.
+ // FIXME: Handle the case where vscale_range is known and equivalent for
+ // caller and callee.
+ bool AssumeVScaleIsEquivalent =
+ F->getReturnType()->isScalableTy() ||
+ any_of(F->getFunctionType()->params(),
+ [](const Type *T) { return T->isScalableTy(); });
+
for (const BasicBlock &BB : *F) {
for (const Instruction &I : BB) {
- // Be conservative for now and assume that any call to inline asm or to
- // intrinsics could could result in non-streaming ops (e.g. calls to
- // @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that
- // all native LLVM instructions can be lowered to compatible instructions.
- if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() &&
- (cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) ||
- isSMEABIRoutineCall(cast<CallInst>(I), TLI)))
+ if (auto *CB = dyn_cast<CallBase>(&I)) {
+ SMECallAttrs CallAttrs(*CB, &TLI.getRuntimeLibcallsInfo());
+
+ // Inline asm must be rejected as it could use SME state.
+ if (CB->isInlineAsm())
+ return true;
+
+ if (CallAttrs.callee().isSMEABIRoutine())
+ return true;
+
+ // If the callee has calls to streaming compatible functions, then those
+ // may have statements which depend on SME state (e.g. vscale or
+ // explicit reads of PSTATE.SM). If we were to inline those calls,
+ // behaviour may change after inlining because the streaming-compatible
+ // function would be executed in a different streaming mode.
+ // Conservatively avoid inlining for now.
+ if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface())
+ return true;
+ }
+
+ if (!ConsiderSM)
+ continue;
+
+ // Inlining operations on fixed-length vectors when the streaming
+ // mode does not match is rejected because performance may be impacted.
+ // This decision should eventually be moved to the cost-model.
+ if (!IsAlwaysInline && (isa<FixedVectorType>(I.getType()) ||
+ any_of(I.operand_values(), [](const Value *V) {
+ return isa<FixedVectorType>(V->getType());
+ })))
+ return true;
+
+ StreamingIncompatibility C = getCompatibilityForChangeToStreamingMode(&I);
+ if (any(C & StreamingIncompatibility::HasIncompatibleInstruction) ||
+ (!AssumeVScaleIsEquivalent &&
+ any(C & StreamingIncompatibility::VScaleDependent)))
return true;
}
}
+
return false;
}
@@ -281,6 +378,48 @@ bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const {
return F.hasFnAttribute("fmv-features");
}
+/// The compiler must not inline when that may alter the behavior of the
+/// program. This is especially relevant around SME which implements different
+/// runtime modes and maintains external state through attributes.
+///
+/// The compiler must not inline when:
+//
+/// * The module is compiled with 'strict-fp' and the called function
+/// implements a different FP environment than the caller.
+///
+/// * The called function has operations that are incompatible in the mode
+/// of the caller, e.g.:
+/// * inlining non-streaming-only instructions into a streaming function.
+/// * inlining streaming-only instructions into a non-streaming function.
+///
+/// Inline asm blocks must be entered and exited in the [streaming] mode of
+/// the parent function. There is no language-level mechanism to inform the
+/// compiler that a particular inline asm block is streaming compatible, so
+/// the compiler must reject this as a candidate for inlining.
+///
+/// * The called function contains vscale-dependent operations but otherwise
+/// does not take/return VL-dependent arguments (see definition in the
+/// ACLE (Arm C/C++ Language Extensions)).
+///
+/// * The called function sets up new ZA/ZT state into a function that already
+/// has ZA or ZT state, as that is not valid as per the ACLE.
+///
+/// The compiler should not inline when:
+//
+/// * The called function has fixed-length vectors and the caller is in
+/// streaming mode, as this may cause performance regressions. This should
+/// never result in an error to be reported.
+///
+/// * The called function sets up new ZA/ZT state into a function that has no
+/// ZA and no ZT state, as the compiler currently cannot transfer the
+/// attribute to the caller. It may also impact performance, but that should
+/// be covered by AArch64TTIImpl::getInlineCallPenalty().
+///
+/// If a callee is incompatible with its caller, then the inliner should prevent
+/// inlining. LLVM's `alwaysinline` attribute has no impact on whether a callee
+/// is inline compatible. This is especially relevant with streaming-mode
+/// attributes, where `alwaysinline`ing a function into a function with a
+/// different streaming mode may otherwise impact runtime behaviour.
bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
const Function *Callee) const {
SMECallAttrs CallAttrs(*Caller, *Callee);
@@ -292,6 +431,9 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
CallAttrs.callee().hasStreamingInterfaceOrBody())
return false;
+ if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0())
+ return false;
+
// When inlining, we should consider the body of the function, not the
// interface.
if (CallAttrs.callee().hasStreamingBody()) {
@@ -299,15 +441,20 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
CallAttrs.callee().set(SMEAttrs::SM_Enabled, true);
}
- if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0())
+ bool ConsiderZA = CallAttrs.requiresZASave();
+ bool ConsiderZT = CallAttrs.requiresPreservingZT0() ||
+ CallAttrs.requiresPreservingAllZAState();
+ bool ConsiderSM = CallAttrs.requiresSMChange();
+
+ // FP environment is interpreted differently between streaming mode and
+ // non-streaming mode, so are not inline-compatible.
+ // FIXME: Analyze whether the callee actually has any FP operations.
+ if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP()))
return false;
- if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() ||
- CallAttrs.requiresPreservingZT0() ||
- CallAttrs.requiresPreservingAllZAState()) {
- if (hasPossibleIncompatibleOps(Callee, *getTLI()))
- return false;
- }
+ if (hasPossibleIncompatibleOps(Callee, *getTLI(), ConsiderZA, ConsiderZT,
+ ConsiderSM))
+ return false;
return BaseT::areInlineCompatible(Caller, Callee);
}
diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll
deleted file mode 100644
index 8a608a1b8e156..0000000000000
--- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs-low-threshold.ll
+++ /dev/null
@@ -1,47 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline -inlinedefault-threshold=1 | FileCheck %s
-
-; This test sets the inline-threshold to 1 such that by default the call to @streaming_callee is not inlined.
-; However, if the call to @streaming_callee requires a streaming-mode change, it should always inline the call because the streaming-mode change is more expensive.
-target triple = "aarch64"
-
-declare void @streaming_compatible_f() #0 "aarch64_pstate_sm_compatible"
-
-; Function @non_streaming_callee doesn't contain any operations that may use ZA
-; state and therefore can be legally inlined into a normal function.
-define void @non_streaming_callee() #0 {
-; CHECK-LABEL: define void @non_streaming_callee
-; CHECK-SAME: () #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: call void @streaming_compatible_f()
-; CHECK-NEXT: call void @streaming_compatible_f()
-; CHECK-NEXT: ret void
-;
- call void @streaming_compatible_f()
- call void @streaming_compatible_f()
- ret void
-}
-
-; Inline call to @non_streaming_callee to remove a streaming mode change.
-define void @streaming_caller_inline() #0 "aarch64_pstate_sm_enabled" {
-; CHECK-LABEL: define void @streaming_caller_inline
-; CHECK-SAME: () #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: call void @streaming_compatible_f()
-; CHECK-NEXT: call void @streaming_compatible_f()
-; CHECK-NEXT: ret void
-;
- call void @non_streaming_callee()
- ret void
-}
-
-; Don't inline call to @non_streaming_callee when the inline-threshold is set to 1, because it does not eliminate a streaming-mode change.
-define void @non_streaming_caller_dont_inline() #0 {
-; CHECK-LABEL: define void @non_streaming_caller_dont_inline
-; CHECK-SAME: () #[[ATTR1]] {
-; CHECK-NEXT: call void @non_streaming_callee()
-; CHECK-NEXT: ret void
-;
- call void @non_streaming_callee()
- ret void
-}
-
-attributes #0 = { "target-features"="+sve,+sme" }
diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
index 077a3aa49fb41..45c3905626623 100644
--- a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
+++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-attrs.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline | FileCheck %s
+; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sve,+sme -S -passes=inline | FileCheck %s
declare i32 @llvm.vscale.i32()
@@ -7,7 +7,7 @@ declare i32 @llvm.vscale.i32()
; by the other functions below. If we see the call to one of these functions
; being replaced by 'llvm.vscale()', then we know it has been inlined.
-define i32 @normal_callee() #0 {
+define i32 @normal_callee() {
; CHECK-LABEL: define i32 @normal_callee
; CHECK-SAME: () #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -19,7 +19,7 @@ entry:
ret i32 %res
}
-define i32 @streaming_callee() #0 "aarch64_pstate_sm_enabled" {
+define i32 @streaming_callee() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i32 @streaming_callee
; CHECK-SAME: () #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -31,7 +31,7 @@ entry:
ret i32 %res
}
-define i32 @locally_streaming_callee() #0 "aarch64_pstate_sm_body" {
+define i32 @locally_streaming_callee() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @locally_streaming_callee
; CHECK-SAME: () #[[ATTR3:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -43,7 +43,7 @@ entry:
ret i32 %res
}
-define i32 @streaming_compatible_callee() #0 "aarch64_pstate_sm_compatible" {
+define i32 @streaming_compatible_callee() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define i32 @streaming_compatible_callee
; CHECK-SAME: () #[[ATTR4:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -55,7 +55,7 @@ entry:
ret i32 %res
}
-define i32 @streaming_compatible_locally_streaming_callee() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
+define i32 @streaming_compatible_locally_streaming_callee() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_callee
; CHECK-SAME: () #[[ATTR5:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -84,7 +84,7 @@ entry:
; [ ] N -> SC
; [ ] N -> N + B
; [ ] N -> SC + B
-define i32 @normal_caller_normal_callee_inline() #0 {
+define i32 @normal_caller_normal_callee_inline() {
; CHECK-LABEL: define i32 @normal_caller_normal_callee_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -101,7 +101,7 @@ entry:
; [ ] N -> SC
; [ ] N -> N + B
; [ ] N -> SC + B
-define i32 @normal_caller_streaming_callee_dont_inline() #0 {
+define i32 @normal_caller_streaming_callee_dont_inline() {
; CHECK-LABEL: define i32 @normal_caller_streaming_callee_dont_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -118,7 +118,7 @@ entry:
; [x] N -> SC
; [ ] N -> N + B
; [ ] N -> SC + B
-define i32 @normal_caller_streaming_compatible_callee_inline() #0 {
+define i32 @normal_caller_streaming_compatible_callee_inline() {
; CHECK-LABEL: define i32 @normal_caller_streaming_compatible_callee_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -135,7 +135,7 @@ entry:
; [ ] N -> SC
; [x] N -> N + B
; [ ] N -> SC + B
-define i32 @normal_caller_locally_streaming_callee_dont_inline() #0 {
+define i32 @normal_caller_locally_streaming_callee_dont_inline() {
; CHECK-LABEL: define i32 @normal_caller_locally_streaming_callee_dont_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -152,7 +152,7 @@ entry:
; [ ] N -> SC
; [ ] N -> N + B
; [x] N -> SC + B
-define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() #0 {
+define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline() {
; CHECK-LABEL: define i32 @normal_caller_streaming_compatible_locally_streaming_callee_dont_inline
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -169,7 +169,7 @@ entry:
; [ ] S -> SC
; [ ] S -> N + B
; [ ] S -> SC + B
-define i32 @streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_enabled" {
+define i32 @streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i32 @streaming_caller_normal_callee_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -186,7 +186,7 @@ entry:
; [ ] S -> SC
; [ ] S -> N + B
; [ ] S -> SC + B
-define i32 @streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" {
+define i32 @streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i32 @streaming_caller_streaming_callee_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -203,7 +203,7 @@ entry:
; [x] S -> SC
; [ ] S -> N + B
; [ ] S -> SC + B
-define i32 @streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_enabled" {
+define i32 @streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i32 @streaming_caller_streaming_compatible_callee_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -220,7 +220,7 @@ entry:
; [ ] S -> SC
; [x] S -> N + B
; [ ] S -> SC + B
-define i32 @streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" {
+define i32 @streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i32 @streaming_caller_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -237,7 +237,7 @@ entry:
; [ ] S -> SC
; [ ] S -> N + B
; [x] S -> SC + B
-define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_enabled" {
+define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i32 @streaming_caller_streaming_compatible_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -254,7 +254,7 @@ entry:
; [ ] N + B -> SC
; [ ] N + B -> N + B
; [ ] N + B -> SC + B
-define i32 @locally_streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_body" {
+define i32 @locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @locally_streaming_caller_normal_callee_dont_inline
; CHECK-SAME: () #[[ATTR3]] {
; CHECK-NEXT: entry:
@@ -271,7 +271,7 @@ entry:
; [ ] N + B -> SC
; [ ] N + B -> N + B
; [ ] N + B -> SC + B
-define i32 @locally_streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_body" {
+define i32 @locally_streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_callee_inline
; CHECK-SAME: () #[[ATTR3]] {
; CHECK-NEXT: entry:
@@ -288,7 +288,7 @@ entry:
; [x] N + B -> SC
; [ ] N + B -> N + B
; [ ] N + B -> SC + B
-define i32 @locally_streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_body" {
+define i32 @locally_streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_compatible_callee_inline
; CHECK-SAME: () #[[ATTR3]] {
; CHECK-NEXT: entry:
@@ -305,7 +305,7 @@ entry:
; [ ] N + B -> SC
; [x] N + B -> N + B
; [ ] N + B -> SC + B
-define i32 @locally_streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_body" {
+define i32 @locally_streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @locally_streaming_caller_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR3]] {
; CHECK-NEXT: entry:
@@ -322,7 +322,7 @@ entry:
; [ ] N + B -> SC
; [ ] N + B -> N + B
; [x] N + B -> SC + B
-define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_body" {
+define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline() "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @locally_streaming_caller_streaming_compatible_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR3]] {
; CHECK-NEXT: entry:
@@ -339,7 +339,7 @@ entry:
; [ ] SC -> SC
; [ ] SC -> N + B
; [ ] SC -> SC + B
-define i32 @streaming_compatible_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" {
+define i32 @streaming_compatible_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define i32 @streaming_compatible_caller_normal_callee_dont_inline
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: entry:
@@ -356,7 +356,7 @@ entry:
; [ ] SC -> SC
; [ ] SC -> N + B
; [ ] SC -> SC + B
-define i32 @streaming_compatible_caller_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" {
+define i32 @streaming_compatible_caller_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_callee_dont_inline
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: entry:
@@ -373,7 +373,7 @@ entry:
; [x] SC -> SC
; [ ] SC -> N + B
; [ ] SC -> SC + B
-define i32 @streaming_compatible_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_compatible" {
+define i32 @streaming_compatible_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_compatible_callee_inline
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: entry:
@@ -390,7 +390,7 @@ entry:
; [ ] SC -> SC
; [x] SC -> N + B
; [ ] SC -> SC + B
-define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" {
+define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define i32 @streaming_compatible_caller_locally_streaming_callee_dont_inline
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: entry:
@@ -407,7 +407,7 @@ entry:
; [ ] SC -> SC
; [ ] SC -> N + B
; [x] SC -> SC + B
-define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" {
+define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline() "aarch64_pstate_sm_compatible" {
; CHECK-LABEL: define i32 @streaming_compatible_caller_streaming_compatible_locally_streaming_callee_dont_inline
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: entry:
@@ -423,7 +423,7 @@ entry:
; [ ] SC + B -> SC
; [ ] SC + B -> N + B
; [ ] SC + B -> SC + B
-define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
+define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_normal_callee_dont_inline
; CHECK-SAME: () #[[ATTR5]] {
; CHECK-NEXT: entry:
@@ -440,7 +440,7 @@ entry:
; [ ] SC + B -> SC
; [ ] SC + B -> N + B
; [ ] SC + B -> SC + B
-define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
+define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_streaming_callee_inline
; CHECK-SAME: () #[[ATTR5]] {
; CHECK-NEXT: entry:
@@ -457,7 +457,7 @@ entry:
; [x] SC + B -> SC
; [ ] SC + B -> N + B
; [ ] SC + B -> SC + B
-define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
+define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_streaming_compatible_callee_inline
; CHECK-SAME: () #[[ATTR5]] {
; CHECK-NEXT: entry:
@@ -474,7 +474,7 @@ entry:
; [ ] SC + B -> SC
; [x] SC + B -> N + B
; [ ] SC + B -> SC + B
-define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
+define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_locally_streaming_callee_inline
; CHECK-SAME: () #[[ATTR5]] {
; CHECK-NEXT: entry:
@@ -491,7 +491,7 @@ entry:
; [ ] SC + B -> SC
; [ ] SC + B -> N + B
; [x] SC + B -> SC + B
-define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline() #0 "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
+define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" {
; CHECK-LABEL: define i32 @streaming_compatible_locally_streaming_caller_and_callee_inline
; CHECK-SAME: () #[[ATTR5]] {
; CHECK-NEXT: entry:
@@ -503,7 +503,7 @@ entry:
ret i32 %res
}
-define void @normal_callee_with_inlineasm() #0 {
+define void @normal_callee_with_inlineasm() {
; CHECK-LABEL: define void @normal_callee_with_inlineasm
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -515,7 +515,7 @@ entry:
ret void
}
-define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() #0 "aarch64_pstate_sm_enabled" {
+define void @streaming_caller_normal_callee_with_inlineasm_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @streaming_caller_normal_callee_with_inlineasm_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -527,7 +527,7 @@ entry:
ret void
}
-define i64 @normal_callee_with_intrinsic_call() #0 {
+define i64 @normal_callee_with_intrinsic_call() {
; CHECK-LABEL: define i64 @normal_callee_with_intrinsic_call
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -539,7 +539,7 @@ entry:
ret i64 %res
}
-define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() #0 "aarch64_pstate_sm_enabled" {
+define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i64 @streaming_caller_normal_callee_with_intrinsic_call_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -553,7 +553,7 @@ entry:
declare i64 @llvm.aarch64.sve.cntb(i32)
-define i64 @normal_callee_call_sme_state() #0 {
+define i64 @normal_callee_call_sme_state() {
; CHECK-LABEL: define i64 @normal_callee_call_sme_state
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: entry:
@@ -569,7 +569,7 @@ entry:
declare {i64, i64} @__arm_sme_state()
-define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() #0 "aarch64_pstate_sm_enabled" {
+define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define i64 @streaming_caller_normal_callee_call_sme_state_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: entry:
@@ -585,7 +585,7 @@ entry:
declare void @nonstreaming_body()
-define void @nonstreaming_caller_single_nonstreaming_callee() #0 {
+define void @nonstreaming_caller_single_nonstreaming_callee() {
; CHECK-LABEL: define void @nonstreaming_caller_single_nonstreaming_callee
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: call void @nonstreaming_body()
@@ -595,7 +595,7 @@ define void @nonstreaming_caller_single_nonstreaming_callee() #0 {
ret void
}
-define void @nonstreaming_caller_multiple_nonstreaming_callees() #0 {
+define void @nonstreaming_caller_multiple_nonstreaming_callees() {
; CHECK-LABEL: define void @nonstreaming_caller_multiple_nonstreaming_callees
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: call void @nonstreaming_body()
@@ -608,7 +608,7 @@ define void @nonstreaming_caller_multiple_nonstreaming_callees() #0 {
}
; Allow inlining, as inline it would not increase the number of streaming-mode changes.
-define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline() #0 "aarch64_pstate_sm_enabled" {
+define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_callee_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: call void @nonstreaming_body()
@@ -619,7 +619,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_single_nonstreaming_ca
}
; Prevent inlining, as inlining it would lead to multiple streaming-mode changes.
-define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() #0 "aarch64_pstate_sm_enabled" {
+define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline() "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: call void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_callees_dont_inline()
@@ -631,7 +631,7 @@ define void @streaming_caller_to_nonstreaming_callee_with_multiple_nonstreaming_
declare void @streaming_compatible_body() "aarch64_pstate_sm_compatible"
-define void @nonstreaming_caller_single_streaming_compatible_callee() #0 {
+define void @nonstreaming_caller_single_streaming_compatible_callee() {
; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: call void @streaming_compatible_body()
@@ -641,42 +641,75 @@ define void @nonstreaming_caller_single_streaming_compatible_callee() #0 {
ret void
}
-define void @nonstreaming_caller_multiple_streaming_compatible_callees() #0 {
-; CHECK-LABEL: define void @nonstreaming_caller_multiple_streaming_compatible_callees
-; CHECK-SAME: () #[[ATTR1]] {
-; CHECK-NEXT: call void @streaming_compatible_body()
+; Reject inlining; while inlining would remove a streaming-mode change, it may also change runtime behaviour.
+define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_dont_inline
+; CHECK-SAME: () #[[ATTR2]] {
+; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee()
+; CHECK-NEXT: ret void
+;
+ call void @nonstreaming_caller_single_streaming_compatible_callee()
+ ret void
+}
+
+define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline() alwaysinline {
+; CHECK-LABEL: define void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline
+; CHECK-SAME: () #[[ATTR6:[0-9]+]] {
; CHECK-NEXT: call void @streaming_compatible_body()
; CHECK-NEXT: ret void
;
- call void @streaming_compatible_body()
call void @streaming_compatible_body()
ret void
}
-; Allow inlining, as inline would remove a streaming-mode change.
-define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline() #0 "aarch64_pstate_sm_enabled" {
-; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_single_streamingcompatible_callee_inline
+; Reject inlining even when forced; it is unclear what the user's intentions were.
+define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_streamingcompatible_callee_dont_inline() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_alwaysinline_callee_with_single_streamingcompatible_callee_dont_inline
; CHECK-SAME: () #[[ATTR2]] {
-; CHECK-NEXT: call void @streaming_compatible_body()
+; CHECK-NEXT: call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline()
; CHECK-NEXT: ret void
;
- call void @nonstreaming_caller_single_streaming_compatible_callee()
+ call void @nonstreaming_caller_single_streaming_compatible_callee_alwaysinline()
ret void
}
-; Allow inlining, as inline would remove several streaming-mode changes.
-define void @streaming_caller_to_nonstreaming_callee_with_multiple_streamingcompatible_callees_inline() #0 "aarch64_pstate_sm_enabled" {
-; CHECK-LABEL: define void @streaming_caller_to_nonstreaming_callee_with_multiple_streamingcompatible_callees_inline
-; CHECK-SAME: () #[[ATTR2]] {
-; CHECK-NEXT: call void @streaming_compatible_body()
-; CHECK-NEXT: call void @streaming_compatible_body()
+define void @invoke_opaque_fptr(ptr %fptr) alwaysinline personality ptr null {
+; CHECK-LABEL: define void @invoke_opaque_fptr
+; CHECK-SAME: (ptr [[FPTR:%.*]]) #[[ATTR6]] personality ptr null {
+; CHECK-NEXT: invoke void [[FPTR]]() #[[ATTR13:[0-9]+]]
+; CHECK-NEXT: to label [[NORMAL_RETURN:%.*]] unwind label [[UNWIND_CLEANUP:%.*]]
+; CHECK: normal_return:
+; CHECK-NEXT: ret void
+; CHECK: unwind_cleanup:
+; CHECK-NEXT: [[EH_INFO:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT: cleanup
+; CHECK-NEXT: resume { ptr, i32 } [[EH_INFO]]
+;
+ invoke void %fptr() "aarch64_pstate_sm_compatible"
+ to label %normal_return unwind label %unwind_cleanup
+
+normal_return:
+ ret void
+
+unwind_cleanup:
+ %eh_info = landingpad { ptr, i32 }
+ cleanup
+ resume { ptr, i32 } %eh_info
+}
+
+; Test that we don't inline '@invoke_opaque_fptr_callee' as it contains a streaming compatible callee,
+; but this time the callee is opaque pointer and called via 'invoke' rather than 'call'.
+define void @callee_has_streaming_compatible_invoke_call_dont_inline(ptr %fptr) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @callee_has_streaming_compatible_invoke_call_dont_inline
+; CHECK-SAME: (ptr [[FPTR:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: call void @invoke_opaque_fptr(ptr [[FPTR]])
; CHECK-NEXT: ret void
;
- call void @nonstreaming_caller_multiple_streaming_compatible_callees()
+ call void @invoke_opaque_fptr(ptr %fptr)
ret void
}
-define void @simple_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_enabled" {
+define void @simple_streaming_function(ptr %ptr) "aarch64_pstate_sm_enabled" {
; CHECK-LABEL: define void @simple_streaming_function
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR2]] {
; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[PTR]], align 16
@@ -687,7 +720,7 @@ define void @simple_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_enabled"
}
; Don't allow inlining a streaming function into a non-streaming function.
-define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) #0 {
+define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) {
; CHECK-LABEL: define void @non_streaming_caller_streaming_callee_dont_inline
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: call void @simple_streaming_function(ptr [[PTR]])
@@ -697,7 +730,7 @@ define void @non_streaming_caller_streaming_callee_dont_inline(ptr %ptr) #0 {
ret void
}
-define void @simple_locally_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_body" {
+define void @simple_locally_streaming_function(ptr %ptr) "aarch64_pstate_sm_body" {
; CHECK-LABEL: define void @simple_locally_streaming_function
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR3]] {
; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[PTR]], align 16
@@ -708,7 +741,7 @@ define void @simple_locally_streaming_function(ptr %ptr) #0 "aarch64_pstate_sm_b
}
; Don't allow inlining a locally-streaming function into a non-streaming function.
-define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) #0 {
+define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr) {
; CHECK-LABEL: define void @non_streaming_caller_locally_streaming_callee_dont_inline
; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: call void @simple_locally_streaming_function(ptr [[PTR]])
@@ -718,4 +751,355 @@ define void @non_streaming_caller_locally_streaming_callee_dont_inline(ptr %ptr)
ret void
}
-attributes #0 = { "target-features"="+sve,+sme" }
+;
+; Don't inline when there are vscale-dependent operations
+;
+
+; It is not safe to inline functions that have vscale-dependent operations in their body
+; when the streaming modes don't match up, unless the interface of the callee takes a
+; vl-dependent argument.
+define ptr @vscale_dependent_op(ptr %p, i64 %k) {
+; CHECK-LABEL: define ptr @vscale_dependent_op
+; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+ %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k
+ ret ptr %res
+}
+
+define ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 %k, <vscale x 4 x i32> %other) {
+; CHECK-LABEL: define ptr @vscale_dependent_op_vl_dependent_args
+; CHECK-SAME: (ptr [[P:%.*]], i64 [[K:%.*]], <vscale x 4 x i32> [[OTHER:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[RES:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 [[K]]
+; CHECK-NEXT: store <vscale x 4 x i32> [[OTHER]], ptr [[RES]], align 16
+; CHECK-NEXT: ret ptr [[RES]]
+;
+ %res = getelementptr <vscale x 4 x i32>, ptr %p, i64 %k
+ store <vscale x 4 x i32> %other, ptr %res
+ ret ptr %res
+}
+
+define void @vscale_dependent_operations(ptr %p, ptr %res1ptr, ptr %res2ptr) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @vscale_dependent_operations
+; CHECK-SAME: (ptr [[P:%.*]], ptr [[RES1PTR:%.*]], ptr [[RES2PTR:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: [[RES1:%.*]] = call ptr @vscale_dependent_op(ptr [[P]], i64 4)
+; CHECK-NEXT: [[RES_I:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[P]], i64 4
+; CHECK-NEXT: store <vscale x 4 x i32> zeroinitializer, ptr [[RES_I]], align 16
+; CHECK-NEXT: store ptr [[RES1]], ptr [[RES1PTR]], align 8
+; CHECK-NEXT: store ptr [[RES_I]], ptr [[RES2PTR]], align 8
+; CHECK-NEXT: ret void
+;
+ %res1 = call ptr @vscale_dependent_op(ptr %p, i64 4)
+ %res2 = call ptr @vscale_dependent_op_vl_dependent_args(ptr %p, i64 4, <vscale x 4 x i32> zeroinitializer)
+ store ptr %res1, ptr %res1ptr
+ store ptr %res2, ptr %res2ptr
+ ret void
+}
+
+; functions with scalable alloca's shouldn't be inlined if the streaming properties don't match
+; as a scalable alloca is a vl-dependent operation.
+define void @scalable_alloca_op(ptr %p) {
+; CHECK-LABEL: define void @scalable_alloca_op
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ALLOCA:%.*]] = alloca <vscale x 4 x i32>, align 1
+; CHECK-NEXT: call void [[P]](ptr [[ALLOCA]])
+; CHECK-NEXT: ret void
+;
+ %alloca = alloca <vscale x 4 x i32>, align 1
+ call void %p(ptr %alloca)
+ ret void
+}
+
+define void @incompatible_scalable_alloca_op_caller(ptr %p) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @incompatible_scalable_alloca_op_caller
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: call void @scalable_alloca_op(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call void @scalable_alloca_op(ptr %p)
+ ret void
+}
+
+; Generic intrinsics that take/return a scalable type, are vscale-dependent operations.
+define i64 @intrinsic_with_scalable_type(ptr %p) {
+; CHECK-LABEL: define i64 @intrinsic_with_scalable_type
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16
+; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]])
+; CHECK-NEXT: ret i64 [[RES]]
+;
+ %ld = load <vscale x 2 x i64>, ptr %p
+ %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld)
+ ret i64 %res
+}
+
+define i64 @intrinsic_with_scalable_type_vscale_must_match(ptr %p, <vscale x 4 x i32> %unused) {
+; CHECK-LABEL: define i64 @intrinsic_with_scalable_type_vscale_must_match
+; CHECK-SAME: (ptr [[P:%.*]], <vscale x 4 x i32> [[UNUSED:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[LD:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16
+; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD]])
+; CHECK-NEXT: ret i64 [[RES]]
+;
+ %ld = load <vscale x 2 x i64>, ptr %p
+ %res = call i64 @llvm.vector.reduce.add(<vscale x 2 x i64> %ld)
+ ret i64 %res
+}
+
+define i64 @generic_intrinsic_with_scalable_types(ptr %p) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define i64 @generic_intrinsic_with_scalable_types
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: [[RES1:%.*]] = call i64 @intrinsic_with_scalable_type(ptr [[P]])
+; CHECK-NEXT: [[LD_I:%.*]] = load <vscale x 2 x i64>, ptr [[P]], align 16
+; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[LD_I]])
+; CHECK-NEXT: [[RES:%.*]] = add i64 [[RES1]], [[RES_I]]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+ %res1 = call i64 @intrinsic_with_scalable_type(ptr %p)
+ %res2 = call i64 @intrinsic_with_scalable_type_vscale_must_match(ptr %p, <vscale x 4 x i32> zeroinitializer)
+ %res = add i64 %res1, %res2
+ ret i64 %res
+}
+
+;
+; Don't inline inline-asm when streaming-modes are incompatible, as the asm may
+; contain instructions that are invalid or may behave differently in the mode of
+; the caller.
+;
+
+define void @inline_asm() {
+; CHECK-LABEL: define void @inline_asm
+; CHECK-SAME: () #[[ATTR1]] {
+; CHECK-NEXT: call void asm sideeffect "", ""()
+; CHECK-NEXT: ret void
+;
+ call void asm sideeffect "", ""()
+ ret void
+}
+
+define void @incompatible_inline_asm_sm() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @incompatible_inline_asm_sm
+; CHECK-SAME: () #[[ATTR2]] {
+; CHECK-NEXT: call void @inline_asm()
+; CHECK-NEXT: ret void
+;
+ call void @inline_asm()
+ ret void
+}
+
+;
+; Don't inline functions that contain calls to SME ABI routines (like __arm_get_current_vg())
+;
+
+declare i64 @__arm_get_current_vg()
+
+define i64 @current_vg() {
+; CHECK-LABEL: define i64 @current_vg
+; CHECK-SAME: () #[[ATTR1]] {
+; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @__arm_get_current_vg()
+; CHECK-NEXT: ret i64 [[VSCALE]]
+;
+ %vscale = call i64 @__arm_get_current_vg()
+ ret i64 %vscale
+}
+
+define i64 @compatible_current_vg_caller() {
+; CHECK-LABEL: define i64 @compatible_current_vg_caller
+; CHECK-SAME: () #[[ATTR1]] {
+; CHECK-NEXT: [[VSCALE_I:%.*]] = call i64 @__arm_get_current_vg()
+; CHECK-NEXT: ret i64 [[VSCALE_I]]
+;
+ %vscale = call i64 @current_vg()
+ ret i64 %vscale
+}
+
+define i64 @incompatible_current_vg_caller() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define i64 @incompatible_current_vg_caller
+; CHECK-SAME: () #[[ATTR2]] {
+; CHECK-NEXT: [[VSCALE:%.*]] = call i64 @current_vg()
+; CHECK-NEXT: ret i64 [[VSCALE]]
+;
+ %vscale = call i64 @current_vg()
+ ret i64 %vscale
+}
+
+;
+; Don't inline NEON intrinsics into a streaming mode functions.
+;
+
+define i32 @neon_intrinsic(<4 x i32> %in) {
+; CHECK-LABEL: define i32 @neon_intrinsic
+; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[RES:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]])
+; CHECK-NEXT: ret i32 [[RES]]
+;
+ %res = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> %in)
+ ret i32 %res
+}
+
+define i32 @incompatible_neon_intrinsic_caller(<4 x i32> %in) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define i32 @incompatible_neon_intrinsic_caller
+; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: [[VSCALE:%.*]] = call i32 @neon_intrinsic(<4 x i32> [[IN]])
+; CHECK-NEXT: ret i32 [[VSCALE]]
+;
+ %vscale = call i32 @neon_intrinsic(<4 x i32> %in)
+ ret i32 %vscale
+}
+
+; a bit of a niche case, but if the caller uses ZA but is not in streaming-mode, a NEON intrinsic is safe.
+define i32 @compatible_neon_intrinsic_caller_za(<4 x i32> %in) "aarch64_pstate_za_enabled" {
+; CHECK-LABEL: define i32 @compatible_neon_intrinsic_caller_za
+; CHECK-SAME: (<4 x i32> [[IN:%.*]]) #[[ATTR7:[0-9]+]] {
+; CHECK-NEXT: [[RES_I:%.*]] = call i32 @llvm.aarch64.neon.uaddv.i32.v4i32(<4 x i32> [[IN]])
+; CHECK-NEXT: ret i32 [[RES_I]]
+;
+ %vscale = call i32 @neon_intrinsic(<4 x i32> %in)
+ ret i32 %vscale
+}
+
+;
+; Don't inline fixed-length vector code, as performance may be affected.
+;
+
+; Functions with fixed-length vectors shouldn't be inlined if the streaming properties don't match
+; as performance may be affected. However, when they have the alwaysinline property, they should
+; still be inlined.
+
+define void @fixed_length_vector_operation_alwaysinline(ptr %p) alwaysinline {
+; CHECK-LABEL: define void @fixed_length_vector_operation_alwaysinline
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16
+; CHECK-NEXT: ret void
+;
+ store <4 x i32> zeroinitializer, ptr %p
+ ret void
+}
+
+define void @fixed_length_vector_operation(ptr %p) {
+; CHECK-LABEL: define void @fixed_length_vector_operation
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16
+; CHECK-NEXT: ret void
+;
+ store <4 x i32> zeroinitializer, ptr %p
+ ret void
+}
+
+define void @fixed_length_vector_operations(ptr %p) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @fixed_length_vector_operations
+; CHECK-SAME: (ptr [[P:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[P]], align 16
+; CHECK-NEXT: call void @fixed_length_vector_operation(ptr [[P]])
+; CHECK-NEXT: ret void
+;
+ call void @fixed_length_vector_operation_alwaysinline(ptr %p)
+ call void @fixed_length_vector_operation(ptr %p)
+ ret void
+}
+
+;
+; strictfp and streaming-mode
+;
+
+define float @strict_fp(float %in) strictfp {
+; CHECK-LABEL: define float @strict_fp
+; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR8:[0-9]+]] {
+; CHECK-NEXT: [[RES:%.*]] = fadd float [[IN]], 4.200000e+01
+; CHECK-NEXT: ret float [[RES]]
+;
+ %res = fadd float %in, 42.0;
+ ret float %res
+}
+
+; floating point environment is different in streaming mode, so don't inline.
+define float @incompatible_fp_environment_sm(float %in) strictfp "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define float @incompatible_fp_environment_sm
+; CHECK-SAME: (float [[IN:%.*]]) #[[ATTR9:[0-9]+]] {
+; CHECK-NEXT: [[RES:%.*]] = call float @strict_fp(float [[IN]])
+; CHECK-NEXT: ret float [[RES]]
+;
+ %res = call float @strict_fp(float %in)
+ ret float %res
+}
+
+;
+; Test we can inline fixed-length gather/scatter operations into streaming functions,
+; as those can be scalarized by the code-generator.
+;
+
+define void @scatter_fixed_length(ptr %ptr_to_ptrs, ptr %ptr_to_vals) alwaysinline {
+; CHECK-LABEL: define void @scatter_fixed_length
+; CHECK-SAME: (ptr [[PTR_TO_PTRS:%.*]], ptr [[PTR_TO_VALS:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT: [[PTRS:%.*]] = load <2 x ptr>, ptr [[PTR_TO_PTRS]], align 16
+; CHECK-NEXT: [[VALS:%.*]] = load <2 x i64>, ptr [[PTR_TO_VALS]], align 16
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VALS]], <2 x ptr> align 8 [[PTRS]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ %ptrs = load <2 x ptr>, ptr %ptr_to_ptrs
+ %vals = load <2 x i64>, ptr %ptr_to_vals
+ call void @llvm.masked.scatter(<2 x i64> %vals, <2 x ptr> %ptrs, i32 8, <2 x i1> splat(i1 true))
+ ret void
+}
+
+define void @scatter_fixed_length_caller(ptr %ptr_to_ptrs, ptr %ptr_to_vals) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @scatter_fixed_length_caller
+; CHECK-SAME: (ptr [[PTR_TO_PTRS:%.*]], ptr [[PTR_TO_VALS:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: [[PTRS_I:%.*]] = load <2 x ptr>, ptr [[PTR_TO_PTRS]], align 16
+; CHECK-NEXT: [[VALS_I:%.*]] = load <2 x i64>, ptr [[PTR_TO_VALS]], align 16
+; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VALS_I]], <2 x ptr> align 8 [[PTRS_I]], <2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ call void @scatter_fixed_length(ptr %ptr_to_ptrs, ptr %ptr_to_vals)
+ ret void
+}
+
+;
+; Test that we never inline a scalable gather/scatter operation into a streaming function.
+;
+
+define void @scatter_scalable_length(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals) alwaysinline {
+; CHECK-LABEL: define void @scatter_scalable_length
+; CHECK-SAME: (<vscale x 2 x ptr> [[PTRS:%.*]], <vscale x 2 x i64> [[VALS:%.*]]) #[[ATTR6]] {
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VALS]], <vscale x 2 x ptr> align 8 [[PTRS]], <vscale x 2 x i1> splat (i1 true))
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.scatter(<vscale x 2 x i64> %vals, <vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> splat(i1 true))
+ ret void
+}
+
+define void @scatter_scalable_length_caller(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @scatter_scalable_length_caller
+; CHECK-SAME: (<vscale x 2 x ptr> [[PTRS:%.*]], <vscale x 2 x i64> [[VALS:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT: call void @scatter_scalable_length(<vscale x 2 x ptr> [[PTRS]], <vscale x 2 x i64> [[VALS]])
+; CHECK-NEXT: ret void
+;
+ call void @scatter_scalable_length(<vscale x 2 x ptr> %ptrs, <vscale x 2 x i64> %vals)
+ ret void
+}
+
+;
+; Test that we inline a function with a call to llvm.vscale() (which takes/returns
+; no scalable vectors) when the function has vl-dependent arguments. The case for
+; non-vl-dependent arguments is already tested above.
+;
+
+define i64 @llvm_vscale_vl_dependent_args(<vscale x 2 x i64> %in) {
+; CHECK-LABEL: define i64 @llvm_vscale_vl_dependent_args
+; CHECK-SAME: (<vscale x 2 x i64> [[IN:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[RES:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: ret i64 [[RES]]
+;
+ %res = call i64 @llvm.vscale.i64()
+ ret i64 %res
+}
+
+define i64 @llvm_vscale_vl_dependent_args_caller() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define i64 @llvm_vscale_vl_dependent_args_caller
+; CHECK-SAME: () #[[ATTR2]] {
+; CHECK-NEXT: [[RES_I:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: ret i64 [[RES_I]]
+;
+ %res = call i64 @llvm_vscale_vl_dependent_args(<vscale x 2 x i64> zeroinitializer)
+ ret i64 %res
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll
new file mode 100644
index 0000000000000..808749251ce0a
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AArch64/sme-pstatesm-inline-transitive.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -mattr=+sve,+sme -S -passes=inline -inlinedefault-threshold=1 < %s | FileCheck %s --check-prefixes=CHECK,TH1
+; RUN: opt -mattr=+sve,+sme -S -passes=inline -inlinedefault-threshold=25 < %s | FileCheck %s --check-prefixes=CHECK,TH25
+
+target triple = "aarch64"
+
+declare void @streaming_f() "aarch64_pstate_sm_enabled"
+
+define void @c() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: define void @c
+; CHECK-SAME: () #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: call void @streaming_f()
+; CHECK-NEXT: call void @streaming_f()
+; CHECK-NEXT: ret void
+;
+ call void @streaming_f()
+ call void @streaming_f()
+ ret void
+}
+
+; Don't inline the call to @c, as inlining a streaming function into a non-streaming function is considered incompatible.
+define void @b() {
+; CHECK-LABEL: define void @b
+; CHECK-SAME: () #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT: call void @c()
+; CHECK-NEXT: ret void
+;
+ call void @c()
+ ret void
+}
+
+; Inline the call to @c in @a, as that avoids streaming mode changes.
+define void @a() "aarch64_pstate_sm_enabled" {
+; TH1-LABEL: define void @a
+; TH1-SAME: () #[[ATTR1]] {
+; TH1-NEXT: call void @c()
+; TH1-NEXT: ret void
+;
+; TH25-LABEL: define void @a
+; TH25-SAME: () #[[ATTR1]] {
+; TH25-NEXT: call void @streaming_f()
+; TH25-NEXT: call void @streaming_f()
+; TH25-NEXT: ret void
+;
+ call void @b()
+ ret void
+}
>From 8ea9a49c19cfc6ace2f830521915aa0548ee0377 Mon Sep 17 00:00:00 2001
From: Matthew Nagy <matthew.nagy at sony.com>
Date: Wed, 7 Oct 2026 10:35:53 +0100
Subject: [PATCH 24/52] =?UTF-8?q?[TySan]=20Make=20TySan=20compatible=20wit?=
=?UTF-8?q?h=20sanitizer=20common=20interceptors=20(par=E2=80=A6=20(#19768?=
=?UTF-8?q?8)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
…tial #183310 reland)
Previous attempt here https://github.com/llvm/llvm-project/pull/183310
Some changes already added back here
https://github.com/llvm/llvm-project/pull/192413/changes
This shouldn't cause the same issue on apple platforms that the last
go-around did, and sets TySan up to be able to run on Sanitizer Common
tests and use its features.
(Edit:) Specifically it allows TySan to use all common sanitizer
interceptors, and adds a little more code to help it use the common
sanitizer features to handle deadly signals
---
compiler-rt/lib/tysan/tysan.cpp | 2 +
compiler-rt/lib/tysan/tysan_interceptors.cpp | 40 +++++++++++++++++++-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/compiler-rt/lib/tysan/tysan.cpp b/compiler-rt/lib/tysan/tysan.cpp
index 8b3b60dc4cf83..6b2adda10c625 100644
--- a/compiler-rt/lib/tysan/tysan.cpp
+++ b/compiler-rt/lib/tysan/tysan.cpp
@@ -507,6 +507,7 @@ static void TySanInitializePlatformEarly() {
namespace __tysan {
bool tysan_inited = false;
bool tysan_init_is_running;
+void InitializeDeadlySignals();
} // namespace __tysan
extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tysan_init() {
@@ -522,6 +523,7 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tysan_init() {
TySanInitializePlatformEarly();
InitializeInterceptors();
+ InitializeDeadlySignals();
if (!MmapFixedNoReserve(ShadowAddr(), AppAddr() - ShadowAddr()))
Die();
diff --git a/compiler-rt/lib/tysan/tysan_interceptors.cpp b/compiler-rt/lib/tysan/tysan_interceptors.cpp
index 1ba613d603ee8..1295f38093db6 100644
--- a/compiler-rt/lib/tysan/tysan_interceptors.cpp
+++ b/compiler-rt/lib/tysan/tysan_interceptors.cpp
@@ -22,6 +22,17 @@
#define TYSAN_INTERCEPT___STRDUP 0
#endif
+#if !SANITIZER_APPLE
+#define TYSAN_INTERCEPT_FUNC(name) \
+ do { \
+ if (!INTERCEPT_FUNCTION(name)) \
+ VReport(1, "TypeSanitizer: failed to intercept '%s'\n", #name); \
+ } while (0)
+#else // Apple interceptors don't need to be initialized with
+ // INTERCEPT_FUNCTION.
+#define TYSAN_INTERCEPT_FUNC(name)
+#endif
+
#if SANITIZER_GLIBC
extern "C" int mallopt(int param, int value);
#endif
@@ -29,6 +40,33 @@ extern "C" int mallopt(int param, int value);
using namespace __sanitizer;
using namespace __tysan;
+namespace __tysan {
+// Defined in tysan.cpp
+void OnStackUnwind(const SignalContext &sig, const void *,
+ BufferedStackTrace *stack);
+
+static void TysanOnDeadlySignal(int signo, void *siginfo, void *context) {
+ HandleDeadlySignal(siginfo, context, GetTid(), &OnStackUnwind, nullptr);
+}
+
+void InitializeDeadlySignals();
+} // namespace __tysan
+
+#define SIGNAL_INTERCEPTOR_ENTER() __tysan::InitializeDeadlySignals()
+#define COMMON_INTERCEPT_FUNCTION(name) TYSAN_INTERCEPT_FUNC(name)
+#include "sanitizer_common/sanitizer_signal_interceptors.inc"
+
+namespace __tysan {
+void InitializeDeadlySignals() {
+ static bool tysanSignalsInitialized = false;
+ if (tysanSignalsInitialized)
+ return;
+ InitializeSignalInterceptors();
+ InstallDeadlySignalHandlers(&TysanOnDeadlySignal);
+ tysanSignalsInitialized = true;
+}
+} // namespace __tysan
+
namespace {
struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
static bool UseImpl() { return !tysan_inited; }
@@ -221,7 +259,7 @@ void InitializeInterceptors() {
TYSAN_MAYBE_INTERCEPT_MEMALIGN;
TYSAN_MAYBE_INTERCEPT___LIBC_MEMALIGN;
TYSAN_MAYBE_INTERCEPT_PVALLOC;
- TYSAN_MAYBE_INTERCEPT_ALIGNED_ALLOC
+ TYSAN_MAYBE_INTERCEPT_ALIGNED_ALLOC;
INTERCEPT_FUNCTION(posix_memalign);
INTERCEPT_FUNCTION(memset);
>From 445981d83ba02fecc7fa68dd0ed5b2cc6b9b91ba Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 11:51:56 +0200
Subject: [PATCH 25/52] RegisterCoalescer: Restore subrange PHI inputs of
pruned undef values (#229551)
When joining erases an IMPLICIT_DEF or a copy, the liveness of undefined
lanes may have started at that instruction, so pruneSubRegValues removes
those subrange values. If such a value was live-out into a subrange PHI,
the PHI is left without an input from that predecessor, even though
after the join the lane value from the earlier def of the joined
register reaches it. The main range does not have this gap because
pruneValues collects the pruned end points and joinVirtRegs restores
them with extendToIndices.
Record the PHI kills of the pruned value, the predecessor block ends
where it is live-out into a PHI as in LiveIntervals::hasPHIKill, and
restore them with extendToIndices once operands are rewritten, together
with the existing subrange shrinking. The rewrite is needed so
computeSubRangeUndefs sees the defs of both registers. Only PHI kills
are restored; the in-block kills of a pruned value are reads of an undef
lane and get undef flags instead.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/CodeGen/RegisterCoalescer.cpp | 97 ++++++++++----
...oalescer-subreg-implicit-def-phi-input.mir | 75 +++++++++++
.../Thumb2/LowOverheadLoops/fast-fp-loops.ll | 20 +--
...oalescer-subreg-implicit-def-phi-input.mir | 126 ++++++++++++++++++
4 files changed, 283 insertions(+), 35 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/coalescer-subreg-implicit-def-phi-input.mir
create mode 100644 llvm/test/CodeGen/X86/coalescer-subreg-implicit-def-phi-input.mir
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 7b0fd8b1ed3be..bcb70421ac2a3 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -159,6 +159,10 @@ class RegisterCoalescer : private LiveRangeEdit::Delegate {
/// shrinkToUses() later.
LaneBitmask ShrinkMask;
+ /// PHI kills of pruned subrange values. The incoming value of a subrange PHI
+ /// may need to be re-extended from earlier defs after operands are rewritten.
+ SmallVector<std::pair<LaneBitmask, SlotIndex>, 4> PHIKills;
+
/// True if the main range of the currently coalesced intervals should be
/// checked for smaller live intervals.
bool ShrinkMainRange = false;
@@ -314,6 +318,11 @@ class RegisterCoalescer : private LiveRangeEdit::Delegate {
/// make sure to set it to the correct physical subregister.
void updateRegDefsUses(Register SrcReg, Register DstReg, unsigned SubIdx);
+ /// Fix up the subranges of \p LI after values were pruned by the join:
+ /// re-extend the pruned PHI inputs in PHIKills to their remaining defs, and
+ /// shrink the subranges in ShrinkMask.
+ void updatePrunedSubRanges(LiveInterval &LI);
+
/// If the given machine operand reads only undefined lanes add an undef
/// flag.
/// This can happen when undef uses were previously concealed by a copy
@@ -2055,6 +2064,33 @@ void RegisterCoalescer::setUndefOnPrunedSubRegUses(LiveInterval &LI,
LIS->shrinkToUses(&LI);
}
+void RegisterCoalescer::updatePrunedSubRanges(LiveInterval &LI) {
+ for (LiveInterval::SubRange &S : LI.subranges()) {
+ SmallVector<SlotIndex, 8> EndPoints;
+ for (auto [Mask, Idx] : PHIKills) {
+ if ((Mask & S.LaneMask).any() && LI.liveAt(Idx.getPrevSlot()))
+ EndPoints.push_back(Idx);
+ }
+
+ if (!EndPoints.empty()) {
+ SmallVector<SlotIndex, 8> Undefs;
+ LI.computeSubRangeUndefs(Undefs, S.LaneMask, *MRI,
+ *LIS->getSlotIndexes());
+ LIS->extendToIndices(S, EndPoints, Undefs);
+ }
+
+ if ((S.LaneMask & ShrinkMask).none())
+ continue;
+
+ LLVM_DEBUG(dbgs() << "Shrink LaneUses (Lane " << PrintLaneMask(S.LaneMask)
+ << ")\n");
+ LIS->shrinkToUses(S, LI.reg());
+ ShrinkMainRange = true;
+ }
+
+ LI.removeEmptySubRanges();
+}
+
RegisterCoalescer::JoinResult RegisterCoalescer::joinCopy(
MachineInstr *CopyMI,
SmallPtrSetImpl<MachineInstr *> &CurrentErasedInstrs) {
@@ -2193,6 +2229,7 @@ RegisterCoalescer::JoinResult RegisterCoalescer::joinCopy(
ShrinkMask = LaneBitmask::getNone();
ShrinkMainRange = false;
+ PHIKills.clear();
// Okay, attempt to join these two intervals. If one of the intervals being
// joined is a physreg and the join succeeds, this method always canonicalizes
@@ -2268,30 +2305,14 @@ RegisterCoalescer::JoinResult RegisterCoalescer::joinCopy(
updateRegDefsUses(CP.getDstReg(), CP.getDstReg(), CP.getDstIdx());
updateRegDefsUses(CP.getSrcReg(), CP.getDstReg(), CP.getSrcIdx());
- // Shrink subregister ranges if necessary.
- if (ShrinkMask.any()) {
- LiveInterval &LI = LIS->getInterval(CP.getDstReg());
- for (LiveInterval::SubRange &S : LI.subranges()) {
- if ((S.LaneMask & ShrinkMask).none())
- continue;
- LLVM_DEBUG(dbgs() << "Shrink LaneUses (Lane " << PrintLaneMask(S.LaneMask)
- << ")\n");
- LIS->shrinkToUses(S, LI.reg());
- ShrinkMainRange = true;
- }
- LI.removeEmptySubRanges();
- }
+ if (ShrinkMask.any() || !PHIKills.empty())
+ updatePrunedSubRanges(LIS->getInterval(CP.getDstReg()));
// CP.getSrcReg()'s live interval has been merged into CP.getDstReg's live
// interval. Since CP.getSrcReg() is in ToBeUpdated set and its live interval
// is not up-to-date, need to update the merged live interval here.
- if (ToBeUpdated.count(CP.getSrcReg()))
- ShrinkMainRange = true;
-
- if (ShrinkMainRange) {
- LiveInterval &LI = LIS->getInterval(CP.getDstReg());
- shrinkToUses(&LI);
- }
+ if (ShrinkMainRange || ToBeUpdated.count(CP.getSrcReg()))
+ shrinkToUses(&LIS->getInterval(CP.getDstReg()));
// SrcReg is guaranteed to be the register whose live interval that is
// being merged.
@@ -2714,7 +2735,9 @@ class JoinVals {
/// Removes subranges starting at copies that get removed. This sometimes
/// happens when undefined subranges are copied around. These ranges contain
/// no useful information and can be removed.
- void pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask);
+ void pruneSubRegValues(
+ LiveInterval &LI, LaneBitmask &ShrinkMask,
+ SmallVectorImpl<std::pair<LaneBitmask, SlotIndex>> &PHIKills);
/// Pruning values in subranges can lead to removing segments in these
/// subranges started by IMPLICIT_DEFs. The corresponding segments in
@@ -3371,6 +3394,23 @@ void JoinVals::pruneValues(JoinVals &Other,
}
}
+/// Collect the ends of the predecessors where \p VNI is live-out into a PHI
+/// value of \p LR. Mirrors LiveIntervals::hasPHIKill.
+static void collectPHIKills(const LiveRange &LR, const VNInfo *VNI,
+ const SlotIndexes &Indexes,
+ SmallVectorImpl<SlotIndex> &PHIKills) {
+ for (const VNInfo *PHI : LR.valnos) {
+ if (PHI->isUnused() || !PHI->isPHIDef())
+ continue;
+ const MachineBasicBlock *PHIMBB = Indexes.getMBBFromIndex(PHI->def);
+ for (const MachineBasicBlock *Pred : PHIMBB->predecessors()) {
+ SlotIndex End = Indexes.getMBBEndIdx(Pred);
+ if (LR.getVNInfoBefore(End) == VNI)
+ PHIKills.push_back(End);
+ }
+ }
+}
+
// Check if the segment consists of a copied live-through value (i.e. the copy
// in the block only extended the liveness, of an undef value which we may need
// to handle).
@@ -3417,7 +3457,9 @@ static bool isLiveThrough(const LiveQueryResult Q) {
/// from that copy cannot disappear. When pruning a value that started
/// at the removed copy, the corresponding identical value must be
/// extended to replace it.
-void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) {
+void JoinVals::pruneSubRegValues(
+ LiveInterval &LI, LaneBitmask &ShrinkMask,
+ SmallVectorImpl<std::pair<LaneBitmask, SlotIndex>> &PHIKills) {
// Look for values being erased.
bool DidPrune = false;
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
@@ -3448,6 +3490,8 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) {
(V.Identical && V.Resolution == CR_Erase && ValueOut->def == Def))) {
LLVM_DEBUG(dbgs() << "\t\tPrune sublane " << PrintLaneMask(S.LaneMask)
<< " at " << Def << "\n");
+ SmallVector<SlotIndex, 8> ValuePHIKills;
+ collectPHIKills(S, ValueOut, *Indexes, ValuePHIKills);
SmallVector<SlotIndex, 8> EndPoints;
LIS->pruneValue(S, Def, &EndPoints);
DidPrune = true;
@@ -3460,6 +3504,9 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) {
// then we can't simply prune V from S. V needs to be replaced
// with V.OtherVNI.
LIS->extendToIndices(S, EndPoints);
+ } else {
+ for (SlotIndex Kill : ValuePHIKills)
+ PHIKills.emplace_back(S.LaneMask, Kill);
}
// We may need to eliminate the subrange if the copy introduced a live
@@ -3788,15 +3835,15 @@ RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
// having stale segments.
LHSVals.pruneMainSegments(LHS, ShrinkMainRange);
- LHSVals.pruneSubRegValues(LHS, ShrinkMask);
- RHSVals.pruneSubRegValues(LHS, ShrinkMask);
+ LHSVals.pruneSubRegValues(LHS, ShrinkMask, PHIKills);
+ RHSVals.pruneSubRegValues(LHS, ShrinkMask, PHIKills);
} else if (TrackSubRegLiveness && !CP.getDstIdx() && CP.getSrcIdx()) {
LHS.createSubRangeFrom(LIS->getVNInfoAllocator(),
CP.getNewRC()->getLaneMask(), LHS);
mergeSubRangeInto(LHS, RHS, TRI->getSubRegIndexLaneMask(CP.getSrcIdx()), CP,
CP.getDstIdx());
LHSVals.pruneMainSegments(LHS, ShrinkMainRange);
- LHSVals.pruneSubRegValues(LHS, ShrinkMask);
+ LHSVals.pruneSubRegValues(LHS, ShrinkMask, PHIKills);
}
// The merging algorithm in LiveInterval::join() can't handle conflicting
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-subreg-implicit-def-phi-input.mir b/llvm/test/CodeGen/AMDGPU/coalescer-subreg-implicit-def-phi-input.mir
new file mode 100644
index 0000000000000..d154b823483c3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-subreg-implicit-def-phi-input.mir
@@ -0,0 +1,75 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-coalescing -passes=register-coalescer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -verify-coalescing -passes='register-coalescer,print<live-intervals>' -o /dev/null %s 2>&1 | FileCheck -check-prefix=INTERVALS %s
+
+# The sub1 IMPLICIT_DEF in bb.3 is erased by coalescing. sub1 must stay live
+# into the PHI in bb.5 along the bb.3 edge by extending
+# the segment of value 0 of sub1 to the end of bb.3.
+
+# INTERVALS-LABEL: Live intervals for machine function: implicit_def_phi_input:
+# INTERVALS: %2 [48r,224B:0)[240r,256B:1)[256B,288r:2) 0 at 48r 1 at 240r 2 at 256B-phi L0000000000000003 [48r,80B:0)[160B,192r:0) 0 at 48r L000000000000000C [48r,224B:0)[240r,256B:1)[256B,288r:3) 0 at 48r 1 at 240r 2 at x 3 at 256B-phi weight:
+
+---
+name: implicit_def_phi_input
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: implicit_def_phi_input
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 undef %1:vreg_64, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_BRANCH %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD undef %5:vreg_64, [[GLOBAL_LOAD_DWORDX2_]].sub0, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.5, implicit undef $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]].sub1:vreg_64 = V_MOV_B32_e32 2, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD undef %7:vreg_64, [[GLOBAL_LOAD_DWORDX2_]].sub1, 0, 0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ bb.0:
+ S_BRANCH %bb.1
+
+ bb.1:
+ %0:vreg_64 = GLOBAL_LOAD_DWORDX2 undef %22:vreg_64, 0, 0, implicit $exec
+ S_CBRANCH_SCC1 %bb.3, implicit undef $scc
+
+ bb.2:
+ undef %1.sub1:vreg_64 = COPY %0.sub1
+ %2:vgpr_32 = COPY %1.sub1
+ %3:vgpr_32 = COPY %2
+ S_BRANCH %bb.5
+
+ bb.3:
+ %3:vgpr_32 = IMPLICIT_DEF
+ GLOBAL_STORE_DWORD undef %20:vreg_64, %0.sub0, 0, 0, implicit $exec
+ S_CBRANCH_SCC1 %bb.5, implicit undef $scc
+
+ bb.4:
+ %3:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
+
+ bb.5:
+ %5:vgpr_32 = COPY %3
+ GLOBAL_STORE_DWORD undef %6:vreg_64, %5, 0, 0, implicit $exec
+ S_BRANCH %bb.1
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index ca1f1036ea674..93c90b3d88325 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -280,7 +280,7 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(ptr nocapture readonly %b, ptr
; CHECK-NEXT: bxeq lr
; CHECK-NEXT: .LBB2_1: @ %vector.ph
; CHECK-NEXT: push {r4, r5, r7, lr}
-; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14}
; CHECK-NEXT: sub sp, #8
; CHECK-NEXT: adds r3, r2, #3
; CHECK-NEXT: vmov.i32 q5, #0x0
@@ -346,8 +346,8 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(ptr nocapture readonly %b, ptr
; CHECK-NEXT: bpl .LBB2_8
; CHECK-NEXT: .LBB2_7: @ %cond.load10
; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
-; CHECK-NEXT: vldr.16 s22, [r0, #6]
-; CHECK-NEXT: vins.f16 s21, s22
+; CHECK-NEXT: vldr.16 s24, [r0, #6]
+; CHECK-NEXT: vins.f16 s21, s24
; CHECK-NEXT: .LBB2_8: @ %else11
; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
; CHECK-NEXT: vcmp.u32 cs, q2, q4
@@ -395,8 +395,8 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(ptr nocapture readonly %b, ptr
; CHECK-NEXT: .LBB2_14: @ %cond.load7
; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
; CHECK-NEXT: vldr.16 s21, [r0, #4]
-; CHECK-NEXT: vmovx.f16 s22, s0
-; CHECK-NEXT: vins.f16 s21, s22
+; CHECK-NEXT: vmovx.f16 s24, s0
+; CHECK-NEXT: vins.f16 s21, s24
; CHECK-NEXT: lsls r2, r2, #28
; CHECK-NEXT: bmi .LBB2_7
; CHECK-NEXT: b .LBB2_8
@@ -414,14 +414,14 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(ptr nocapture readonly %b, ptr
; CHECK-NEXT: .LBB2_17: @ %cond.load20
; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
; CHECK-NEXT: vldr.16 s25, [r1, #4]
-; CHECK-NEXT: vmovx.f16 s26, s0
-; CHECK-NEXT: vins.f16 s25, s26
+; CHECK-NEXT: vmovx.f16 s28, s0
+; CHECK-NEXT: vins.f16 s25, s28
; CHECK-NEXT: lsls r2, r2, #28
; CHECK-NEXT: bpl.w .LBB2_2
; CHECK-NEXT: .LBB2_18: @ %cond.load23
; CHECK-NEXT: @ in Loop: Header=BB2_3 Depth=1
-; CHECK-NEXT: vldr.16 s26, [r1, #6]
-; CHECK-NEXT: vins.f16 s25, s26
+; CHECK-NEXT: vldr.16 s28, [r1, #6]
+; CHECK-NEXT: vins.f16 s25, s28
; CHECK-NEXT: b .LBB2_2
; CHECK-NEXT: .LBB2_19: @ %middle.block
; CHECK-NEXT: vdup.32 q0, r12
@@ -433,7 +433,7 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(ptr nocapture readonly %b, ptr
; CHECK-NEXT: vmov r0, s1
; CHECK-NEXT: vadd.f32 q0, q0, r0
; CHECK-NEXT: add sp, #8
-; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14}
; CHECK-NEXT: pop {r4, r5, r7, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.20:
diff --git a/llvm/test/CodeGen/X86/coalescer-subreg-implicit-def-phi-input.mir b/llvm/test/CodeGen/X86/coalescer-subreg-implicit-def-phi-input.mir
new file mode 100644
index 0000000000000..c586172d664dd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coalescer-subreg-implicit-def-phi-input.mir
@@ -0,0 +1,126 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-subreg-liveness -verify-coalescing -passes=register-coalescer -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-subreg-liveness -verify-coalescing -passes='register-coalescer,print<live-intervals>' -o /dev/null %s 2>&1 | FileCheck -check-prefix=INTERVALS %s
+
+# Erasing the high-byte IMPLICIT_DEF in bb.3 makes the high-byte value from
+# bb.1 reach the use in bb.5, so the high-byte subrange is live through bb.3.
+
+# INTERVALS-LABEL: Live intervals for machine function: recompute_after_subreg_implicit_def:
+# INTERVALS: %1 [48r,240B:0)[256r,288B:1)[288B,320r:2) 0 at 48r 1 at 256r 2 at 288B-phi L0000000000000001 [48r,96B:0)[176B,208r:0) 0 at 48r L0000000000000002 [48r,240B:0)[256r,288B:1)[288B,320r:3) 0 at 48r 1 at 256r 2 at x 3 at 288B-phi weight:
+
+---
+name: recompute_after_subreg_implicit_def
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: recompute_after_subreg_implicit_def
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOV16ri:%[0-9]+]]:gr16_abcd = MOV16ri 1
+ ; CHECK-NEXT: TEST8rr [[MOV16ri]].sub_8bit, [[MOV16ri]].sub_8bit, implicit-def $eflags
+ ; CHECK-NEXT: JCC_1 %bb.3, 5, implicit killed $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: JMP_1 %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TEST8rr [[MOV16ri]].sub_8bit, [[MOV16ri]].sub_8bit, implicit-def $eflags
+ ; CHECK-NEXT: JCC_1 %bb.5, 5, implicit killed $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: undef [[MOV16ri:%[0-9]+]].sub_8bit_hi:gr16_abcd = MOV8ri 2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TEST8rr [[MOV16ri]].sub_8bit_hi, [[MOV16ri]].sub_8bit_hi, implicit-def dead $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ bb.0:
+ JMP_1 %bb.1
+
+ bb.1:
+ %0:gr16_abcd = MOV16ri 1
+ TEST8rr %0.sub_8bit, %0.sub_8bit, implicit-def $eflags
+ JCC_1 %bb.3, 5, implicit killed $eflags
+
+ bb.2:
+ undef %1.sub_8bit_hi:gr16_abcd = COPY %0.sub_8bit_hi
+ %2:gr8_abcd_h = COPY %1.sub_8bit_hi
+ %3:gr8_abcd_h = COPY %2
+ JMP_1 %bb.5
+
+ bb.3:
+ %3:gr8_abcd_h = IMPLICIT_DEF
+ TEST8rr %0.sub_8bit, %0.sub_8bit, implicit-def $eflags
+ JCC_1 %bb.5, 5, implicit killed $eflags
+
+ bb.4:
+ %4:gr8_abcd_h = MOV8ri 2
+ %3:gr8_abcd_h = COPY %4
+
+ bb.5:
+ %5:gr8_abcd_h = COPY %3
+ TEST8rr %5, %5, implicit-def dead $eflags
+ JMP_1 %bb.1
+...
+
+# The high-byte IMPLICIT_DEF in the loop latch is erased by coalescing. The
+# high byte must stay live around the backedge by extending the segment of
+# value 2 to the beginning of bb.3.
+
+# INTERVALS-LABEL: Live intervals for machine function: loop_backedge:
+# INTERVALS: %0 [16r,48B:0)[48B,192r:1) 0 at 16r 1 at 48B-phi L0000000000000002 [16r,48B:0)[48B,176B:2) 0 at 16r 1 at x 2 at 48B-phi L0000000000000001 [16r,192r:0) 0 at 16r weight:
+
+---
+name: loop_backedge
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: loop_backedge
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[MOV16ri:%[0-9]+]]:gr16_abcd = MOV16ri 1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TEST8rr [[MOV16ri]].sub_8bit_hi, [[MOV16ri]].sub_8bit_hi, implicit-def $eflags
+ ; CHECK-NEXT: JCC_1 %bb.3, 5, implicit killed $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: TEST8rr [[MOV16ri]].sub_8bit, [[MOV16ri]].sub_8bit, implicit-def dead $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: RET 0, implicit [[MOV16ri]].sub_8bit
+ bb.0:
+ %0:gr16_abcd = MOV16ri 1
+ %3:gr8_abcd_h = COPY %0.sub_8bit_hi
+
+ bb.1:
+ %5:gr8_abcd_h = COPY %3
+ TEST8rr %5, %5, implicit-def $eflags
+ JCC_1 %bb.3, 5, implicit killed $eflags
+
+ bb.2:
+ %3:gr8_abcd_h = IMPLICIT_DEF
+ TEST8rr %0.sub_8bit, %0.sub_8bit, implicit-def dead $eflags
+ JMP_1 %bb.1
+
+ bb.3:
+ RET 0, implicit %0.sub_8bit
+
+...
>From 7348e7196bfbd52576b7a180e5edff64cee3b3bf Mon Sep 17 00:00:00 2001
From: Vikram Hegde <Vikram.Hegde at amd.com>
Date: Wed, 7 Oct 2026 15:26:31 +0530
Subject: [PATCH 26/52] [AMDGPU][GIsel] Properly handle unsupported intrinsic
translation (#228040)
unsupported intrinsic results are defined with G_IMPLICIT_DEF,
preventing malformed translation. Basically tries to mimic SDAG path
which creates poison results.
Assisted by: Opus 5
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 33 ++++++++++++++------
llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll | 4 +++
2 files changed, 27 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index f87d57f14ba0c..a18292dddbe9e 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -337,6 +337,11 @@ class IRTranslatorImpl {
const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos = {});
+ /// Report an intrinsic the subtarget does not support and define its results
+ /// with G_IMPLICIT_DEF. Prevents creating a malformed MIR.
+ bool handleUnsupportedIntrinsic(const CallBase &CB, Intrinsic::ID ID,
+ MachineIRBuilder &MIRBuilder);
+
/// When an invoke or a cleanupret unwinds to the next EH pad, there are
/// many places it could ultimately go. In the IR, we have a single unwind
/// destination, but in the machine CFG, we enumerate all the possible blocks.
@@ -3670,11 +3675,8 @@ bool IRTranslatorImpl::translateCall(const User &U,
assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
- if (!MF->getSubtarget().isIntrinsicSupported(ID)) {
- const Function &Fn = MF->getFunction();
- Fn.getContext().diagnose(
- DiagnosticInfoUnsupportedTargetIntrinsic(Fn, ID, CI.getDebugLoc()));
- }
+ if (!MF->getSubtarget().isIntrinsicSupported(ID))
+ return handleUnsupportedIntrinsic(CI, ID, MIRBuilder);
if (translateKnownIntrinsic(CI, ID, MIRBuilder))
return true;
@@ -3685,15 +3687,26 @@ bool IRTranslatorImpl::translateCall(const User &U,
return translateIntrinsic(CI, ID, MIRBuilder, Infos);
}
+bool IRTranslatorImpl::handleUnsupportedIntrinsic(
+ const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
+ const Function &F = MF->getFunction();
+ F.getContext().diagnose(
+ DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
+
+ if (!CB.getType()->isVoidTy()) {
+ for (Register Reg : getOrCreateVRegs(CB))
+ MIRBuilder.buildUndef(Reg);
+ }
+
+ return true;
+}
+
/// Translate a call or callbr to an intrinsic.
bool IRTranslatorImpl::translateIntrinsic(
const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
ArrayRef<TargetLowering::IntrinsicInfo> TgtMemIntrinsicInfos) {
- if (!MF->getSubtarget().isIntrinsicSupported(ID)) {
- const Function &F = MF->getFunction();
- F.getContext().diagnose(
- DiagnosticInfoUnsupportedTargetIntrinsic(F, ID, CB.getDebugLoc()));
- }
+ if (!MF->getSubtarget().isIntrinsicSupported(ID))
+ return handleUnsupportedIntrinsic(CB, ID, MIRBuilder);
ArrayRef<Register> ResultRegs;
if (!CB.getType()->isVoidTy())
diff --git a/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll b/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll
index 192c8b4d95039..bf108456b8bab 100644
--- a/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-wakeup-barrier.ll
@@ -5,6 +5,10 @@
; RUN: not llc -global-isel=0 -mtriple=amdgpu12.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: not llc -global-isel=1 -mtriple=amdgpu12.00 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
+; Check that we report the error rather than crashing on pre-gfx12.
+; RUN: not llc -global-isel=0 -mtriple=amdgpu9.42 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -global-isel=1 -mtriple=amdgpu9.42 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR %s
+
; ERR: error: <unknown>:0:0: in function @kernel1 void (ptr addrspace(1), ptr addrspace(15)): llvm.amdgcn.s.wakeup.barrier requires target feature 's-wakeup-barrier-inst'
@bar = internal addrspace(15) global target("amdgcn.named.barrier", 0) poison
>From 869dd2d1903da070326f62ef571005688117bc1b Mon Sep 17 00:00:00 2001
From: Charles Zablit <c_zablit at apple.com>
Date: Wed, 7 Oct 2026 10:57:53 +0100
Subject: [PATCH 27/52] [lldb][Windows] Honor the last stdio file action when
launching a process (#229435)
lldb starts lldb-server with its stdio first closed and then reopened on
the null device. On Windows, the launcher only looked at the first of
those two actions, so lldb-server inherited lldb's own stdout and
stderr.
lldb-server's messages, like "Connection established." and "lldb-server
exiting...", are buffered and get written when it exits, in the middle
of lldb's output. That makes Shell tests that check lldb's output flaky,
such as `NativePDB/local-variables.cpp`.
The launcher now uses the last action that opens or closes each stdio
descriptor, which is the order the actions are applied in. Duplicating a
descriptor onto another one doesn't change it, so those actions are
skipped.
The new test runs a program through lldb-server and checks that none of
lldb-server's messages show up in lldb's output.
In a stress run of `local-variables.cpp`, failures went from 11 in 640
runs to none.
---
.../source/Host/windows/ProcessLauncherWindows.cpp | 8 +++++++-
.../Shell/Process/Windows/lldb_server_stdio.cpp | 14 ++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
create mode 100644 lldb/test/Shell/Process/Windows/lldb_server_stdio.cpp
diff --git a/lldb/source/Host/windows/ProcessLauncherWindows.cpp b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
index c0583f54bb316..cd09ff86a1946 100644
--- a/lldb/source/Host/windows/ProcessLauncherWindows.cpp
+++ b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
@@ -336,7 +336,13 @@ llvm::ErrorOr<std::vector<HANDLE>> ProcessLauncherWindows::GetInheritedHandles(
HANDLE
ProcessLauncherWindows::GetStdioHandle(const ProcessLaunchInfo &launch_info,
int fd) {
- const FileAction *action = launch_info.GetFileActionForFD(fd);
+ const FileAction *action = nullptr;
+ for (size_t i = 0; i < launch_info.GetNumFileActions(); ++i) {
+ const FileAction *candidate = launch_info.GetFileActionAtIndex(i);
+ if (candidate->GetFD() == fd &&
+ candidate->GetAction() != FileAction::eFileActionDuplicate)
+ action = candidate;
+ }
if (action == nullptr)
return nullptr;
const std::string path = action->GetFileSpec().GetPath();
diff --git a/lldb/test/Shell/Process/Windows/lldb_server_stdio.cpp b/lldb/test/Shell/Process/Windows/lldb_server_stdio.cpp
new file mode 100644
index 0000000000000..1046ddf2da169
--- /dev/null
+++ b/lldb/test/Shell/Process/Windows/lldb_server_stdio.cpp
@@ -0,0 +1,14 @@
+// clang-format off
+
+// lldb-server's stdio is the null device, so nothing it prints ends up in
+// lldb's output.
+
+// REQUIRES: target-windows
+// RUN: %build -o %t.exe -- %s
+// RUN: env LLDB_USE_LLDB_SERVER=1 %lldb -b -o run -f %t.exe 2>&1 | \
+// RUN: FileCheck %s --implicit-check-not="Connection established" \
+// RUN: --implicit-check-not="lldb-server exiting"
+
+// CHECK: Process {{[0-9]+}} exited with status = 0
+
+int main() { return 0; }
>From c198ce3123dda9510f303c9badcf13f55bd04abd Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 12:01:41 +0200
Subject: [PATCH 28/52] Targets: Remove redundant TRI arguments from InstrInfo
helpers (#228165)
Continue with cleanups enabled by #158224. This is directly available in
TargetInstrInfo.
Co-authored-by: Claude Opus 5 <noreply at anthropic.com>
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 14 ++++++--------
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 3 +--
.../lib/Target/Hexagon/HexagonHardwareLoops.cpp | 4 ++--
llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp | 4 ++--
llvm/lib/Target/Hexagon/HexagonInstrInfo.h | 3 +--
llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp | 6 +++---
llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp | 2 +-
.../Target/Hexagon/HexagonVLIWPacketizer.cpp | 4 ++--
llvm/lib/Target/Lanai/LanaiInstrInfo.cpp | 15 +++++++--------
llvm/lib/Target/Lanai/LanaiInstrInfo.h | 3 +--
llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 17 +++++++++--------
llvm/lib/Target/PowerPC/PPCInstrInfo.h | 3 +--
llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 10 ++++------
llvm/lib/Target/RISCV/RISCVInstrInfo.h | 3 +--
14 files changed, 41 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index c1f2515134c2e..1ed73b119ab73 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1626,7 +1626,6 @@ bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
const MachineInstr &MIa, const MachineInstr &MIb) const {
- const TargetRegisterInfo *TRI = &getRegisterInfo();
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
TypeSize WidthA(0, false), WidthB(0, false);
@@ -1647,9 +1646,9 @@ bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
// If OffsetAIsScalable and OffsetBIsScalable are both true, they
// are assumed to have the same scale (vscale).
if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
- WidthA, TRI) &&
+ WidthA) &&
getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
- WidthB, TRI)) {
+ WidthB)) {
if (BaseOpA->isIdenticalTo(*BaseOpB) &&
OffsetAIsScalable == OffsetBIsScalable) {
int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
@@ -3690,7 +3689,7 @@ bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
const MachineOperand *BaseOp;
TypeSize WidthN(0, false);
if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
- WidthN, &RI))
+ WidthN))
return false;
// The maximum vscale is 16 under AArch64, return the maximal extent for the
// vector.
@@ -4696,8 +4695,7 @@ static bool isPostIndexLdStOpcode(unsigned Opcode) {
bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
- bool &OffsetIsScalable, TypeSize &Width,
- const TargetRegisterInfo *TRI) const {
+ bool &OffsetIsScalable, TypeSize &Width) const {
assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
// Handle only loads/stores with base register followed by immediate offset.
if (LdSt.getNumExplicitOperands() == 3) {
@@ -11268,8 +11266,8 @@ void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
// Is this a load or store with an immediate offset with SP as the base?
if (!MI.mayLoadOrStore() ||
- !getMemOperandWithOffsetWidth(MI, Base, Offset, OffsetIsScalable, Width,
- &RI) ||
+ !getMemOperandWithOffsetWidth(MI, Base, Offset, OffsetIsScalable,
+ Width) ||
(Base->isReg() && Base->getReg() != AArch64::SP))
continue;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 4654fce39b02e..7efdc3b7026ff 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -335,8 +335,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
bool getMemOperandWithOffsetWidth(const MachineInstr &MI,
const MachineOperand *&BaseOp,
int64_t &Offset, bool &OffsetIsScalable,
- TypeSize &Width,
- const TargetRegisterInfo *TRI) const;
+ TypeSize &Width) const;
/// Return the immediate offset of the base register in a load/store \p LdSt.
MachineOperand &getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const;
diff --git a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
index b1b02907b2fcb..263a01cd04f6b 100644
--- a/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonHardwareLoops.cpp
@@ -1375,7 +1375,7 @@ bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L,
// if the immediate fits in the instructions. Otherwise, we need to
// create a new virtual register.
int64_t CountImm = TripCount->getImm();
- if (!TII->isValidOffset(LOOP_i, CountImm, TRI)) {
+ if (!TII->isValidOffset(LOOP_i, CountImm)) {
Register CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass);
BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::A2_tfrsi), CountReg)
.addImm(CountImm);
@@ -1931,7 +1931,7 @@ bool HexagonHardwareLoops::fixupInductionVariable(MachineLoop *L) {
// the immediate to be constant-extended. There are some exceptions
// though. Make sure the new combination will work.
if (CmpImmOp->isImm() && !TII->isExtendable(*PredDef) &&
- !TII->isValidOffset(PredDef->getOpcode(), CmpImm, TRI, false))
+ !TII->isValidOffset(PredDef->getOpcode(), CmpImm, false))
return false;
// Make sure that the compare happens after the bump. Otherwise,
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index e78bf6b726663..cf21b39008747 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -2785,7 +2785,7 @@ bool HexagonInstrInfo::isValidAutoIncImm(const EVT VT, int Offset) const {
}
bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset,
- const TargetRegisterInfo *TRI, bool Extend) const {
+ bool Extend) const {
// This function is to check whether the "Offset" is in the correct range of
// the given "Opcode". If "Offset" is not in the correct range, "A2_addi" is
// inserted to calculate the final address. Due to this reason, the function
@@ -2848,7 +2848,7 @@ bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset,
case Hexagon::V6_vgathermhq_pseudo:
case Hexagon::V6_vgathermwq_pseudo:
case Hexagon::V6_vgathermhwq_pseudo: {
- unsigned VectorSize = TRI->getSpillSize(Hexagon::HvxVRRegClass);
+ unsigned VectorSize = RegInfo.getSpillSize(Hexagon::HvxVRRegClass);
assert(isPowerOf2_32(VectorSize));
if (Offset & (VectorSize-1))
return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 5e5c93b2b4ccb..fa40614b46565 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -423,8 +423,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
const MachineInstr &MI2) const;
bool isHVXVec(const MachineInstr &MI) const;
bool isValidAutoIncImm(const EVT VT, const int Offset) const;
- bool isValidOffset(unsigned Opcode, int Offset,
- const TargetRegisterInfo *TRI, bool Extend = true) const;
+ bool isValidOffset(unsigned Opcode, int Offset, bool Extend = true) const;
bool isVecAcc(const MachineInstr &MI) const;
bool isVecALU(const MachineInstr &MI) const;
bool isVecUsableNextPacket(const MachineInstr &ProdMI,
diff --git a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
index 7a4b51ad11082..9d9a919c95f9a 100644
--- a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
@@ -364,13 +364,13 @@ bool HexagonOptAddrMode::isValidOffset(MachineInstr *MI, int Offset) {
case Hexagon::V6_vgathermhq_pseudo:
case Hexagon::V6_vgathermwq_pseudo:
case Hexagon::V6_vgathermhwq_pseudo:
- return HII->isValidOffset(MI->getOpcode(), Offset, HRI, false);
+ return HII->isValidOffset(MI->getOpcode(), Offset, false);
default:
if (HII->getAddrMode(*MI) == HexagonII::BaseImmOffset) {
// The immediates are mentioned in multiples of vector counts
unsigned AlignMask = HII->getMemAccessSize(*MI) - 1;
if ((AlignMask & Offset) == 0)
- return HII->isValidOffset(MI->getOpcode(), Offset, HRI, false);
+ return HII->isValidOffset(MI->getOpcode(), Offset, false);
}
return false;
}
@@ -399,7 +399,7 @@ bool HexagonOptAddrMode::isValidOffset(MachineInstr *MI, int Offset) {
if ((AlignMask & Offset) != 0)
return false;
- return HII->isValidOffset(MI->getOpcode(), Offset, HRI, false);
+ return HII->isValidOffset(MI->getOpcode(), Offset, false);
}
unsigned HexagonOptAddrMode::getBaseOpPosition(MachineInstr *MI) {
diff --git a/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp b/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp
index 8797a38ef0735..cbfb31ca6c62f 100644
--- a/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonRegisterInfo.cpp
@@ -290,7 +290,7 @@ bool HexagonRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
break;
}
- if (!HII.isValidOffset(Opc, RealOffset, this)) {
+ if (!HII.isValidOffset(Opc, RealOffset)) {
// If the offset is not valid, calculate the address in a temporary
// register and use it with offset 0.
int InstOffset = 0;
diff --git a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
index 8026a8a177e59..ae50cd17ccb7a 100644
--- a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -475,7 +475,7 @@ bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) {
unsigned FrameSize = MF.getFrameInfo().getStackSize();
MachineOperand &Off = MI.getOperand(1);
int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
- if (HII->isValidOffset(Opc, NewOff, HRI)) {
+ if (HII->isValidOffset(Opc, NewOff)) {
Off.setImm(NewOff);
return true;
}
@@ -526,7 +526,7 @@ bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) {
return false;
int64_t Offset = MI.getOperand(OPI).getImm();
- if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI))
+ if (!HII->isValidOffset(MI.getOpcode(), Offset + Incr))
return false;
MI.getOperand(OPI).setImm(Offset + Incr);
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index 33773023f6664..5612e137b6a97 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -99,13 +99,12 @@ bool LanaiInstrInfo::areMemAccessesTriviallyDisjoint(
// base registers are identical, and the offset of a lower memory access +
// the width doesn't overlap the offset of a higher memory access,
// then the memory accesses are different.
- const TargetRegisterInfo *TRI = &getRegisterInfo();
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
LocationSize WidthA = LocationSize::precise(0),
WidthB = LocationSize::precise(0);
- if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
- getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
+ if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA) &&
+ getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
int LowOffset = std::min(OffsetA, OffsetB);
int HighOffset = std::max(OffsetA, OffsetB);
@@ -735,9 +734,10 @@ Register LanaiInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
return 0;
}
-bool LanaiInstrInfo::getMemOperandWithOffsetWidth(
- const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
- LocationSize &Width, const TargetRegisterInfo * /*TRI*/) const {
+bool LanaiInstrInfo::getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
+ const MachineOperand *&BaseOp,
+ int64_t &Offset,
+ LocationSize &Width) const {
// Handle only loads/stores with base register followed by immediate offset
// and with add as ALU op.
if (LdSt.getNumOperands() != 4)
@@ -793,8 +793,7 @@ bool LanaiInstrInfo::getMemOperandsWithOffsetWidth(
case Lanai::LDBz_RI:
const MachineOperand *BaseOp;
OffsetIsScalable = false;
- if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width,
- &getRegisterInfo()))
+ if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
return false;
BaseOps.push_back(BaseOp);
return true;
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
index 2ac81edf49fad..a0c5a7f360a4c 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
@@ -77,8 +77,7 @@ class LanaiInstrInfo : public LanaiGenInstrInfo {
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
- int64_t &Offset, LocationSize &Width,
- const TargetRegisterInfo *TRI) const;
+ int64_t &Offset, LocationSize &Width) const;
std::pair<unsigned, unsigned>
decomposeMachineOperandsTargetFlags(unsigned TF) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index e8935ceae4fb3..67b8f481dfcad 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -2894,7 +2894,7 @@ bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
const MachineOperand *BaseOp;
OffsetIsScalable = false;
- if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, &RI))
+ if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
return false;
BaseOps.push_back(BaseOp);
return true;
@@ -2984,8 +2984,8 @@ bool PPCInstrInfo::shouldClusterMemOps(
LocationSize Width1 = LocationSize::precise(0),
Width2 = LocationSize::precise(0);
const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
- if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, &RI) ||
- !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, &RI) ||
+ if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1) ||
+ !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2) ||
Width1 != Width2)
return false;
@@ -5821,9 +5821,10 @@ MachineInstr *PPCInstrInfo::findLoopInstr(
// Return true if get the base operand, byte offset of an instruction and the
// memory width. Width is the size of memory that is being loaded/stored.
-bool PPCInstrInfo::getMemOperandWithOffsetWidth(
- const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
- LocationSize &Width, const TargetRegisterInfo *TRI) const {
+bool PPCInstrInfo::getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
+ const MachineOperand *&BaseReg,
+ int64_t &Offset,
+ LocationSize &Width) const {
if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
return false;
@@ -5859,8 +5860,8 @@ bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
int64_t OffsetA = 0, OffsetB = 0;
LocationSize WidthA = LocationSize::precise(0),
WidthB = LocationSize::precise(0);
- if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, &RI) &&
- getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, &RI)) {
+ if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA) &&
+ getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
int LowOffset = std::min(OffsetA, OffsetB);
int HighOffset = std::max(OffsetA, OffsetB);
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index d86cf04facaff..55fee0b8e1418 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -669,8 +669,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
/// loaded/stored (e.g. 1, 2, 4, 8).
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
- int64_t &Offset, LocationSize &Width,
- const TargetRegisterInfo *TRI) const;
+ int64_t &Offset, LocationSize &Width) const;
bool optimizeCmpPostRA(MachineInstr &MI) const;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 23d566581a1ae..053727f08dff3 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -3596,8 +3596,7 @@ bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
}
const MachineOperand *BaseOp;
OffsetIsScalable = false;
- if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width,
- &getRegisterInfo()))
+ if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
return false;
BaseOps.push_back(BaseOp);
return true;
@@ -3672,7 +3671,7 @@ bool RISCVInstrInfo::shouldClusterMemOps(
// function) and set it as appropriate.
bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
- LocationSize &Width, const TargetRegisterInfo *TRI) const {
+ LocationSize &Width) const {
if (!LdSt.mayLoadOrStore())
return false;
@@ -3708,13 +3707,12 @@ bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
// base registers are identical, and the offset of a lower memory access +
// the width doesn't overlap the offset of a higher memory access,
// then the memory accesses are different.
- const TargetRegisterInfo *TRI = STI.getRegisterInfo();
const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
int64_t OffsetA = 0, OffsetB = 0;
LocationSize WidthA = LocationSize::precise(0),
WidthB = LocationSize::precise(0);
- if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
- getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
+ if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA) &&
+ getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB)) {
if (BaseOpA->isIdenticalTo(*BaseOpB)) {
int LowOffset = std::min(OffsetA, OffsetB);
int HighOffset = std::max(OffsetA, OffsetB);
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 7cfb7f70521b5..091a40a4cd379 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -216,8 +216,7 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt,
const MachineOperand *&BaseOp,
- int64_t &Offset, LocationSize &Width,
- const TargetRegisterInfo *TRI) const;
+ int64_t &Offset, LocationSize &Width) const;
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
const MachineInstr &MIb) const override;
>From 0865fa8f7778b08bac640b3718835293ac8979e1 Mon Sep 17 00:00:00 2001
From: Charles Zablit <c_zablit at apple.com>
Date: Wed, 7 Oct 2026 11:03:11 +0100
Subject: [PATCH 29/52] [lldb][test] Check cryptography.x509 in the strict test
requirements (#229409)
`lldb_find_python_module(cryptography)` only runs `import cryptography`,
which does not load the native `cryptography.hazmat.bindings._rust`. An
install without cffi (`pip install --no-deps cryptography`) passed the
check, then every `from cryptography import x509` in `TestSymStore.py`
failed with `"ModuleNotFoundError: No module named '_cffi_backend'"`.
This patch imports `cryptography.x509` instead so such an install fails
at configure time.
This was uncovered in swiftlang testing.
---
lldb/test/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index 59e3c31793372..288723d0099a4 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -12,7 +12,7 @@ endif()
if(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS)
message(STATUS "Enforcing strict test requirements for LLDB")
# Lit uses psutil to do per-test timeouts.
- set(useful_python_modules psutil packaging cryptography)
+ set(useful_python_modules psutil packaging cryptography.x509)
if(NOT WIN32)
# We no longer vendor pexpect and it is not used on Windows.
>From 804efba8a498197855111bd290728aac4cfdbc21 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 12:05:38 +0200
Subject: [PATCH 30/52] PowerPC: Promote 32-bit ops to 64-bit in place for
EXTSW elimination (#229054)
promoteInstr32To64ForElimEXTSW rebuilt a 32-bit instruction as its
64-bit counterpart (e.g. SRAWI -> SRAWI8) with BuildMI, which adds the opcode's
implicit defs, and then copied over the original operands, which already carry them.
This produced duplicate, inconsistent defs:
%8:g8rc = SRAWI8 killed %7, 8, implicit-def $carry, implicit-def dead $carry
The 32-bit and 64-bit opcodes have the same operand layout, so mutate the
original instruction with setDesc instead. This keeps the original implicit defs
with their dead flags, and also preserves the memoperands and MI flags
that rebuilding dropped. The preserved memoperand on a promoted LHA8 allows
the scheduler to reorder it with spills in aix-cc-abi.ll.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 21 +++++--------------
llvm/test/CodeGen/PowerPC/aix-cc-abi.ll | 12 +++++------
.../convert-rr-to-ri-instrs-out-of-range.mir | 12 +++++------
.../PowerPC/convert-rr-to-ri-instrs.mir | 5 ++---
4 files changed, 19 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 67b8f481dfcad..80b7dcf2efc8b 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -5484,7 +5484,6 @@ void PPCInstrInfo::promoteInstr32To64ForElimEXTSW(const Register &Reg,
// PPC::GRCRegClass or PPC::GPRC_and_GPRC_NOR0RegClass, we need to promote
// the operand to PPC::G8CRegClass or PPC::G8RC_and_G8RC_NOR0RegClass,
// respectively.
- SmallVector<Register> PromoteRegs(MI->getNumOperands());
for (unsigned i = 1; i < MI->getNumOperands(); i++) {
MachineOperand &Operand = MI->getOperand(i);
if (!Operand.isReg())
@@ -5507,30 +5506,20 @@ void PPCInstrInfo::promoteInstr32To64ForElimEXTSW(const Register &Reg,
.addReg(TmpReg)
.addReg(OperandReg)
.addImm(PPC::sub_32);
- PromoteRegs[i] = DstTmpReg;
+ Operand.setReg(DstTmpReg);
+ Operand.setIsKill();
}
}
Register NewDefinedReg = MRI->createVirtualRegister(NewRC);
-
- BuildMI(*MBB, MI, DL, get(NewOpcode), NewDefinedReg);
- MachineBasicBlock::instr_iterator Iter(MI);
- --Iter;
- MachineInstrBuilder MIBuilder(*Iter->getMF(), Iter);
- for (unsigned i = 1; i < MI->getNumOperands(); i++) {
- if (PromoteRegs[i])
- MIBuilder.addReg(PromoteRegs[i], RegState::Kill);
- else
- Iter->addOperand(MI->getOperand(i));
- }
-
- MI->eraseFromParent();
+ MI->setDesc(MCID);
+ MI->getOperand(0).setReg(NewDefinedReg);
// A defined register may be used by other instructions that are 32-bit.
// After the defined register is promoted to 64-bit for the promoted
// instruction, we need to demote the 64-bit defined register back to a
// 32-bit register
- BuildMI(*MBB, ++Iter, DL, get(PPC::COPY), SrcReg)
+ BuildMI(*MBB, std::next(MI->getIterator()), DL, get(PPC::COPY), SrcReg)
.addReg(NewDefinedReg, RegState::Kill, PPC::sub_32);
}
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll b/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
index 6838d61fff8de..6a3ece77cf367 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
@@ -1238,30 +1238,30 @@ define i64 @test_ints_stack(i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6
; ASM64PWR4-LABEL: test_ints_stack:
; ASM64PWR4: # %bb.0: # %entry
; ASM64PWR4-NEXT: add 3, 3, 4
-; ASM64PWR4-NEXT: std 31, -8(1) # 8-byte Folded Spill
+; ASM64PWR4-NEXT: lha 0, 126(1)
; ASM64PWR4-NEXT: add 3, 3, 5
; ASM64PWR4-NEXT: add 3, 3, 6
; ASM64PWR4-NEXT: add 3, 3, 7
-; ASM64PWR4-NEXT: std 2, -16(1) # 8-byte Folded Spill
; ASM64PWR4-NEXT: add 3, 3, 8
-; ASM64PWR4-NEXT: add 3, 3, 9
; ASM64PWR4-NEXT: ld 6, 112(1)
+; ASM64PWR4-NEXT: add 3, 3, 9
; ASM64PWR4-NEXT: add 3, 3, 10
; ASM64PWR4-NEXT: extsw 3, 3
-; ASM64PWR4-NEXT: lha 0, 126(1)
+; ASM64PWR4-NEXT: lbz 5, 135(1)
; ASM64PWR4-NEXT: add 3, 3, 6
; ASM64PWR4-NEXT: add 3, 3, 0
-; ASM64PWR4-NEXT: lbz 5, 135(1)
; ASM64PWR4-NEXT: lwz 7, 140(1)
; ASM64PWR4-NEXT: add 3, 3, 5
; ASM64PWR4-NEXT: lwa 12, 148(1)
; ASM64PWR4-NEXT: add 3, 3, 7
; ASM64PWR4-NEXT: add 3, 3, 12
+; ASM64PWR4-NEXT: std 31, -8(1) # 8-byte Folded Spill
; ASM64PWR4-NEXT: ld 31, 152(1)
; ASM64PWR4-NEXT: lbz 5, 167(1)
; ASM64PWR4-NEXT: add 3, 3, 31
-; ASM64PWR4-NEXT: lwa 11, 172(1)
+; ASM64PWR4-NEXT: std 2, -16(1) # 8-byte Folded Spill
; ASM64PWR4-NEXT: add 3, 3, 5
+; ASM64PWR4-NEXT: lwa 11, 172(1)
; ASM64PWR4-NEXT: add 3, 3, 11
; ASM64PWR4-NEXT: lbz 2, 183(1)
; ASM64PWR4-NEXT: lbz 6, 191(1)
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
index b9ca97a8bd68f..01e87731fc3aa 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
@@ -712,20 +712,20 @@ frameInfo:
hasMustTailInVarArgFunc: false
savePoint: []
restorePoint: []
-fixedStack:
-stack:
-constants:
+fixedStack:
+stack:
+constants:
body: |
bb.0.entry:
liveins: $x3, $x4
-
+
%1 = COPY $x4
%0 = COPY $x3
%2 = LI 48
%3 = COPY %0.sub_32
%4 = SRAW killed %3, killed %2, implicit-def dead $carry
; CHECK: LI 48
- ; CHECK: SRAW8 killed %7, killed %9, implicit-def $carry, implicit-def dead $carry
+ ; CHECK: SRAW8 killed %7, killed %9, implicit-def dead $carry
; CHECK-LATE: sraw 3, 3, 4
%5 = EXTSW_32_64 killed %4
$x3 = COPY %5
@@ -782,7 +782,7 @@ body: |
%2 = LI 80
%3 = COPY %0.sub_32
%4 = SRAW_rec killed %3, %2, implicit-def dead $carry, implicit-def $cr0
- ; CHECK: SRAW8_rec killed %10, killed %12, implicit-def $carry, implicit-def $cr0, implicit-def dead $carry, implicit-def $cr0
+ ; CHECK: SRAW8_rec killed %10, killed %12, implicit-def dead $carry, implicit-def $cr0
; CHECK-LATE: sraw. 3, 3, 4
%5 = COPY killed $cr0
%6 = ISEL %2, %4, %5.sub_eq
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
index 3462bbf3a7e9e..bee9637b4f570 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
@@ -5038,8 +5038,7 @@ body: |
%4 = SRAW killed %3, killed %2, implicit-def dead $carry
; CHECK: %6:g8rc = IMPLICIT_DEF
; CHECK-NEXT: %7:g8rc = INSERT_SUBREG %6, %3, %subreg.sub_32
-
- ; CHECK-NEXT: %8:g8rc = SRAWI8 killed %7, 15, implicit-def $carry, implicit-def dead $carry
+ ; CHECK-NEXT: %8:g8rc = SRAWI8 killed %7, 15, implicit-def dead $carry
; CHECK-NEXT: %4:gprc = COPY killed %8.sub_32
; CHECK-LATE: srawi 3, 3, 15
%5 = EXTSW_32_64 killed %4
@@ -5097,7 +5096,7 @@ body: |
%2 = LI 8
%3 = COPY %0.sub_32
%4 = SRAW_rec killed %3, %2, implicit-def dead $carry, implicit-def $cr0
- ; CHECK: %11:g8rc = SRAWI8_rec killed %10, 8, implicit-def $carry, implicit-def $cr0, implicit-def dead $carry, implicit-def $cr0
+ ; CHECK: %11:g8rc = SRAWI8_rec killed %10, 8, implicit-def dead $carry, implicit-def $cr0
; CHECK-LATE: srawi. 3, 3, 8
%5 = COPY killed $cr0
%6 = ISEL %2, %4, %5.sub_eq
>From 1ebfb7a71d0f11cbe3c240d875ef2c1690be3491 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 7 Oct 2026 11:11:57 +0100
Subject: [PATCH 31/52] [RISCV] Add Tenstorrent Ascalon XG (#229030)
Ascalon XG is the global variant of Ascalon X, to meet export
restrictions: it lacks the Zvkng extension and has reduced vector FP64
throughput. Add it as tt-ascalon-xg to RISCVProcessors. Restructure the
Ascalon scheduling model into a multiclass similar to the SiFive7 model,
so that both cores share the same definitions. On Ascalon XG, vector
instructions producing FP64 results (SEW=64 add/mul/fma, and the SEW=32
widening add/mul/fma and conversions) additionally occupy an FP64 unit
for 4x the cycles and have increased latency.
tt-ascalon-x is unchanged.
Ref:
https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=2bfe853b3b1703216a32a48d2bc61363140c6aea
Assisted-by: llm
---
.../riscv-tt-ascalon-xg.c | 99 +
.../test/Misc/target-invalid-cpu-note/riscv.c | 2 +
llvm/docs/ReleaseNotes.md | 3 +
llvm/lib/Target/RISCV/RISCVProcessors.td | 69 +-
llvm/lib/Target/RISCV/RISCVSchedTTAscalonX.td | 1755 +++++++++--------
.../tools/llvm-mca/RISCV/tt-ascalon-x/vfp64.s | 175 ++
6 files changed, 1230 insertions(+), 873 deletions(-)
create mode 100644 clang/test/Driver/print-enabled-extensions/riscv-tt-ascalon-xg.c
create mode 100644 llvm/test/tools/llvm-mca/RISCV/tt-ascalon-x/vfp64.s
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-tt-ascalon-xg.c b/clang/test/Driver/print-enabled-extensions/riscv-tt-ascalon-xg.c
new file mode 100644
index 0000000000000..b046d71faabc4
--- /dev/null
+++ b/clang/test/Driver/print-enabled-extensions/riscv-tt-ascalon-xg.c
@@ -0,0 +1,99 @@
+// REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=tt-ascalon-xg --print-enabled-extensions | FileCheck %s
+
+// CHECK: Extensions enabled for the given RISC-V target
+// CHECK-EMPTY:
+// CHECK-NEXT: Name Version Description
+// CHECK-NEXT: i 2.1 'I' (Base Integer Instruction Set)
+// CHECK-NEXT: m 2.0 'M' (Integer Multiplication and Division)
+// CHECK-NEXT: a 2.1 'A' (Atomic Instructions)
+// CHECK-NEXT: f 2.2 'F' (Single-Precision Floating-Point)
+// CHECK-NEXT: d 2.2 'D' (Double-Precision Floating-Point)
+// CHECK-NEXT: c 2.0 'C' (Compressed Instructions)
+// CHECK-NEXT: b 1.0 'B' (the collection of the Zba, Zbb, Zbs extensions)
+// CHECK-NEXT: v 1.0 'V' (Vector Extension for Application Processors)
+// CHECK-NEXT: h 1.0 'H' (Hypervisor)
+// CHECK-NEXT: zic64b 1.0 'Zic64b' (Cache Block Size Is 64 Bytes)
+// CHECK-NEXT: zicbom 1.0 'Zicbom' (Cache-Block Management Instructions)
+// CHECK-NEXT: zicbop 1.0 'Zicbop' (Cache-Block Prefetch Instructions)
+// CHECK-NEXT: zicboz 1.0 'Zicboz' (Cache-Block Zero Instructions)
+// CHECK-NEXT: ziccamoa 1.0 'Ziccamoa' (Main Memory Supports All Atomics in A)
+// CHECK-NEXT: ziccif 1.0 'Ziccif' (Main Memory Supports Instruction Fetch with Atomicity Requirement)
+// CHECK-NEXT: zicclsm 1.0 'Zicclsm' (Main Memory Supports Misaligned Loads/Stores)
+// CHECK-NEXT: ziccrse 1.0 'Ziccrse' (Main Memory Supports Forward Progress on LR/SC Sequences)
+// CHECK-NEXT: zicntr 2.0 'Zicntr' (Base Counters and Timers)
+// CHECK-NEXT: zicond 1.0 'Zicond' (Integer Conditional Operations)
+// CHECK-NEXT: zicsr 2.0 'Zicsr' (CSRs)
+// CHECK-NEXT: zifencei 2.0 'Zifencei' (fence.i)
+// CHECK-NEXT: zihintntl 1.0 'Zihintntl' (Non-Temporal Locality Hints)
+// CHECK-NEXT: zihintpause 2.0 'Zihintpause' (Pause Hint)
+// CHECK-NEXT: zihpm 2.0 'Zihpm' (Hardware Performance Counters)
+// CHECK-NEXT: zimop 1.0 'Zimop' (May-Be-Operations)
+// CHECK-NEXT: zmmul 1.0 'Zmmul' (Integer Multiplication)
+// CHECK-NEXT: za64rs 1.0 'Za64rs' (Reservation Set Size of at Most 64 Bytes)
+// CHECK-NEXT: zaamo 1.0 'Zaamo' (Atomic Memory Operations)
+// CHECK-NEXT: zalrsc 1.0 'Zalrsc' (Load-Reserved/Store-Conditional)
+// CHECK-NEXT: zawrs 1.0 'Zawrs' (Wait on Reservation Set)
+// CHECK-NEXT: zfa 1.0 'Zfa' (Additional Floating-Point)
+// CHECK-NEXT: zfbfmin 1.0 'Zfbfmin' (Scalar BF16 Converts)
+// CHECK-NEXT: zfh 1.0 'Zfh' (Half-Precision Floating-Point)
+// CHECK-NEXT: zfhmin 1.0 'Zfhmin' (Half-Precision Floating-Point Minimal)
+// CHECK-NEXT: zca 1.0 'Zca' (part of the C extension, excluding compressed floating point loads/stores)
+// CHECK-NEXT: zcb 1.0 'Zcb' (Compressed basic bit manipulation instructions)
+// CHECK-NEXT: zcd 1.0 'Zcd' (Compressed Double-Precision Floating-Point Instructions)
+// CHECK-NEXT: zcmop 1.0 'Zcmop' (Compressed May-Be-Operations)
+// CHECK-NEXT: zba 1.0 'Zba' (Address Generation Instructions)
+// CHECK-NEXT: zbb 1.0 'Zbb' (Basic Bit-Manipulation)
+// CHECK-NEXT: zbs 1.0 'Zbs' (Single-Bit Instructions)
+// CHECK-NEXT: zkr 1.0 'Zkr' (Entropy Source Extension)
+// CHECK-NEXT: zkt 1.0 'Zkt' (Data Independent Execution Latency)
+// CHECK-NEXT: zvbb 1.0 'Zvbb' (Vector basic bit-manipulation instructions)
+// CHECK-NEXT: zvbc 1.0 'Zvbc' (Vector Carryless Multiplication)
+// CHECK-NEXT: zve32f 1.0 'Zve32f' (Vector Extensions for Embedded Processors with maximal 32 EEW and F extension)
+// CHECK-NEXT: zve32x 1.0 'Zve32x' (Vector Extensions for Embedded Processors with maximal 32 EEW)
+// CHECK-NEXT: zve64d 1.0 'Zve64d' (Vector Extensions for Embedded Processors with maximal 64 EEW, F and D extension)
+// CHECK-NEXT: zve64f 1.0 'Zve64f' (Vector Extensions for Embedded Processors with maximal 64 EEW and F extension)
+// CHECK-NEXT: zve64x 1.0 'Zve64x' (Vector Extensions for Embedded Processors with maximal 64 EEW)
+// CHECK-NEXT: zvfbfmin 1.0 'Zvfbfmin' (Vector BF16 Converts)
+// CHECK-NEXT: zvfbfwma 1.0 'Zvfbfwma' (Vector BF16 widening mul-add)
+// CHECK-NEXT: zvfh 1.0 'Zvfh' (Vector Half-Precision Floating-Point)
+// CHECK-NEXT: zvfhmin 1.0 'Zvfhmin' (Vector Half-Precision Floating-Point Minimal)
+// CHECK-NEXT: zvkb 1.0 'Zvkb' (Vector Bit-manipulation used in Cryptography)
+// CHECK-NEXT: zvkt 1.0 'Zvkt' (Vector Data-Independent Execution Latency)
+// CHECK-NEXT: zvl128b 1.0 'Zvl128b' (Minimum Vector Length 128)
+// CHECK-NEXT: zvl256b 1.0 'Zvl256b' (Minimum Vector Length 256)
+// CHECK-NEXT: zvl32b 1.0 'Zvl32b' (Minimum Vector Length 32)
+// CHECK-NEXT: zvl64b 1.0 'Zvl64b' (Minimum Vector Length 64)
+// CHECK-NEXT: sha 1.0 'Sha' (Augmented Hypervisor)
+// CHECK-NEXT: shcounterenw 1.0 'Shcounterenw' (Support writeable hcounteren enable bit for any hpmcounter that is not read-only zero)
+// CHECK-NEXT: shgatpa 1.0 'Shgatpa' (SvNNx4 mode supported for all modes supported by satp, as well as Bare)
+// CHECK-NEXT: shtvala 1.0 'Shtvala' (htval provides all needed values)
+// CHECK-NEXT: shvsatpa 1.0 'Shvsatpa' (vsatp supports all modes supported by satp)
+// CHECK-NEXT: shvstvala 1.0 'Shvstvala' (vstval provides all needed values)
+// CHECK-NEXT: shvstvecd 1.0 'Shvstvecd' (vstvec supports Direct mode)
+// CHECK-NEXT: smaia 1.0 'Smaia' (Advanced Interrupt Architecture Machine Level)
+// CHECK-NEXT: smmpm 1.0 'Smmpm' (Machine-level Pointer Masking for M-mode)
+// CHECK-NEXT: smnpm 1.0 'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)
+// CHECK-NEXT: smrnmi 1.0 'Smrnmi' (Resumable Non-Maskable Interrupts)
+// CHECK-NEXT: smstateen 1.0 'Smstateen' (Machine-mode view of the state-enable extension)
+// CHECK-NEXT: ssaia 1.0 'Ssaia' (Advanced Interrupt Architecture Supervisor Level)
+// CHECK-NEXT: ssccptr 1.0 'Ssccptr' (Main memory supports page table reads)
+// CHECK-NEXT: sscofpmf 1.0 'Sscofpmf' (Count Overflow and Mode-Based Filtering)
+// CHECK-NEXT: sscounterenw 1.0 'Sscounterenw' (Support writeable scounteren enable bit for any hpmcounter that is not read-only zero)
+// CHECK-NEXT: ssnpm 1.0 'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)
+// CHECK-NEXT: ssstateen 1.0 'Ssstateen' (Supervisor-mode view of the state-enable extension)
+// CHECK-NEXT: ssstrict 1.0 'Ssstrict' (No non-conforming extensions are present)
+// CHECK-NEXT: sstc 1.0 'Sstc' (Supervisor-mode timer interrupts)
+// CHECK-NEXT: sstvala 1.0 'Sstvala' (stval provides all needed values)
+// CHECK-NEXT: sstvecd 1.0 'Sstvecd' (stvec supports Direct mode)
+// CHECK-NEXT: ssu64xl 1.0 'Ssu64xl' (UXLEN=64 supported)
+// CHECK-NEXT: supm 1.0 'Supm' (Indicates User-mode Pointer Masking)
+// CHECK-NEXT: svade 1.0 'Svade' (Raise exceptions on improper A/D bits)
+// CHECK-NEXT: svbare 1.0 'Svbare' (satp mode Bare supported)
+// CHECK-NEXT: svinval 1.0 'Svinval' (Fine-Grained Address-Translation Cache Invalidation)
+// CHECK-NEXT: svnapot 1.0 'Svnapot' (NAPOT Translation Contiguity)
+// CHECK-NEXT: svpbmt 1.0 'Svpbmt' (Page-Based Memory Types)
+// CHECK-EMPTY:
+// CHECK-NEXT: Experimental extensions
+// CHECK-EMPTY:
+// CHECK-NEXT: ISA String: rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_b1p0_v1p0_h1p0_zic64b1p0_zicbom1p0_zicbop1p0_zicboz1p0_ziccamoa1p0_ziccif1p0_zicclsm1p0_ziccrse1p0_zicntr2p0_zicond1p0_zicsr2p0_zifencei2p0_zihintntl1p0_zihintpause2p0_zihpm2p0_zimop1p0_zmmul1p0_za64rs1p0_zaamo1p0_zalrsc1p0_zawrs1p0_zfa1p0_zfbfmin1p0_zfh1p0_zfhmin1p0_zca1p0_zcb1p0_zcd1p0_zcmop1p0_zba1p0_zbb1p0_zbs1p0_zkr1p0_zkt1p0_zvbb1p0_zvbc1p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvfbfmin1p0_zvfbfwma1p0_zvfh1p0_zvfhmin1p0_zvkb1p0_zvkt1p0_zvl128b1p0_zvl256b1p0_zvl32b1p0_zvl64b1p0_sha1p0_shcounterenw1p0_shgatpa1p0_shtvala1p0_shvsatpa1p0_shvstvala1p0_shvstvecd1p0_smaia1p0_smmpm1p0_smnpm1p0_smrnmi1p0_smstateen1p0_ssaia1p0_ssccptr1p0_sscofpmf1p0_sscounterenw1p0_ssnpm1p0_ssstateen1p0_ssstrict1p0_sstc1p0_sstvala1p0_sstvecd1p0_ssu64xl1p0_supm1p0_svade1p0_svbare1p0_svinval1p0_svnapot1p0_svpbmt1p0
diff --git a/clang/test/Misc/target-invalid-cpu-note/riscv.c b/clang/test/Misc/target-invalid-cpu-note/riscv.c
index 86fdf400ac461..df13001fedf0d 100644
--- a/clang/test/Misc/target-invalid-cpu-note/riscv.c
+++ b/clang/test/Misc/target-invalid-cpu-note/riscv.c
@@ -60,6 +60,7 @@
// RISCV64-SAME: {{^}}, syntacore-scr5-rv64
// RISCV64-SAME: {{^}}, syntacore-scr7
// RISCV64-SAME: {{^}}, tt-ascalon-x
+// RISCV64-SAME: {{^}}, tt-ascalon-xg
// RISCV64-SAME: {{^}}, xiangshan-kunminghu
// RISCV64-SAME: {{^}}, xiangshan-nanhu
// RISCV64-SAME: {{^}}, xt-c910v2
@@ -129,6 +130,7 @@
// TUNE-RISCV64-SAME: {{^}}, syntacore-scr5-rv64
// TUNE-RISCV64-SAME: {{^}}, syntacore-scr7
// TUNE-RISCV64-SAME: {{^}}, tt-ascalon-x
+// TUNE-RISCV64-SAME: {{^}}, tt-ascalon-xg
// TUNE-RISCV64-SAME: {{^}}, xiangshan-kunminghu
// TUNE-RISCV64-SAME: {{^}}, xiangshan-nanhu
// TUNE-RISCV64-SAME: {{^}}, xt-c910v2
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 4e2882de82a02..b369e95263deb 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -298,6 +298,9 @@ Makes programs 10x faster by doing Special New Thing.
latest specification, placing ``p`` after ``v`` and removing unused ``n``.
* Adds experimental assembler support for the `Xqccmi` (Qualcomm 16-bit Instruction Lookup Table) vendor extension.
* Added `-mcpu=gaisler-gr765` for the 64-bit GR765 processor.
+* Added `-mcpu=tt-ascalon-xg` for the Tenstorrent Ascalon XG processor, the
+ global variant of Ascalon X without `Zvkng` and with reduced vector FP64
+ throughput.
### Changes to the WebAssembly Backend
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index eb4e2214dc389..04a2fabc2c4fe 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -732,38 +732,47 @@ def SYNTACORE_SCR7 : RISCVProcessorModel<"syntacore-scr7",
FeatureStdExtZkn],
[TuneNoDefaultUnroll, TunePostRAScheduler]>;
+defvar TTAscalonXGFeatures = !listconcat(RVA23S64Features,
+ [FeatureStdExtSmaia,
+ FeatureStdExtSmmpm,
+ FeatureStdExtSmnpm,
+ FeatureStdExtSmrnmi,
+ FeatureStdExtSmstateen,
+ FeatureStdExtSsaia,
+ FeatureStdExtSsstrict,
+ FeatureStdExtZfbfmin,
+ FeatureStdExtZfh,
+ FeatureStdExtZkr,
+ FeatureStdExtZvbc,
+ FeatureStdExtZvfbfmin,
+ FeatureStdExtZvfbfwma,
+ FeatureStdExtZvfh,
+ FeatureStdExtZvl256b,
+ FeatureUnalignedScalarMem,
+ FeatureUnalignedVectorMem]);
+
+defvar TTAscalonTuneFeatures = [TuneNoDefaultUnroll,
+ TuneNLogNVRGather,
+ TuneOptimizedNF2SegmentLoadStore,
+ TuneOptimizedNF3SegmentLoadStore,
+ TuneOptimizedNF4SegmentLoadStore,
+ TuneOptimizedNF5SegmentLoadStore,
+ TuneOptimizedNF6SegmentLoadStore,
+ TuneOptimizedNF7SegmentLoadStore,
+ TuneOptimizedNF8SegmentLoadStore,
+ TuneOptimizedZeroStrideLoad,
+ TunePostRAScheduler];
+
def TENSTORRENT_ASCALON_X : RISCVProcessorModel<"tt-ascalon-x",
TTAscalonXModel,
- !listconcat(RVA23S64Features,
- [FeatureStdExtSmaia,
- FeatureStdExtSmmpm,
- FeatureStdExtSmnpm,
- FeatureStdExtSmrnmi,
- FeatureStdExtSmstateen,
- FeatureStdExtSsaia,
- FeatureStdExtSsstrict,
- FeatureStdExtZfbfmin,
- FeatureStdExtZfh,
- FeatureStdExtZkr,
- FeatureStdExtZvbc,
- FeatureStdExtZvfbfmin,
- FeatureStdExtZvfbfwma,
- FeatureStdExtZvfh,
- FeatureStdExtZvkng,
- FeatureStdExtZvl256b,
- FeatureUnalignedScalarMem,
- FeatureUnalignedVectorMem]),
- [TuneNoDefaultUnroll,
- TuneNLogNVRGather,
- TuneOptimizedNF2SegmentLoadStore,
- TuneOptimizedNF3SegmentLoadStore,
- TuneOptimizedNF4SegmentLoadStore,
- TuneOptimizedNF5SegmentLoadStore,
- TuneOptimizedNF6SegmentLoadStore,
- TuneOptimizedNF7SegmentLoadStore,
- TuneOptimizedNF8SegmentLoadStore,
- TuneOptimizedZeroStrideLoad,
- TunePostRAScheduler]>;
+ !listconcat(TTAscalonXGFeatures,
+ [FeatureStdExtZvkng]),
+ TTAscalonTuneFeatures>;
+
+def TENSTORRENT_ASCALON_XG : RISCVProcessorModel<"tt-ascalon-xg",
+ TTAscalonXGModel,
+ TTAscalonXGFeatures,
+ TTAscalonTuneFeatures>;
def XIANGSHAN_NANHU : RISCVProcessorModel<"xiangshan-nanhu",
XiangShanNanHuModel,
diff --git a/llvm/lib/Target/RISCV/RISCVSchedTTAscalonX.td b/llvm/lib/Target/RISCV/RISCVSchedTTAscalonX.td
index d6a1f341e3e33..6a1c5aceae75f 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedTTAscalonX.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedTTAscalonX.td
@@ -1,4 +1,4 @@
-//=- RISCVSchedTTAscalonX.td - TT Ascalon X Sched Defs -------*- tablegen -*-=//
+//=- RISCVSchedTTAscalonX.td - TT Ascalon X/XG Sched Defs ----*- tablegen -*-=//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -95,7 +95,7 @@ class AscalonGetCyclesStridedSegmented<string mx, int sew> {
//===----------------------------------------------------------------------===//
-def TTAscalonXModel : SchedMachineModel {
+class TTAscalonSchedMachineModel : SchedMachineModel {
let IssueWidth = 8; // 8-way decode and dispatch
let MicroOpBufferSize = 256; // 256 micro-op re-order buffer
let LoadLatency = 4; // Optimistic load latency
@@ -108,935 +108,1004 @@ def TTAscalonXModel : SchedMachineModel {
HasStdExtZcmt, HasStdExtZknd, HasStdExtZkne,
HasStdExtZknh, HasStdExtZksed, HasStdExtZksh,
HasStdExtZkr];
-}
-
-let SchedModel = TTAscalonXModel in {
-//===----------------------------------------------------------------------===//
-// Define each kind of processor resource and number available.
-
-let BufferSize = 16 in {
- def AscalonLS : ProcResource<3>;
- def AscalonFXA : ProcResource<1>; // ALU, FP/VEC -> INT, MUL, DIV, CSR
- def AscalonFXB : ProcResource<1>; // ALU, INT -> FP/VEC
- def AscalonFXC : ProcResource<2>; // ALU, BR
- def AscalonFXD : ProcResource<2>; // ALU
- def AscalonFX : ProcResGroup<[AscalonFXA, AscalonFXB, AscalonFXC, AscalonFXD]>;
- // FP
- def AscalonFPA : ProcResource<1>; // Pipe A also handles FP/VEC -> INT
- def AscalonFPB : ProcResource<1>;
- def AscalonFP : ProcResGroup<[AscalonFPA, AscalonFPB]>;
- // Vector
- def AscalonVA : ProcResource<1>;
- def AscalonVB : ProcResource<1>;
- def AscalonV : ProcResGroup<[AscalonVA, AscalonVB]>;
+ string Name;
+ bit HasReducedFP64VectorThroughput;
}
+def TTAscalonXModel : TTAscalonSchedMachineModel {
+ // TODO: This should technically be AscalonX.
+ let Name = "Ascalon";
+ let HasReducedFP64VectorThroughput = false;
+}
-//===----------------------------------------------------------------------===//
+def TTAscalonXGModel : TTAscalonSchedMachineModel {
+ let Name = "AscalonXG";
+ let HasReducedFP64VectorThroughput = true;
+}
-// Branching
-def : WriteRes<WriteJmp, [AscalonFXC]>;
-def : WriteRes<WriteJal, [AscalonFXC]>;
-def : WriteRes<WriteJalr, [AscalonFXC]>;
-
-// Integer arithmetic and logic
-def : WriteRes<WriteIALU32, [AscalonFX]>;
-def : WriteRes<WriteIALU, [AscalonFX]>;
-def : WriteRes<WriteShiftImm32, [AscalonFX]>;
-def : WriteRes<WriteShiftImm, [AscalonFX]>;
-def : WriteRes<WriteShiftReg32, [AscalonFX]>;
-def : WriteRes<WriteShiftReg, [AscalonFX]>;
-
-// Integer multiplication
-let Latency = 3 in {
-def : WriteRes<WriteIMul, [AscalonFXA]>;
-def : WriteRes<WriteIMul32, [AscalonFXA]>;
+/// Latency and occupancy of the extra FP64 vector unit on Ascalon XG for
+/// vector instructions producing FP64 results.
+class AscalonXGFP64Vec<int Cycles> {
+ int Latency = !add(Cycles, 8);
+ int ReleaseAtCycles = !mul(Cycles, 4);
}
-// Integer division
-// Worst case latency is used.
+multiclass TTAscalonSchedResources<bit ReducedFP64Vec> {
+ //===--------------------------------------------------------------------===//
+ // Define each kind of processor resource and number available.
+ let BufferSize = 16 in {
+ def LS : ProcResource<3>;
+ def FXA : ProcResource<1>; // ALU, FP/VEC -> INT, MUL, DIV, CSR
+ def FXB : ProcResource<1>; // ALU, INT -> FP/VEC
+ def FXC : ProcResource<2>; // ALU, BR
+ def FXD : ProcResource<2>; // ALU
+ def FX : ProcResGroup<[!cast<ProcResource>(NAME # "FXA"),
+ !cast<ProcResource>(NAME # "FXB"),
+ !cast<ProcResource>(NAME # "FXC"),
+ !cast<ProcResource>(NAME # "FXD")]>;
+ // FP
+ def FPA : ProcResource<1>; // Pipe A also handles FP/VEC -> INT
+ def FPB : ProcResource<1>;
+ def FP : ProcResGroup<[!cast<ProcResource>(NAME # "FPA"),
+ !cast<ProcResource>(NAME # "FPB")]>;
+ // Vector
+ def VA : ProcResource<1>;
+ def VB : ProcResource<1>;
+ def V : ProcResGroup<[!cast<ProcResource>(NAME # "VA"),
+ !cast<ProcResource>(NAME # "VB")]>;
+ // Extra unit that models the reduced throughput of vector instructions
+ // producing FP64 results.
+ if ReducedFP64Vec then
+ def FP64V : ProcResource<2>;
+ }
-let Latency = 7, ReleaseAtCycles = [7] in {
- def : WriteRes<WriteIDiv32, [AscalonFXA]>;
- def : WriteRes<WriteIDiv, [AscalonFXA]>;
- def : WriteRes<WriteIRem32, [AscalonFXA]>;
- def : WriteRes<WriteIRem, [AscalonFXA]>;
-}
+ defvar AscalonLS = !cast<ProcResource>(NAME # "LS");
+ defvar AscalonFXA = !cast<ProcResource>(NAME # "FXA");
+ defvar AscalonFXB = !cast<ProcResource>(NAME # "FXB");
+ defvar AscalonFXC = !cast<ProcResource>(NAME # "FXC");
+ defvar AscalonFXD = !cast<ProcResource>(NAME # "FXD");
+ defvar AscalonFX = !cast<ProcResGroup>(NAME # "FX");
+ defvar AscalonFPA = !cast<ProcResource>(NAME # "FPA");
+ defvar AscalonFPB = !cast<ProcResource>(NAME # "FPB");
+ defvar AscalonFP = !cast<ProcResGroup>(NAME # "FP");
+ defvar AscalonVA = !cast<ProcResource>(NAME # "VA");
+ defvar AscalonVB = !cast<ProcResource>(NAME # "VB");
+ defvar AscalonV = !cast<ProcResGroup>(NAME # "V");
+
+ // Only used when ReducedFP64Vec is set; fall back to the vector pipes
+ // otherwise so that the defvar is always well-formed.
+ defvar AscalonFP64V = !if(ReducedFP64Vec,
+ !cast<ProcResource>(NAME # "FP64V"), AscalonV);
+
+ //===--------------------------------------------------------------------===//
+ // Branching
+ def : WriteRes<WriteJmp, [AscalonFXC]>;
+ def : WriteRes<WriteJal, [AscalonFXC]>;
+ def : WriteRes<WriteJalr, [AscalonFXC]>;
+
+ // Integer arithmetic and logic
+ def : WriteRes<WriteIALU32, [AscalonFX]>;
+ def : WriteRes<WriteIALU, [AscalonFX]>;
+ def : WriteRes<WriteShiftImm32, [AscalonFX]>;
+ def : WriteRes<WriteShiftImm, [AscalonFX]>;
+ def : WriteRes<WriteShiftReg32, [AscalonFX]>;
+ def : WriteRes<WriteShiftReg, [AscalonFX]>;
+
+ // Integer multiplication
+ let Latency = 3 in {
+ def : WriteRes<WriteIMul, [AscalonFXA]>;
+ def : WriteRes<WriteIMul32, [AscalonFXA]>;
+ }
-// Bitmanip
-def : WriteRes<WriteRotateImm, [AscalonFX]>;
-def : WriteRes<WriteRotateImm32, [AscalonFX]>;
-def : WriteRes<WriteRotateReg, [AscalonFX]>;
-def : WriteRes<WriteRotateReg32, [AscalonFX]>;
-
-def : WriteRes<WriteCLZ, [AscalonFX]>;
-def : WriteRes<WriteCLZ32, [AscalonFX]>;
-def : WriteRes<WriteCTZ, [AscalonFX]>;
-def : WriteRes<WriteCTZ32, [AscalonFX]>;
-
-def : WriteRes<WriteCPOP, [AscalonFX]>;
-def : WriteRes<WriteCPOP32, [AscalonFX]>;
-
-def : WriteRes<WriteORCB, [AscalonFX]>;
-
-def : WriteRes<WriteIMinMax, [AscalonFX]>;
-
-def : WriteRes<WriteREV8, [AscalonFX]>;
-
-def : WriteRes<WriteSHXADD, [AscalonFX]>;
-def : WriteRes<WriteSHXADD32, [AscalonFX]>;
-
-// Single-bit instructions
-def : WriteRes<WriteSingleBit, [AscalonFX]>;
-def : WriteRes<WriteSingleBitImm, [AscalonFX]>;
-def : WriteRes<WriteBEXT, [AscalonFX]>;
-def : WriteRes<WriteBEXTI, [AscalonFX]>;
-
-// Memory
-def : WriteRes<WriteSTB, [AscalonLS]>;
-def : WriteRes<WriteSTH, [AscalonLS]>;
-def : WriteRes<WriteSTW, [AscalonLS]>;
-def : WriteRes<WriteSTD, [AscalonLS]>;
-def : WriteRes<WriteFST16, [AscalonLS]>;
-def : WriteRes<WriteFST32, [AscalonLS]>;
-def : WriteRes<WriteFST64, [AscalonLS]>;
-
-let Latency = 4 in {
-def : WriteRes<WriteLDB, [AscalonLS]>;
-def : WriteRes<WriteLDH, [AscalonLS]>;
-def : WriteRes<WriteLDW, [AscalonLS]>;
-def : WriteRes<WriteLDD, [AscalonLS]>;
-def : WriteRes<WriteFLD16, [AscalonLS]>;
-def : WriteRes<WriteFLD32, [AscalonLS]>;
-def : WriteRes<WriteFLD64, [AscalonLS]>;
-}
+ // Integer division
+ // Worst case latency is used.
-// Atomic memory
-def : WriteRes<WriteAtomicSTW, [AscalonLS]>;
-def : WriteRes<WriteAtomicSTD, [AscalonLS]>;
+ let Latency = 7, ReleaseAtCycles = [7] in {
+ def : WriteRes<WriteIDiv32, [AscalonFXA]>;
+ def : WriteRes<WriteIDiv, [AscalonFXA]>;
+ def : WriteRes<WriteIRem32, [AscalonFXA]>;
+ def : WriteRes<WriteIRem, [AscalonFXA]>;
+ }
-let Latency = 4 in {
-def : WriteRes<WriteAtomicW, [AscalonLS]>;
-def : WriteRes<WriteAtomicD, [AscalonLS]>;
-def : WriteRes<WriteAtomicLDW, [AscalonLS]>;
-def : WriteRes<WriteAtomicLDD, [AscalonLS]>;
-}
+ // Bitmanip
+ def : WriteRes<WriteRotateImm, [AscalonFX]>;
+ def : WriteRes<WriteRotateImm32, [AscalonFX]>;
+ def : WriteRes<WriteRotateReg, [AscalonFX]>;
+ def : WriteRes<WriteRotateReg32, [AscalonFX]>;
+
+ def : WriteRes<WriteCLZ, [AscalonFX]>;
+ def : WriteRes<WriteCLZ32, [AscalonFX]>;
+ def : WriteRes<WriteCTZ, [AscalonFX]>;
+ def : WriteRes<WriteCTZ32, [AscalonFX]>;
+
+ def : WriteRes<WriteCPOP, [AscalonFX]>;
+ def : WriteRes<WriteCPOP32, [AscalonFX]>;
+
+ def : WriteRes<WriteORCB, [AscalonFX]>;
+
+ def : WriteRes<WriteIMinMax, [AscalonFX]>;
+
+ def : WriteRes<WriteREV8, [AscalonFX]>;
+
+ def : WriteRes<WriteSHXADD, [AscalonFX]>;
+ def : WriteRes<WriteSHXADD32, [AscalonFX]>;
+
+ // Single-bit instructions
+ def : WriteRes<WriteSingleBit, [AscalonFX]>;
+ def : WriteRes<WriteSingleBitImm, [AscalonFX]>;
+ def : WriteRes<WriteBEXT, [AscalonFX]>;
+ def : WriteRes<WriteBEXTI, [AscalonFX]>;
+
+ // Memory
+ def : WriteRes<WriteSTB, [AscalonLS]>;
+ def : WriteRes<WriteSTH, [AscalonLS]>;
+ def : WriteRes<WriteSTW, [AscalonLS]>;
+ def : WriteRes<WriteSTD, [AscalonLS]>;
+ def : WriteRes<WriteFST16, [AscalonLS]>;
+ def : WriteRes<WriteFST32, [AscalonLS]>;
+ def : WriteRes<WriteFST64, [AscalonLS]>;
+
+ let Latency = 4 in {
+ def : WriteRes<WriteLDB, [AscalonLS]>;
+ def : WriteRes<WriteLDH, [AscalonLS]>;
+ def : WriteRes<WriteLDW, [AscalonLS]>;
+ def : WriteRes<WriteLDD, [AscalonLS]>;
+ def : WriteRes<WriteFLD16, [AscalonLS]>;
+ def : WriteRes<WriteFLD32, [AscalonLS]>;
+ def : WriteRes<WriteFLD64, [AscalonLS]>;
+ }
-// Half precision.
-let Latency = 3 in {
-def : WriteRes<WriteFAdd16, [AscalonFP]>;
-def : WriteRes<WriteFMul16, [AscalonFP]>;
-def : WriteRes<WriteFMA16, [AscalonFP]>;
-def : WriteRes<WriteFSGNJ16, [AscalonFP]>;
-def : WriteRes<WriteFMinMax16, [AscalonFP]>;
-}
+ // Atomic memory
+ def : WriteRes<WriteAtomicSTW, [AscalonLS]>;
+ def : WriteRes<WriteAtomicSTD, [AscalonLS]>;
-let Latency = 7, ReleaseAtCycles = [7] in {
-def : WriteRes<WriteFDiv16, [AscalonFP]>;
-def : WriteRes<WriteFSqrt16, [AscalonFP]>;
-}
+ let Latency = 4 in {
+ def : WriteRes<WriteAtomicW, [AscalonLS]>;
+ def : WriteRes<WriteAtomicD, [AscalonLS]>;
+ def : WriteRes<WriteAtomicLDW, [AscalonLS]>;
+ def : WriteRes<WriteAtomicLDD, [AscalonLS]>;
+ }
-// Single precision.
-let Latency = 3 in {
-def : WriteRes<WriteFAdd32, [AscalonFP]>;
-def : WriteRes<WriteFMul32, [AscalonFP]>;
-def : WriteRes<WriteFMA32, [AscalonFP]>;
-def : WriteRes<WriteFSGNJ32, [AscalonFP]>;
-def : WriteRes<WriteFMinMax32, [AscalonFP]>;
-}
+ // Half precision.
+ let Latency = 3 in {
+ def : WriteRes<WriteFAdd16, [AscalonFP]>;
+ def : WriteRes<WriteFMul16, [AscalonFP]>;
+ def : WriteRes<WriteFMA16, [AscalonFP]>;
+ def : WriteRes<WriteFSGNJ16, [AscalonFP]>;
+ def : WriteRes<WriteFMinMax16, [AscalonFP]>;
+ }
-let Latency = 7, ReleaseAtCycles = [7] in {
-def : WriteRes<WriteFDiv32, [AscalonFP]>;
-def : WriteRes<WriteFSqrt32, [AscalonFP]>;
-}
+ let Latency = 7, ReleaseAtCycles = [7] in {
+ def : WriteRes<WriteFDiv16, [AscalonFP]>;
+ def : WriteRes<WriteFSqrt16, [AscalonFP]>;
+ }
-// Double precision
-let Latency = 3 in {
-def : WriteRes<WriteFAdd64, [AscalonFP]>;
-def : WriteRes<WriteFMul64, [AscalonFP]>;
-def : WriteRes<WriteFMA64, [AscalonFP]>;
-def : WriteRes<WriteFSGNJ64, [AscalonFP]>;
-def : WriteRes<WriteFMinMax64, [AscalonFP]>;
-}
+ // Single precision.
+ let Latency = 3 in {
+ def : WriteRes<WriteFAdd32, [AscalonFP]>;
+ def : WriteRes<WriteFMul32, [AscalonFP]>;
+ def : WriteRes<WriteFMA32, [AscalonFP]>;
+ def : WriteRes<WriteFSGNJ32, [AscalonFP]>;
+ def : WriteRes<WriteFMinMax32, [AscalonFP]>;
+ }
-let Latency = 12, ReleaseAtCycles = [12] in {
-def : WriteRes<WriteFDiv64, [AscalonFP]>;
-def : WriteRes<WriteFSqrt64, [AscalonFP]>;
-}
+ let Latency = 7, ReleaseAtCycles = [7] in {
+ def : WriteRes<WriteFDiv32, [AscalonFP]>;
+ def : WriteRes<WriteFSqrt32, [AscalonFP]>;
+ }
-// Conversions
-def : WriteRes<WriteFCvtI32ToF16, [AscalonFXB]>;
-def : WriteRes<WriteFCvtI32ToF32, [AscalonFXB]>;
-def : WriteRes<WriteFCvtI32ToF64, [AscalonFXB]>;
-def : WriteRes<WriteFCvtI64ToF16, [AscalonFXB]>;
-def : WriteRes<WriteFCvtI64ToF32, [AscalonFXB]>;
-def : WriteRes<WriteFCvtI64ToF64, [AscalonFXB]>;
-def : WriteRes<WriteFCvtF16ToI32, [AscalonFXA]>;
-def : WriteRes<WriteFCvtF16ToI64, [AscalonFXA]>;
-def : WriteRes<WriteFCvtF16ToF32, [AscalonFP]>;
-def : WriteRes<WriteFCvtF16ToF64, [AscalonFP]>;
-def : WriteRes<WriteFCvtF32ToI32, [AscalonFXA]>;
-def : WriteRes<WriteFCvtF32ToI64, [AscalonFXA]>;
-def : WriteRes<WriteFCvtF32ToF16, [AscalonFP]>;
-def : WriteRes<WriteFCvtF32ToF64, [AscalonFP]>;
-def : WriteRes<WriteFCvtF64ToI32, [AscalonFXA]>;
-def : WriteRes<WriteFCvtF64ToI64, [AscalonFXA]>;
-def : WriteRes<WriteFCvtF64ToF16, [AscalonFP]>;
-def : WriteRes<WriteFCvtF64ToF32, [AscalonFP]>;
-
-def : WriteRes<WriteFClass16, [AscalonFP]>;
-def : WriteRes<WriteFClass32, [AscalonFP]>;
-def : WriteRes<WriteFClass64, [AscalonFP]>;
-def : WriteRes<WriteFCmp16, [AscalonFP]>;
-def : WriteRes<WriteFCmp32, [AscalonFP]>;
-def : WriteRes<WriteFCmp64, [AscalonFP]>;
-
-def : WriteRes<WriteFMovI16ToF16, [AscalonFXB]>;
-def : WriteRes<WriteFMovF16ToI16, [AscalonFXA]>;
-def : WriteRes<WriteFMovI32ToF32, [AscalonFXB]>;
-def : WriteRes<WriteFMovF32ToI32, [AscalonFXA]>;
-def : WriteRes<WriteFMovI64ToF64, [AscalonFXB]>;
-def : WriteRes<WriteFMovF64ToI64, [AscalonFXA]>;
-
-// Others
-def : WriteRes<WriteCSR, [AscalonFXA]>;
-def : WriteRes<WriteNop, [AscalonFX]>;
-
-def : InstRW<[WriteIALU], (instrs COPY)>;
+ // Double precision
+ let Latency = 3 in {
+ def : WriteRes<WriteFAdd64, [AscalonFP]>;
+ def : WriteRes<WriteFMul64, [AscalonFP]>;
+ def : WriteRes<WriteFMA64, [AscalonFP]>;
+ def : WriteRes<WriteFSGNJ64, [AscalonFP]>;
+ def : WriteRes<WriteFMinMax64, [AscalonFP]>;
+ }
-//===----------------------------------------------------------------------===//
-// Bypass and advance
-def : ReadAdvance<ReadJmp, 0>;
-def : ReadAdvance<ReadJalr, 0>;
-def : ReadAdvance<ReadCSR, 0>;
-def : ReadAdvance<ReadStoreData, 0>;
-def : ReadAdvance<ReadMemBase, 0>;
-def : ReadAdvance<ReadIALU, 0>;
-def : ReadAdvance<ReadIALU32, 0>;
-def : ReadAdvance<ReadShiftImm, 0>;
-def : ReadAdvance<ReadShiftImm32, 0>;
-def : ReadAdvance<ReadShiftReg, 0>;
-def : ReadAdvance<ReadShiftReg32, 0>;
-def : ReadAdvance<ReadIDiv, 0>;
-def : ReadAdvance<ReadIDiv32, 0>;
-def : ReadAdvance<ReadIRem, 0>;
-def : ReadAdvance<ReadIRem32, 0>;
-def : ReadAdvance<ReadIMul, 0>;
-def : ReadAdvance<ReadIMul32, 0>;
-def : ReadAdvance<ReadAtomicWA, 0>;
-def : ReadAdvance<ReadAtomicWD, 0>;
-def : ReadAdvance<ReadAtomicDA, 0>;
-def : ReadAdvance<ReadAtomicDD, 0>;
-def : ReadAdvance<ReadAtomicLDW, 0>;
-def : ReadAdvance<ReadAtomicLDD, 0>;
-def : ReadAdvance<ReadAtomicSTW, 0>;
-def : ReadAdvance<ReadAtomicSTD, 0>;
-def : ReadAdvance<ReadFStoreData, 0>;
-def : ReadAdvance<ReadFMemBase, 0>;
-def : ReadAdvance<ReadFAdd16, 0>;
-def : ReadAdvance<ReadFAdd32, 0>;
-def : ReadAdvance<ReadFAdd64, 0>;
-def : ReadAdvance<ReadFMul16, 0>;
-def : ReadAdvance<ReadFMA16, 0>;
-def : ReadAdvance<ReadFMA16Addend, 0>;
-def : ReadAdvance<ReadFMul32, 0>;
-def : ReadAdvance<ReadFMul64, 0>;
-def : ReadAdvance<ReadFMA32, 0>;
-def : ReadAdvance<ReadFMA32Addend, 0>;
-def : ReadAdvance<ReadFMA64, 0>;
-def : ReadAdvance<ReadFMA64Addend, 0>;
-def : ReadAdvance<ReadFDiv16, 0>;
-def : ReadAdvance<ReadFDiv32, 0>;
-def : ReadAdvance<ReadFDiv64, 0>;
-def : ReadAdvance<ReadFSqrt16, 0>;
-def : ReadAdvance<ReadFSqrt32, 0>;
-def : ReadAdvance<ReadFSqrt64, 0>;
-def : ReadAdvance<ReadFCmp16, 0>;
-def : ReadAdvance<ReadFCmp32, 0>;
-def : ReadAdvance<ReadFCmp64, 0>;
-def : ReadAdvance<ReadFSGNJ16, 0>;
-def : ReadAdvance<ReadFSGNJ32, 0>;
-def : ReadAdvance<ReadFSGNJ64, 0>;
-def : ReadAdvance<ReadFMinMax16, 0>;
-def : ReadAdvance<ReadFMinMax32, 0>;
-def : ReadAdvance<ReadFMinMax64, 0>;
-def : ReadAdvance<ReadFCvtF16ToI32, 0>;
-def : ReadAdvance<ReadFCvtF16ToI64, 0>;
-def : ReadAdvance<ReadFCvtF32ToI32, 0>;
-def : ReadAdvance<ReadFCvtF32ToI64, 0>;
-def : ReadAdvance<ReadFCvtF64ToI32, 0>;
-def : ReadAdvance<ReadFCvtF64ToI64, 0>;
-def : ReadAdvance<ReadFCvtI32ToF16, 0>;
-def : ReadAdvance<ReadFCvtI32ToF32, 0>;
-def : ReadAdvance<ReadFCvtI32ToF64, 0>;
-def : ReadAdvance<ReadFCvtI64ToF16, 0>;
-def : ReadAdvance<ReadFCvtI64ToF32, 0>;
-def : ReadAdvance<ReadFCvtI64ToF64, 0>;
-def : ReadAdvance<ReadFCvtF32ToF64, 0>;
-def : ReadAdvance<ReadFCvtF64ToF32, 0>;
-def : ReadAdvance<ReadFCvtF16ToF32, 0>;
-def : ReadAdvance<ReadFCvtF32ToF16, 0>;
-def : ReadAdvance<ReadFCvtF16ToF64, 0>;
-def : ReadAdvance<ReadFCvtF64ToF16, 0>;
-def : ReadAdvance<ReadFMovF16ToI16, 0>;
-def : ReadAdvance<ReadFMovI16ToF16, 0>;
-def : ReadAdvance<ReadFMovF32ToI32, 0>;
-def : ReadAdvance<ReadFMovI32ToF32, 0>;
-def : ReadAdvance<ReadFMovF64ToI64, 0>;
-def : ReadAdvance<ReadFMovI64ToF64, 0>;
-def : ReadAdvance<ReadFClass16, 0>;
-def : ReadAdvance<ReadFClass32, 0>;
-def : ReadAdvance<ReadFClass64, 0>;
-
-// Bitmanip
-def : ReadAdvance<ReadRotateImm, 0>;
-def : ReadAdvance<ReadRotateImm32, 0>;
-def : ReadAdvance<ReadRotateReg, 0>;
-def : ReadAdvance<ReadRotateReg32, 0>;
-def : ReadAdvance<ReadCLZ, 0>;
-def : ReadAdvance<ReadCLZ32, 0>;
-def : ReadAdvance<ReadCTZ, 0>;
-def : ReadAdvance<ReadCTZ32, 0>;
-def : ReadAdvance<ReadCPOP, 0>;
-def : ReadAdvance<ReadCPOP32, 0>;
-def : ReadAdvance<ReadORCB, 0>;
-def : ReadAdvance<ReadIMinMax, 0>;
-def : ReadAdvance<ReadREV8, 0>;
-def : ReadAdvance<ReadSHXADD, 0>;
-def : ReadAdvance<ReadSHXADD32, 0>;
-// Single-bit instructions
-def : ReadAdvance<ReadSingleBit, 0>;
-def : ReadAdvance<ReadSingleBitImm, 0>;
+ let Latency = 12, ReleaseAtCycles = [12] in {
+ def : WriteRes<WriteFDiv64, [AscalonFP]>;
+ def : WriteRes<WriteFSqrt64, [AscalonFP]>;
+ }
-//===----------------------------------------------------------------------===//
-// Vector
-def : WriteRes<WriteRdVLENB, [AscalonFXA]>;
-
-// Configuration-Setting Instructions
-def : WriteRes<WriteVSETVLI, [AscalonV]>;
-def : WriteRes<WriteVSETIVLI, [AscalonV]>;
-let Latency = 2 in {
- def : WriteRes<WriteVSETVL, [AscalonV]>;
-}
+ // Conversions
+ def : WriteRes<WriteFCvtI32ToF16, [AscalonFXB]>;
+ def : WriteRes<WriteFCvtI32ToF32, [AscalonFXB]>;
+ def : WriteRes<WriteFCvtI32ToF64, [AscalonFXB]>;
+ def : WriteRes<WriteFCvtI64ToF16, [AscalonFXB]>;
+ def : WriteRes<WriteFCvtI64ToF32, [AscalonFXB]>;
+ def : WriteRes<WriteFCvtI64ToF64, [AscalonFXB]>;
+ def : WriteRes<WriteFCvtF16ToI32, [AscalonFXA]>;
+ def : WriteRes<WriteFCvtF16ToI64, [AscalonFXA]>;
+ def : WriteRes<WriteFCvtF16ToF32, [AscalonFP]>;
+ def : WriteRes<WriteFCvtF16ToF64, [AscalonFP]>;
+ def : WriteRes<WriteFCvtF32ToI32, [AscalonFXA]>;
+ def : WriteRes<WriteFCvtF32ToI64, [AscalonFXA]>;
+ def : WriteRes<WriteFCvtF32ToF16, [AscalonFP]>;
+ def : WriteRes<WriteFCvtF32ToF64, [AscalonFP]>;
+ def : WriteRes<WriteFCvtF64ToI32, [AscalonFXA]>;
+ def : WriteRes<WriteFCvtF64ToI64, [AscalonFXA]>;
+ def : WriteRes<WriteFCvtF64ToF16, [AscalonFP]>;
+ def : WriteRes<WriteFCvtF64ToF32, [AscalonFP]>;
+
+ def : WriteRes<WriteFClass16, [AscalonFP]>;
+ def : WriteRes<WriteFClass32, [AscalonFP]>;
+ def : WriteRes<WriteFClass64, [AscalonFP]>;
+ def : WriteRes<WriteFCmp16, [AscalonFP]>;
+ def : WriteRes<WriteFCmp32, [AscalonFP]>;
+ def : WriteRes<WriteFCmp64, [AscalonFP]>;
+
+ def : WriteRes<WriteFMovI16ToF16, [AscalonFXB]>;
+ def : WriteRes<WriteFMovF16ToI16, [AscalonFXA]>;
+ def : WriteRes<WriteFMovI32ToF32, [AscalonFXB]>;
+ def : WriteRes<WriteFMovF32ToI32, [AscalonFXA]>;
+ def : WriteRes<WriteFMovI64ToF64, [AscalonFXB]>;
+ def : WriteRes<WriteFMovF64ToI64, [AscalonFXA]>;
+
+ // Others
+ def : WriteRes<WriteCSR, [AscalonFXA]>;
+ def : WriteRes<WriteNop, [AscalonFX]>;
+
+ def : InstRW<[WriteIALU], (instrs COPY)>;
+
+ //===--------------------------------------------------------------------===//
+ // Bypass and advance
+ def : ReadAdvance<ReadJmp, 0>;
+ def : ReadAdvance<ReadJalr, 0>;
+ def : ReadAdvance<ReadCSR, 0>;
+ def : ReadAdvance<ReadStoreData, 0>;
+ def : ReadAdvance<ReadMemBase, 0>;
+ def : ReadAdvance<ReadIALU, 0>;
+ def : ReadAdvance<ReadIALU32, 0>;
+ def : ReadAdvance<ReadShiftImm, 0>;
+ def : ReadAdvance<ReadShiftImm32, 0>;
+ def : ReadAdvance<ReadShiftReg, 0>;
+ def : ReadAdvance<ReadShiftReg32, 0>;
+ def : ReadAdvance<ReadIDiv, 0>;
+ def : ReadAdvance<ReadIDiv32, 0>;
+ def : ReadAdvance<ReadIRem, 0>;
+ def : ReadAdvance<ReadIRem32, 0>;
+ def : ReadAdvance<ReadIMul, 0>;
+ def : ReadAdvance<ReadIMul32, 0>;
+ def : ReadAdvance<ReadAtomicWA, 0>;
+ def : ReadAdvance<ReadAtomicWD, 0>;
+ def : ReadAdvance<ReadAtomicDA, 0>;
+ def : ReadAdvance<ReadAtomicDD, 0>;
+ def : ReadAdvance<ReadAtomicLDW, 0>;
+ def : ReadAdvance<ReadAtomicLDD, 0>;
+ def : ReadAdvance<ReadAtomicSTW, 0>;
+ def : ReadAdvance<ReadAtomicSTD, 0>;
+ def : ReadAdvance<ReadFStoreData, 0>;
+ def : ReadAdvance<ReadFMemBase, 0>;
+ def : ReadAdvance<ReadFAdd16, 0>;
+ def : ReadAdvance<ReadFAdd32, 0>;
+ def : ReadAdvance<ReadFAdd64, 0>;
+ def : ReadAdvance<ReadFMul16, 0>;
+ def : ReadAdvance<ReadFMA16, 0>;
+ def : ReadAdvance<ReadFMA16Addend, 0>;
+ def : ReadAdvance<ReadFMul32, 0>;
+ def : ReadAdvance<ReadFMul64, 0>;
+ def : ReadAdvance<ReadFMA32, 0>;
+ def : ReadAdvance<ReadFMA32Addend, 0>;
+ def : ReadAdvance<ReadFMA64, 0>;
+ def : ReadAdvance<ReadFMA64Addend, 0>;
+ def : ReadAdvance<ReadFDiv16, 0>;
+ def : ReadAdvance<ReadFDiv32, 0>;
+ def : ReadAdvance<ReadFDiv64, 0>;
+ def : ReadAdvance<ReadFSqrt16, 0>;
+ def : ReadAdvance<ReadFSqrt32, 0>;
+ def : ReadAdvance<ReadFSqrt64, 0>;
+ def : ReadAdvance<ReadFCmp16, 0>;
+ def : ReadAdvance<ReadFCmp32, 0>;
+ def : ReadAdvance<ReadFCmp64, 0>;
+ def : ReadAdvance<ReadFSGNJ16, 0>;
+ def : ReadAdvance<ReadFSGNJ32, 0>;
+ def : ReadAdvance<ReadFSGNJ64, 0>;
+ def : ReadAdvance<ReadFMinMax16, 0>;
+ def : ReadAdvance<ReadFMinMax32, 0>;
+ def : ReadAdvance<ReadFMinMax64, 0>;
+ def : ReadAdvance<ReadFCvtF16ToI32, 0>;
+ def : ReadAdvance<ReadFCvtF16ToI64, 0>;
+ def : ReadAdvance<ReadFCvtF32ToI32, 0>;
+ def : ReadAdvance<ReadFCvtF32ToI64, 0>;
+ def : ReadAdvance<ReadFCvtF64ToI32, 0>;
+ def : ReadAdvance<ReadFCvtF64ToI64, 0>;
+ def : ReadAdvance<ReadFCvtI32ToF16, 0>;
+ def : ReadAdvance<ReadFCvtI32ToF32, 0>;
+ def : ReadAdvance<ReadFCvtI32ToF64, 0>;
+ def : ReadAdvance<ReadFCvtI64ToF16, 0>;
+ def : ReadAdvance<ReadFCvtI64ToF32, 0>;
+ def : ReadAdvance<ReadFCvtI64ToF64, 0>;
+ def : ReadAdvance<ReadFCvtF32ToF64, 0>;
+ def : ReadAdvance<ReadFCvtF64ToF32, 0>;
+ def : ReadAdvance<ReadFCvtF16ToF32, 0>;
+ def : ReadAdvance<ReadFCvtF32ToF16, 0>;
+ def : ReadAdvance<ReadFCvtF16ToF64, 0>;
+ def : ReadAdvance<ReadFCvtF64ToF16, 0>;
+ def : ReadAdvance<ReadFMovF16ToI16, 0>;
+ def : ReadAdvance<ReadFMovI16ToF16, 0>;
+ def : ReadAdvance<ReadFMovF32ToI32, 0>;
+ def : ReadAdvance<ReadFMovI32ToF32, 0>;
+ def : ReadAdvance<ReadFMovF64ToI64, 0>;
+ def : ReadAdvance<ReadFMovI64ToF64, 0>;
+ def : ReadAdvance<ReadFClass16, 0>;
+ def : ReadAdvance<ReadFClass32, 0>;
+ def : ReadAdvance<ReadFClass64, 0>;
+
+ // Bitmanip
+ def : ReadAdvance<ReadRotateImm, 0>;
+ def : ReadAdvance<ReadRotateImm32, 0>;
+ def : ReadAdvance<ReadRotateReg, 0>;
+ def : ReadAdvance<ReadRotateReg32, 0>;
+ def : ReadAdvance<ReadCLZ, 0>;
+ def : ReadAdvance<ReadCLZ32, 0>;
+ def : ReadAdvance<ReadCTZ, 0>;
+ def : ReadAdvance<ReadCTZ32, 0>;
+ def : ReadAdvance<ReadCPOP, 0>;
+ def : ReadAdvance<ReadCPOP32, 0>;
+ def : ReadAdvance<ReadORCB, 0>;
+ def : ReadAdvance<ReadIMinMax, 0>;
+ def : ReadAdvance<ReadREV8, 0>;
+ def : ReadAdvance<ReadSHXADD, 0>;
+ def : ReadAdvance<ReadSHXADD32, 0>;
+ // Single-bit instructions
+ def : ReadAdvance<ReadSingleBit, 0>;
+ def : ReadAdvance<ReadSingleBitImm, 0>;
+
+ //===--------------------------------------------------------------------===//
+ // Vector
+ def : WriteRes<WriteRdVLENB, [AscalonFXA]>;
-// Vector Loads and Stores
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLDE", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDFF", [AscalonLS], mx, IsWorstCase>;
+ // Configuration-Setting Instructions
+ def : WriteRes<WriteVSETVLI, [AscalonV]>;
+ def : WriteRes<WriteVSETIVLI, [AscalonV]>;
+ let Latency = 2 in {
+ def : WriteRes<WriteVSETVL, [AscalonV]>;
}
- defm "" : LMULWriteResMX<"WriteVSTE", [AscalonLS], mx, IsWorstCase>;
-}
-foreach mx = SchedMxList in {
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- defm "" : LMULWriteResMX<"WriteVLDM", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTM", [AscalonLS], mx, IsWorstCase>;
-}
+ // Vector Loads and Stores
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLDE", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDFF", [AscalonLS], mx, IsWorstCase>;
+ }
+ defm "" : LMULWriteResMX<"WriteVSTE", [AscalonLS], mx, IsWorstCase>;
+ }
-foreach mx = SchedMxListEEW8 in {
- defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLDS8", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDUX8", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDOX8", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTS8", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTUX8", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTOX8", [AscalonLS], mx, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ defm "" : LMULWriteResMX<"WriteVLDM", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTM", [AscalonLS], mx, IsWorstCase>;
}
-}
-foreach mx = SchedMxListEEW16 in {
- defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLDS16", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDUX16", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDOX16", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTS16", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTUX16", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTOX16", [AscalonLS], mx, IsWorstCase>;
+
+ foreach mx = SchedMxListEEW8 in {
+ defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLDS8", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDUX8", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDOX8", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTS8", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTUX8", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTOX8", [AscalonLS], mx, IsWorstCase>;
+ }
}
-}
-foreach mx = SchedMxListEEW32 in {
- defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLDS32", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDUX32", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDOX32", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTS32", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTUX32", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTOX32", [AscalonLS], mx, IsWorstCase>;
+ foreach mx = SchedMxListEEW16 in {
+ defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLDS16", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDUX16", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDOX16", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTS16", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTUX16", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTOX16", [AscalonLS], mx, IsWorstCase>;
+ }
}
-}
-foreach mx = SchedMxListEEW64 in {
- defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLDS64", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDUX64", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLDOX64", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTS64", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTUX64", [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSTOX64", [AscalonLS], mx, IsWorstCase>;
+ foreach mx = SchedMxListEEW32 in {
+ defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLDS32", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDUX32", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDOX32", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTS32", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTUX32", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTOX32", [AscalonLS], mx, IsWorstCase>;
+ }
+ }
+ foreach mx = SchedMxListEEW64 in {
+ defvar Cycles = AscalonGetCyclesLMUL<mx, 2>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLDS64", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDUX64", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLDOX64", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTS64", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTUX64", [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSTOX64", [AscalonLS], mx, IsWorstCase>;
+ }
}
-}
-// VLD*R is not LMUL aware
-def : WriteRes<WriteVLD1R, [AscalonLS]>;
-def : WriteRes<WriteVLD2R, [AscalonLS]>;
-def : WriteRes<WriteVLD4R, [AscalonLS]>;
-def : WriteRes<WriteVLD8R, [AscalonLS]>;
-// VST*R is not LMUL aware
-def : WriteRes<WriteVST1R, [AscalonLS]>;
-def : WriteRes<WriteVST2R, [AscalonLS]>;
-def : WriteRes<WriteVST4R, [AscalonLS]>;
-def : WriteRes<WriteVST8R, [AscalonLS]>;
-
-// Segmented Loads and Stores
-foreach mx = SchedMxList in {
- foreach eew = [8, 16, 32, 64] in {
- foreach nf=2-8 in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLSEGFF" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ // VLD*R is not LMUL aware
+ def : WriteRes<WriteVLD1R, [AscalonLS]>;
+ def : WriteRes<WriteVLD2R, [AscalonLS]>;
+ def : WriteRes<WriteVLD4R, [AscalonLS]>;
+ def : WriteRes<WriteVLD8R, [AscalonLS]>;
+ // VST*R is not LMUL aware
+ def : WriteRes<WriteVST1R, [AscalonLS]>;
+ def : WriteRes<WriteVST2R, [AscalonLS]>;
+ def : WriteRes<WriteVST4R, [AscalonLS]>;
+ def : WriteRes<WriteVST8R, [AscalonLS]>;
+
+ // Segmented Loads and Stores
+ foreach mx = SchedMxList in {
+ foreach eew = [8, 16, 32, 64] in {
+ foreach nf=2-8 in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLSEGFF" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ }
+ let Latency = 1, AcquireAtCycles = [1], ReleaseAtCycles = [!add(1, Cycles)] in
+ defm "" : LMULWriteResMX<"WriteVSSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
}
- let Latency = 1, AcquireAtCycles = [1], ReleaseAtCycles = [!add(1, Cycles)] in
- defm "" : LMULWriteResMX<"WriteVSSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
}
}
-}
-foreach mx = SchedMxList in {
- foreach nf=2-8 in {
- foreach eew = [8, 16, 32, 64] in {
- defvar Cycles = AscalonGetCyclesStridedSegmented<mx, eew>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles in {
- defm "" : LMULWriteResMX<"WriteVLSSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLUXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVLOXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSSSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSUXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSOXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ foreach nf=2-8 in {
+ foreach eew = [8, 16, 32, 64] in {
+ defvar Cycles = AscalonGetCyclesStridedSegmented<mx, eew>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles in {
+ defm "" : LMULWriteResMX<"WriteVLSSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLUXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVLOXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSSSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSUXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSOXSEG" # nf # "e" # eew, [AscalonLS], mx, IsWorstCase>;
+ }
}
}
}
-}
-// Vector Fixed-Point Arithmetic Instructions
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVSALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVAALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVAALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSMulV", [AscalonFXA, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSMulX", [AscalonFXA, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSShiftV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSShiftX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSShiftI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ // Vector Fixed-Point Arithmetic Instructions
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULWriteResMX<"WriteVSALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVAALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVAALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSMulV", [AscalonFXA, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSMulX", [AscalonFXA, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSShiftV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSShiftX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSShiftI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ }
}
-}
-// Narrowing
-foreach mx = SchedMxListW in {
- defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVNClipV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVNClipX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVNClipI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ // Narrowing
+ foreach mx = SchedMxListW in {
+ defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULWriteResMX<"WriteVNClipV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVNClipX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVNClipI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ }
}
-}
-// Configuration-Setting Instructions
-def : ReadAdvance<ReadVSETVLI, 1>;
-def : ReadAdvance<ReadVSETVL, 1>;
-
-// Vector Loads and Stores
-def : ReadAdvance<ReadVLDX, 0>;
-def : ReadAdvance<ReadVSTX, 0>;
-defm "" : LMULReadAdvance<"ReadVSTEV", 0>;
-defm "" : LMULReadAdvance<"ReadVSTM", 0>;
-def : ReadAdvance<ReadVLDSX, 0>;
-def : ReadAdvance<ReadVSTSX, 0>;
-defm "" : LMULReadAdvance<"ReadVSTS8V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTS16V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTS32V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTS64V", 0>;
-defm "" : LMULReadAdvance<"ReadVLDUXV", 0>;
-defm "" : LMULReadAdvance<"ReadVLDOXV", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX8", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX16", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX32", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX64", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUXV", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX8V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX16V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX32V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTUX64V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX8", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX16", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX32", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX64", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOXV", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX8V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX16V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX32V", 0>;
-defm "" : LMULReadAdvance<"ReadVSTOX64V", 0>;
-// LMUL Aware
-def : ReadAdvance<ReadVST1R, 0>;
-def : ReadAdvance<ReadVST2R, 0>;
-def : ReadAdvance<ReadVST4R, 0>;
-def : ReadAdvance<ReadVST8R, 0>;
-
-// Vector Integer Arithmetic Instructions
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVIALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICALUMV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICALUMX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICALUMI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVShiftV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVShiftX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVShiftI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMinMaxV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMinMaxX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMulV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMulX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMulAddV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMulAddX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMergeV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMergeX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMergeI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMovV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMovX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIMovI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICmpV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICmpX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVICmpI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- }
-}
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVExtV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- }
-}
-foreach mx = SchedMxList in {
- foreach sew = SchedSEWSet<mx>.val in {
- defvar Cycles = AscalonGetCyclesDivOrSqrt<mx, sew>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+ // Configuration-Setting Instructions
+ def : ReadAdvance<ReadVSETVLI, 1>;
+ def : ReadAdvance<ReadVSETVL, 1>;
+
+ // Vector Loads and Stores
+ def : ReadAdvance<ReadVLDX, 0>;
+ def : ReadAdvance<ReadVSTX, 0>;
+ defm "" : LMULReadAdvance<"ReadVSTEV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTM", 0>;
+ def : ReadAdvance<ReadVLDSX, 0>;
+ def : ReadAdvance<ReadVSTSX, 0>;
+ defm "" : LMULReadAdvance<"ReadVSTS8V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTS16V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTS32V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTS64V", 0>;
+ defm "" : LMULReadAdvance<"ReadVLDUXV", 0>;
+ defm "" : LMULReadAdvance<"ReadVLDOXV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX8", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX16", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX32", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX64", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUXV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX8V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX16V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX32V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTUX64V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX8", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX16", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX32", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX64", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOXV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX8V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX16V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX32V", 0>;
+ defm "" : LMULReadAdvance<"ReadVSTOX64V", 0>;
+ // LMUL Aware
+ def : ReadAdvance<ReadVST1R, 0>;
+ def : ReadAdvance<ReadVST2R, 0>;
+ def : ReadAdvance<ReadVST4R, 0>;
+ def : ReadAdvance<ReadVST8R, 0>;
+
+ // Vector Integer Arithmetic Instructions
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVIDivV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVIDivX", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICALUMV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICALUMX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICALUMI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVShiftV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVShiftX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVShiftI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMinMaxV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMinMaxX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMulV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMulX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMulAddV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMulAddX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMergeV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMergeX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMergeI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMovV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMovX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIMovI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICmpV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICmpX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVICmpI", [AscalonFX, AscalonV], mx, IsWorstCase>;
}
}
-}
-
-// Widening
-foreach mx = SchedMxListW in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVIWALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIWALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIWALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIWMulV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIWMulX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIWMulAddV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIWMulAddX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULWriteResMX<"WriteVExtV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ }
}
-}
-// Narrowing
-foreach mx = SchedMxListW in {
- defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVNShiftV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVNShiftX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVNShiftI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ foreach sew = SchedSEWSet<mx>.val in {
+ defvar Cycles = AscalonGetCyclesDivOrSqrt<mx, sew>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVIDivV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVIDivX", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ }
+ }
}
-}
-// Vector Floating-Point Instructions
-foreach mx = SchedMxListF in {
- foreach sew = SchedSEWSet<mx, isF=1>.val in {
+ // Widening
+ foreach mx = SchedMxListW in {
defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWALUV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWALUX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWALUI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWMulV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWMulX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWMulAddV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIWMulAddX", [AscalonFX, AscalonV], mx, IsWorstCase>;
}
}
-}
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [AscalonFPA, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFClassV", [AscalonFP, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFMergeV", [AscalonFP, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFMovV", [AscalonFP, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFCmpV", [AscalonFP, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFCmpF", [AscalonFP, AscalonV], mx, IsWorstCase>;
- }
-}
-foreach mx = SchedMxListF in {
- foreach sew = SchedSEWSet<mx, isF=1>.val in {
- defvar Cycles = AscalonGetCyclesDivOrSqrt<mx, sew>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+ // Narrowing
+ foreach mx = SchedMxListW in {
+ defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVFSqrtV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFDivV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFDivF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVNShiftV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVNShiftX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVNShiftI", [AscalonFX, AscalonV], mx, IsWorstCase>;
}
}
-}
-// Widening
-foreach mx = SchedMxListW in {
- foreach sew = SchedSEWSet<mx, isF=0, isWidening=1>.val in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListW>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtIToFV", [AscalonFXB, AscalonV], mx, sew, IsWorstCase>;
+ // Vector Floating-Point Instructions
+ foreach mx = SchedMxListF in {
+ foreach sew = SchedSEWSet<mx, isF=1>.val in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+ // FP64 add/mul/fma have reduced throughput on Ascalon XG.
+ defvar IsFP64 = !and(ReducedFP64Vec, !eq(sew, 64));
+ defvar FP64 = AscalonXGFP64Vec<Cycles>;
+ defvar FPVRes = !if(IsFP64, [AscalonFP, AscalonV, AscalonFP64V],
+ [AscalonFP, AscalonV]);
+ let Latency = !if(IsFP64, FP64.Latency, Cycles),
+ ReleaseAtCycles = !if(IsFP64, [1, Cycles, FP64.ReleaseAtCycles],
+ [1, Cycles]) in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjF", FPVRes, mx, sew, IsWorstCase>;
+ }
+ }
}
-}
-foreach mx = SchedMxListFW in {
- foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+ foreach mx = SchedMxList in {
defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtFToFV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [AscalonFPA, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFClassV", [AscalonFP, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFMergeV", [AscalonFP, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFMovV", [AscalonFP, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFCmpV", [AscalonFP, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFCmpF", [AscalonFP, AscalonV], mx, IsWorstCase>;
}
}
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListFW>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in
- defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [AscalonFPA, AscalonV], mx, IsWorstCase>;
-}
-// Narrowing
-foreach mx = SchedMxListW in {
- defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVFNCvtFToIV", [AscalonFPA, AscalonV], mx, IsWorstCase>;
+ foreach mx = SchedMxListF in {
+ foreach sew = SchedSEWSet<mx, isF=1>.val in {
+ defvar Cycles = AscalonGetCyclesDivOrSqrt<mx, sew>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFSqrtV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFDivV", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFDivF", [AscalonFP, AscalonV], mx, sew, IsWorstCase>;
+ }
+ }
}
-}
-foreach mx = SchedMxListFW in {
- foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+
+ // Widening
+ foreach mx = SchedMxListW in {
+ foreach sew = SchedSEWSet<mx, isF=0, isWidening=1>.val in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListW>.c;
+ defvar IsFP64 = !and(ReducedFP64Vec, !eq(sew, 32));
+ defvar FP64 = AscalonXGFP64Vec<Cycles>;
+ defvar FXBVRes = !if(IsFP64, [AscalonFXB, AscalonV, AscalonFP64V],
+ [AscalonFXB, AscalonV]);
+ let Latency = !if(IsFP64, FP64.Latency, Cycles),
+ ReleaseAtCycles = !if(IsFP64, [1, Cycles, FP64.ReleaseAtCycles],
+ [1, Cycles]) in
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtIToFV", FXBVRes, mx, sew, IsWorstCase>;
+ }
+ }
+ foreach mx = SchedMxListFW in {
+ foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+ defvar IsFP64 = !and(ReducedFP64Vec, !eq(sew, 32));
+ defvar FP64 = AscalonXGFP64Vec<Cycles>;
+ defvar FPVRes = !if(IsFP64, [AscalonFP, AscalonV, AscalonFP64V],
+ [AscalonFP, AscalonV]);
+ let Latency = !if(IsFP64, FP64.Latency, Cycles),
+ ReleaseAtCycles = !if(IsFP64, [1, Cycles, FP64.ReleaseAtCycles],
+ [1, Cycles]) in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", FPVRes, mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtFToFV", FPVRes, mx, sew, IsWorstCase>;
+ }
+ }
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListFW>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in
+ defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [AscalonFPA, AscalonV], mx, IsWorstCase>;
+ }
+ // Narrowing
+ foreach mx = SchedMxListW in {
defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxListW>.c;
let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [AscalonFXB, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [AscalonFXB, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFNCvtFToIV", [AscalonFPA, AscalonV], mx, IsWorstCase>;
+ }
+ }
+ foreach mx = SchedMxListFW in {
+ foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+ defvar Cycles = AscalonGetCyclesNarrowing<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [AscalonFXB, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [AscalonFXB, AscalonV], mx, sew, IsWorstCase>;
+ }
}
}
-}
-// Vector Reduction Instructions
-foreach mx = SchedMxList in {
- foreach sew = SchedSEWSet<mx>.val in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVIRedV_From", [AscalonFX, AscalonV],
- mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVIRedMinMaxV_From", [AscalonFX, AscalonV],
+ // Vector Reduction Instructions
+ foreach mx = SchedMxList in {
+ foreach sew = SchedSEWSet<mx>.val in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVIRedV_From", [AscalonFX, AscalonV],
+ mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVIRedMinMaxV_From", [AscalonFX, AscalonV],
+ mx, sew, IsWorstCase>;
+ }
+ }
+ }
+
+ foreach mx = SchedMxListWRed in {
+ foreach sew = SchedSEWSet<mx, 0, 1>.val in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListWRed>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in
+ defm "" : LMULSEWWriteResMXSEW<"WriteVIWRedV_From", [AscalonFX, AscalonV],
mx, sew, IsWorstCase>;
}
}
-}
-foreach mx = SchedMxListWRed in {
- foreach sew = SchedSEWSet<mx, 0, 1>.val in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListWRed>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in
- defm "" : LMULSEWWriteResMXSEW<"WriteVIWRedV_From", [AscalonFX, AscalonV],
- mx, sew, IsWorstCase>;
+ foreach mx = SchedMxListF in {
+ foreach sew = SchedSEWSet<mx, 1>.val in {
+ defvar RedCycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+ let Latency = RedCycles, ReleaseAtCycles = [1, RedCycles] in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFRedV_From", [AscalonFX, AscalonV],
+ mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFRedMinMaxV_From", [AscalonFX, AscalonV],
+ mx, sew, IsWorstCase>;
+ }
+ defvar OrdRedCycles = AscalonGetCyclesLMUL<mx, 18>.c;
+ let Latency = OrdRedCycles, ReleaseAtCycles = [1, OrdRedCycles] in
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFRedOV_From", [AscalonFX, AscalonV],
+ mx, sew, IsWorstCase>;
+ }
}
-}
-foreach mx = SchedMxListF in {
- foreach sew = SchedSEWSet<mx, 1>.val in {
- defvar RedCycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
- let Latency = RedCycles, ReleaseAtCycles = [1, RedCycles] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVFRedV_From", [AscalonFX, AscalonV],
+ foreach mx = SchedMxListFWRed in {
+ foreach sew = SchedSEWSet<mx, 1, 1>.val in {
+ defvar RedCycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListFWRed, 1>.c;
+ let Latency = RedCycles, ReleaseAtCycles = [1, RedCycles] in
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedV_From", [AscalonFX, AscalonV],
mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVFRedMinMaxV_From", [AscalonFX, AscalonV],
+ defvar OrdRedCycles = AscalonGetCyclesLMUL<mx, 18>.c;
+ let Latency = OrdRedCycles, ReleaseAtCycles = [1, OrdRedCycles] in
+ defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedOV_From", [AscalonFX, AscalonV],
mx, sew, IsWorstCase>;
}
- defvar OrdRedCycles = AscalonGetCyclesLMUL<mx, 18>.c;
- let Latency = OrdRedCycles, ReleaseAtCycles = [1, OrdRedCycles] in
- defm "" : LMULSEWWriteResMXSEW<"WriteVFRedOV_From", [AscalonFX, AscalonV],
- mx, sew, IsWorstCase>;
}
-}
-foreach mx = SchedMxListFWRed in {
- foreach sew = SchedSEWSet<mx, 1, 1>.val in {
- defvar RedCycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxListFWRed, 1>.c;
- let Latency = RedCycles, ReleaseAtCycles = [1, RedCycles] in
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedV_From", [AscalonFX, AscalonV],
- mx, sew, IsWorstCase>;
- defvar OrdRedCycles = AscalonGetCyclesLMUL<mx, 18>.c;
- let Latency = OrdRedCycles, ReleaseAtCycles = [1, OrdRedCycles] in
- defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedOV_From", [AscalonFX, AscalonV],
- mx, sew, IsWorstCase>;
+ // Vector Mask Instructions
+ foreach mx = SchedMxList in {
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ defm "" : LMULWriteResMX<"WriteVMALUV", [AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVMSFSV", [AscalonV], mx, IsWorstCase>;
+ let Latency = 2, ReleaseAtCycles = [1, 2] in {
+ defm "" : LMULWriteResMX<"WriteVMPopV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVMFFSV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ }
+ }
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
+ defm "" : LMULWriteResMX<"WriteVIotaV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVIdxV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ }
}
-}
-// Vector Mask Instructions
-foreach mx = SchedMxList in {
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- defm "" : LMULWriteResMX<"WriteVMALUV", [AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVMSFSV", [AscalonV], mx, IsWorstCase>;
+ // Vector Permutation Instructions
let Latency = 2, ReleaseAtCycles = [1, 2] in {
- defm "" : LMULWriteResMX<"WriteVMPopV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVMFFSV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ def : WriteRes<WriteVMovSX, [AscalonFX, AscalonV]>;
+ def : WriteRes<WriteVMovXS, [AscalonFX, AscalonV]>;
+ def : WriteRes<WriteVMovSF, [AscalonFX, AscalonV]>;
+ def : WriteRes<WriteVMovFS, [AscalonFX, AscalonV]>;
}
-}
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = Cycles, ReleaseAtCycles = [1, Cycles] in {
- defm "" : LMULWriteResMX<"WriteVIotaV", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVIdxV", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = !mul(Cycles, 2), ReleaseAtCycles = [Cycles, !mul(Cycles, 2)] in {
+ defm "" : LMULWriteResMX<"WriteVRGatherVX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVRGatherVI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ }
}
-}
-// Vector Permutation Instructions
-let Latency = 2, ReleaseAtCycles = [1, 2] in {
- def : WriteRes<WriteVMovSX, [AscalonFX, AscalonV]>;
- def : WriteRes<WriteVMovXS, [AscalonFX, AscalonV]>;
- def : WriteRes<WriteVMovSF, [AscalonFX, AscalonV]>;
- def : WriteRes<WriteVMovFS, [AscalonFX, AscalonV]>;
-}
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = !mul(Cycles, 2), ReleaseAtCycles = [Cycles, !mul(Cycles, 2)] in {
- defm "" : LMULWriteResMX<"WriteVRGatherVX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVRGatherVI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ foreach sew = SchedSEWSet<mx>.val in {
+ defvar Cycles = AscalonGetCyclesVRGatherVV<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+ let Latency = !add(Cycles, 3), ReleaseAtCycles = [1, !add(1, Cycles)] in {
+ defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherEI16VV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ defm "" : LMULSEWWriteResMXSEW<"WriteVCompressV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ }
+ }
}
-}
-foreach mx = SchedMxList in {
- foreach sew = SchedSEWSet<mx>.val in {
- defvar Cycles = AscalonGetCyclesVRGatherVV<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
- let Latency = !add(Cycles, 3), ReleaseAtCycles = [1, !add(1, Cycles)] in {
- defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherEI16VV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
- defm "" : LMULSEWWriteResMXSEW<"WriteVCompressV", [AscalonFX, AscalonV], mx, sew, IsWorstCase>;
+ foreach mx = SchedMxList in {
+ defvar Cycles = AscalonGetCyclesDefault<mx>.c;
+ defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
+ let Latency = 4, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
+ defm "" : LMULWriteResMX<"WriteVSlideUpX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSlideDownX", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSlideUpI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVSlideDownI", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVISlide1Up", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVISlide1Down", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFSlide1Up", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ defm "" : LMULWriteResMX<"WriteVFSlide1Down", [AscalonFX, AscalonV], mx, IsWorstCase>;
}
}
-}
-foreach mx = SchedMxList in {
- defvar Cycles = AscalonGetCyclesDefault<mx>.c;
- defvar IsWorstCase = AscalonIsWorstCaseMX<mx, SchedMxList>.c;
- let Latency = 4, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
- defm "" : LMULWriteResMX<"WriteVSlideUpX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSlideDownX", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSlideUpI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVSlideDownI", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVISlide1Up", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVISlide1Down", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFSlide1Up", [AscalonFX, AscalonV], mx, IsWorstCase>;
- defm "" : LMULWriteResMX<"WriteVFSlide1Down", [AscalonFX, AscalonV], mx, IsWorstCase>;
+ // Whole vector register move, vmv<N>.v, not LMUL aware
+ let Latency = 1, ReleaseAtCycles = [1] in
+ def : WriteRes<WriteVMov1V, [AscalonV]>;
+ let Latency = 2, ReleaseAtCycles = [2] in
+ def : WriteRes<WriteVMov2V, [AscalonV]>;
+ let Latency = 4, ReleaseAtCycles = [4] in
+ def : WriteRes<WriteVMov4V, [AscalonV]>;
+ let Latency = 8, ReleaseAtCycles = [8] in
+ def : WriteRes<WriteVMov8V, [AscalonV]>;
+
+ // Vector Integer Arithmetic Instructions
+ defm : LMULReadAdvance<"ReadVIALUV", 0>;
+ defm : LMULReadAdvance<"ReadVIALUX", 0>;
+ defm : LMULReadAdvanceW<"ReadVIWALUV", 0>;
+ defm : LMULReadAdvanceW<"ReadVIWALUX", 0>;
+ defm : LMULReadAdvance<"ReadVExtV", 0>;
+ defm : LMULReadAdvance<"ReadVICALUV", 0>;
+ defm : LMULReadAdvance<"ReadVICALUX", 0>;
+ defm : LMULReadAdvance<"ReadVShiftV", 0>;
+ defm : LMULReadAdvance<"ReadVShiftX", 0>;
+ defm : LMULReadAdvanceW<"ReadVNShiftV", 0>;
+ defm : LMULReadAdvanceW<"ReadVNShiftX", 0>;
+ defm : LMULReadAdvance<"ReadVICmpV", 0>;
+ defm : LMULReadAdvance<"ReadVICmpX", 0>;
+ defm : LMULReadAdvance<"ReadVIMinMaxV", 0>;
+ defm : LMULReadAdvance<"ReadVIMinMaxX", 0>;
+ defm : LMULReadAdvance<"ReadVIMulV", 0>;
+ defm : LMULReadAdvance<"ReadVIMulX", 0>;
+ defm : LMULSEWReadAdvance<"ReadVIDivV", 0>;
+ defm : LMULSEWReadAdvance<"ReadVIDivX", 0>;
+ defm : LMULReadAdvanceW<"ReadVIWMulV", 0>;
+ defm : LMULReadAdvanceW<"ReadVIWMulX", 0>;
+ defm : LMULReadAdvance<"ReadVIMulAddV", 0>;
+ defm : LMULReadAdvance<"ReadVIMulAddX", 0>;
+ defm : LMULReadAdvanceW<"ReadVIWMulAddV", 0>;
+ defm : LMULReadAdvanceW<"ReadVIWMulAddX", 0>;
+ defm : LMULReadAdvance<"ReadVIMergeV", 0>;
+ defm : LMULReadAdvance<"ReadVIMergeX", 0>;
+ defm : LMULReadAdvance<"ReadVIMovV", 0>;
+ defm : LMULReadAdvance<"ReadVIMovX", 0>;
+
+ // Vector Fixed-Point Arithmetic Instructions
+ defm "" : LMULReadAdvance<"ReadVSALUV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSALUX", 0>;
+ defm "" : LMULReadAdvance<"ReadVAALUV", 0>;
+ defm "" : LMULReadAdvance<"ReadVAALUX", 0>;
+ defm "" : LMULReadAdvance<"ReadVSMulV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSMulX", 0>;
+ defm "" : LMULReadAdvance<"ReadVSShiftV", 0>;
+ defm "" : LMULReadAdvance<"ReadVSShiftX", 0>;
+ defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
+ defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
+
+ // Vector Floating-Point Instructions
+ defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
+ defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
+ defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
+ defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
+ defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
+ defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
+ defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
+ defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
+ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
+ defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
+ defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
+ defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
+ defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
+
+ // Vector Reduction Instructions
+ def : ReadAdvance<ReadVIRedV, 0>;
+ def : ReadAdvance<ReadVIRedV0, 0>;
+ def : ReadAdvance<ReadVIWRedV, 0>;
+ def : ReadAdvance<ReadVIWRedV0, 0>;
+ def : ReadAdvance<ReadVFRedV, 0>;
+ def : ReadAdvance<ReadVFRedV0, 0>;
+ def : ReadAdvance<ReadVFRedOV, 0>;
+ def : ReadAdvance<ReadVFRedOV0, 0>;
+ def : ReadAdvance<ReadVFWRedV, 0>;
+ def : ReadAdvance<ReadVFWRedV0, 0>;
+ def : ReadAdvance<ReadVFWRedOV, 0>;
+ def : ReadAdvance<ReadVFWRedOV0, 0>;
+
+ // Vector Mask Instructions
+ defm "" : LMULReadAdvance<"ReadVMALUV", 0>;
+ defm "" : LMULReadAdvance<"ReadVMPopV", 0>;
+ defm "" : LMULReadAdvance<"ReadVMFFSV", 0>;
+ defm "" : LMULReadAdvance<"ReadVMSFSV", 0>;
+ defm "" : LMULReadAdvance<"ReadVIotaV", 0>;
+
+ // Vector Permutation Instructions
+ def : ReadAdvance<ReadVMovXS, 0>;
+ def : ReadAdvance<ReadVMovSX_V, 0>;
+ def : ReadAdvance<ReadVMovSX_X, 0>;
+ def : ReadAdvance<ReadVMovFS, 0>;
+ def : ReadAdvance<ReadVMovSF_V, 0>;
+ def : ReadAdvance<ReadVMovSF_F, 0>;
+ defm "" : LMULReadAdvance<"ReadVISlideV", 0>;
+ defm "" : LMULReadAdvance<"ReadVISlideX", 0>;
+ defm "" : LMULReadAdvance<"ReadVFSlideV", 0>;
+ defm "" : LMULReadAdvance<"ReadVFSlideF", 0>;
+ defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_data", 0>;
+ defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_index", 0>;
+ defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_data", 0>;
+ defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_index", 0>;
+ defm "" : LMULReadAdvance<"ReadVRGatherVX_data", 0>;
+ defm "" : LMULReadAdvance<"ReadVRGatherVX_index", 0>;
+ defm "" : LMULReadAdvance<"ReadVRGatherVI_data", 0>;
+ defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>;
+ // LMUL Aware
+ def : ReadAdvance<ReadVMov1V, 0>;
+ def : ReadAdvance<ReadVMov2V, 0>;
+ def : ReadAdvance<ReadVMov4V, 0>;
+ def : ReadAdvance<ReadVMov8V, 0>;
+
+ // Others
+ def : ReadAdvance<ReadVMask, 0>;
+ def : ReadAdvance<ReadVPassthru_WorstCase, 0>;
+ foreach mx = SchedMxList in {
+ def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx), 0>;
+ foreach sew = SchedSEWSet<mx>.val in
+ def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx # "_E" # sew), 0>;
}
-}
-// Whole vector register move, vmv<N>.v, not LMUL aware
-let Latency = 1, ReleaseAtCycles = [1] in
- def : WriteRes<WriteVMov1V, [AscalonV]>;
-let Latency = 2, ReleaseAtCycles = [2] in
- def : WriteRes<WriteVMov2V, [AscalonV]>;
-let Latency = 4, ReleaseAtCycles = [4] in
- def : WriteRes<WriteVMov4V, [AscalonV]>;
-let Latency = 8, ReleaseAtCycles = [8] in
- def : WriteRes<WriteVMov8V, [AscalonV]>;
-
-// Vector Integer Arithmetic Instructions
-defm : LMULReadAdvance<"ReadVIALUV", 0>;
-defm : LMULReadAdvance<"ReadVIALUX", 0>;
-defm : LMULReadAdvanceW<"ReadVIWALUV", 0>;
-defm : LMULReadAdvanceW<"ReadVIWALUX", 0>;
-defm : LMULReadAdvance<"ReadVExtV", 0>;
-defm : LMULReadAdvance<"ReadVICALUV", 0>;
-defm : LMULReadAdvance<"ReadVICALUX", 0>;
-defm : LMULReadAdvance<"ReadVShiftV", 0>;
-defm : LMULReadAdvance<"ReadVShiftX", 0>;
-defm : LMULReadAdvanceW<"ReadVNShiftV", 0>;
-defm : LMULReadAdvanceW<"ReadVNShiftX", 0>;
-defm : LMULReadAdvance<"ReadVICmpV", 0>;
-defm : LMULReadAdvance<"ReadVICmpX", 0>;
-defm : LMULReadAdvance<"ReadVIMinMaxV", 0>;
-defm : LMULReadAdvance<"ReadVIMinMaxX", 0>;
-defm : LMULReadAdvance<"ReadVIMulV", 0>;
-defm : LMULReadAdvance<"ReadVIMulX", 0>;
-defm : LMULSEWReadAdvance<"ReadVIDivV", 0>;
-defm : LMULSEWReadAdvance<"ReadVIDivX", 0>;
-defm : LMULReadAdvanceW<"ReadVIWMulV", 0>;
-defm : LMULReadAdvanceW<"ReadVIWMulX", 0>;
-defm : LMULReadAdvance<"ReadVIMulAddV", 0>;
-defm : LMULReadAdvance<"ReadVIMulAddX", 0>;
-defm : LMULReadAdvanceW<"ReadVIWMulAddV", 0>;
-defm : LMULReadAdvanceW<"ReadVIWMulAddX", 0>;
-defm : LMULReadAdvance<"ReadVIMergeV", 0>;
-defm : LMULReadAdvance<"ReadVIMergeX", 0>;
-defm : LMULReadAdvance<"ReadVIMovV", 0>;
-defm : LMULReadAdvance<"ReadVIMovX", 0>;
-
-// Vector Fixed-Point Arithmetic Instructions
-defm "" : LMULReadAdvance<"ReadVSALUV", 0>;
-defm "" : LMULReadAdvance<"ReadVSALUX", 0>;
-defm "" : LMULReadAdvance<"ReadVAALUV", 0>;
-defm "" : LMULReadAdvance<"ReadVAALUX", 0>;
-defm "" : LMULReadAdvance<"ReadVSMulV", 0>;
-defm "" : LMULReadAdvance<"ReadVSMulX", 0>;
-defm "" : LMULReadAdvance<"ReadVSShiftV", 0>;
-defm "" : LMULReadAdvance<"ReadVSShiftX", 0>;
-defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
-defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
-
-// Vector Floating-Point Instructions
-defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
-defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
-defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
-defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
-defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
-defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
-defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
-defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
-defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
-
-// Vector Reduction Instructions
-def : ReadAdvance<ReadVIRedV, 0>;
-def : ReadAdvance<ReadVIRedV0, 0>;
-def : ReadAdvance<ReadVIWRedV, 0>;
-def : ReadAdvance<ReadVIWRedV0, 0>;
-def : ReadAdvance<ReadVFRedV, 0>;
-def : ReadAdvance<ReadVFRedV0, 0>;
-def : ReadAdvance<ReadVFRedOV, 0>;
-def : ReadAdvance<ReadVFRedOV0, 0>;
-def : ReadAdvance<ReadVFWRedV, 0>;
-def : ReadAdvance<ReadVFWRedV0, 0>;
-def : ReadAdvance<ReadVFWRedOV, 0>;
-def : ReadAdvance<ReadVFWRedOV0, 0>;
-
-// Vector Mask Instructions
-defm "" : LMULReadAdvance<"ReadVMALUV", 0>;
-defm "" : LMULReadAdvance<"ReadVMPopV", 0>;
-defm "" : LMULReadAdvance<"ReadVMFFSV", 0>;
-defm "" : LMULReadAdvance<"ReadVMSFSV", 0>;
-defm "" : LMULReadAdvance<"ReadVIotaV", 0>;
-
-// Vector Permutation Instructions
-def : ReadAdvance<ReadVMovXS, 0>;
-def : ReadAdvance<ReadVMovSX_V, 0>;
-def : ReadAdvance<ReadVMovSX_X, 0>;
-def : ReadAdvance<ReadVMovFS, 0>;
-def : ReadAdvance<ReadVMovSF_V, 0>;
-def : ReadAdvance<ReadVMovSF_F, 0>;
-defm "" : LMULReadAdvance<"ReadVISlideV", 0>;
-defm "" : LMULReadAdvance<"ReadVISlideX", 0>;
-defm "" : LMULReadAdvance<"ReadVFSlideV", 0>;
-defm "" : LMULReadAdvance<"ReadVFSlideF", 0>;
-defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_data", 0>;
-defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_index", 0>;
-defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_data", 0>;
-defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_index", 0>;
-defm "" : LMULReadAdvance<"ReadVRGatherVX_data", 0>;
-defm "" : LMULReadAdvance<"ReadVRGatherVX_index", 0>;
-defm "" : LMULReadAdvance<"ReadVRGatherVI_data", 0>;
-defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>;
-// LMUL Aware
-def : ReadAdvance<ReadVMov1V, 0>;
-def : ReadAdvance<ReadVMov2V, 0>;
-def : ReadAdvance<ReadVMov4V, 0>;
-def : ReadAdvance<ReadVMov8V, 0>;
-
-// Others
-def : ReadAdvance<ReadVMask, 0>;
-def : ReadAdvance<ReadVPassthru_WorstCase, 0>;
-foreach mx = SchedMxList in {
- def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx), 0>;
- foreach sew = SchedSEWSet<mx>.val in
- def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx # "_E" # sew), 0>;
+ //===--------------------------------------------------------------------===//
+ // Unsupported extensions
+ defm : UnsupportedSchedQ;
+ defm : UnsupportedSchedZabha;
+ defm : UnsupportedSchedZbc;
+ defm : UnsupportedSchedZbkb;
+ defm : UnsupportedSchedZbkx;
+ defm : UnsupportedSchedZfa;
+ defm : UnsupportedSchedZvk;
+ defm : UnsupportedSchedSFB;
+ defm : UnsupportedSchedXsf;
}
-//===----------------------------------------------------------------------===//
-// Unsupported extensions
-defm : UnsupportedSchedQ;
-defm : UnsupportedSchedZabha;
-defm : UnsupportedSchedZbc;
-defm : UnsupportedSchedZbkb;
-defm : UnsupportedSchedZbkx;
-defm : UnsupportedSchedZfa;
-defm : UnsupportedSchedZvk;
-defm : UnsupportedSchedSFB;
-defm : UnsupportedSchedXsf;
+foreach model = [TTAscalonXModel, TTAscalonXGModel] in {
+ let SchedModel = model in
+ defm model.Name
+ : TTAscalonSchedResources<model.HasReducedFP64VectorThroughput>;
}
diff --git a/llvm/test/tools/llvm-mca/RISCV/tt-ascalon-x/vfp64.s b/llvm/test/tools/llvm-mca/RISCV/tt-ascalon-x/vfp64.s
new file mode 100644
index 0000000000000..218a55ccdcbd6
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/tt-ascalon-x/vfp64.s
@@ -0,0 +1,175 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=tt-ascalon-xg --instruction-tables=full < %s | FileCheck %s
+
+# Vector instructions producing FP64 results have reduced throughput on
+# Ascalon XG: all other vector instructions are unaffected.
+
+vsetvli zero, zero, e32, m1, ta, ma
+vfadd.vv v8, v9, v10
+vfmul.vv v8, v9, v10
+vfmacc.vv v8, v9, v10
+vfwadd.vv v8, v10, v11
+vfwmul.vv v8, v10, v11
+vfwmacc.vv v8, v10, v11
+vfwcvt.f.f.v v8, v10
+vfwcvt.f.x.v v8, v10
+vfsqrt.v v8, v9
+vfmin.vv v8, v9, v10
+vfcvt.f.x.v v8, v9
+vfcvt.f.xu.v v8, v9
+vfmin.vf v8, v9, fa0
+vfmax.vf v8, v9, fa0
+vfsgnj.vv v8, v9, v10
+vfsgnj.vf v8, v9, fa0
+vsetvli zero, zero, e64, m1, ta, ma
+vfadd.vv v8, v9, v10
+vfmul.vv v8, v9, v10
+vfmacc.vv v8, v9, v10
+vfsqrt.v v8, v9
+vfmin.vv v8, v9, v10
+vfcvt.f.x.v v8, v9
+vfcvt.f.xu.v v8, v9
+vfmin.vf v8, v9, fa0
+vfmax.vf v8, v9, fa0
+vfsgnj.vv v8, v9, v10
+vfsgnj.vf v8, v9, fa0
+vfredusum.vs v8, v9, v10
+vsetvli zero, zero, e64, m4, ta, ma
+vfadd.vv v8, v12, v16
+vfmul.vv v8, v12, v16
+vfmacc.vv v8, v12, v16
+vfcvt.f.x.v v8, v12
+vfmin.vv v8, v12, v16
+vfmin.vf v8, v12, fa0
+vfsgnj.vv v8, v12, v16
+vfsgnj.vf v8, v12, fa0
+
+# CHECK: Resources:
+# CHECK-NEXT: [0] - AscalonXGFP:2 AscalonXGFPA, AscalonXGFPB
+# CHECK-NEXT: [1] - AscalonXGFP64V:2
+# CHECK-NEXT: [2] - AscalonXGFPA:1
+# CHECK-NEXT: [3] - AscalonXGFPB:1
+# CHECK-NEXT: [4] - AscalonXGFX:6 AscalonXGFXA, AscalonXGFXB, AscalonXGFXC, AscalonXGFXC, AscalonXGFXD, AscalonXGFXD
+# CHECK-NEXT: [5] - AscalonXGFXA:1
+# CHECK-NEXT: [6] - AscalonXGFXB:1
+# CHECK-NEXT: [7] - AscalonXGFXC:2
+# CHECK-NEXT: [8] - AscalonXGFXD:2
+# CHECK-NEXT: [9] - AscalonXGLS:3
+# CHECK-NEXT: [10] - AscalonXGV:2 AscalonXGVA, AscalonXGVB
+# CHECK-NEXT: [11] - AscalonXGVA:1
+# CHECK-NEXT: [12] - AscalonXGVB:1
+
+# CHECK: Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+# CHECK-NEXT: [7]: Bypass Latency
+# CHECK-NEXT: [8]: Resources (<Name> | <Name>[<ReleaseAtCycle>] | <Name>[<AcquireAtCycle>,<ReleaseAtCycle])
+# CHECK-NEXT: [9]: LLVM Opcode Name
+
+# CHECK: [1] [2] [3] [4] [5] [6] [7] [8] [9] Instructions:
+# CHECK-NEXT: 1 1 0.50 U 0 AscalonXGV VSETVLI vsetvli zero, zero, e32, m1, ta, ma
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFADD_VV vfadd.vv v8, v9, v10
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFMUL_VV vfmul.vv v8, v9, v10
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFMACC_VV vfmacc.vv v8, v9, v10
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFWADD_VV vfwadd.vv v8, v10, v11
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFWMUL_VV vfwmul.vv v8, v10, v11
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFWMACC_VV vfwmacc.vv v8, v10, v11
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFWCVT_F_F_V vfwcvt.f.f.v v8, v10
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP64V[4],AscalonXGFX,AscalonXGFXB,AscalonXGV VFWCVT_F_X_V vfwcvt.f.x.v v8, v10
+# CHECK-NEXT: 1 5 2.50 5 AscalonXGFP,AscalonXGV[5] VFSQRT_V vfsqrt.v v8, v9
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFMIN_VV vfmin.vv v8, v9, v10
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFCVT_F_X_V vfcvt.f.x.v v8, v9
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFCVT_F_XU_V vfcvt.f.xu.v v8, v9
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFMIN_VF vfmin.vf v8, v9, fa0
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFMAX_VF vfmax.vf v8, v9, fa0
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFSGNJ_VV vfsgnj.vv v8, v9, v10
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFP,AscalonXGV VFSGNJ_VF vfsgnj.vf v8, v9, fa0
+# CHECK-NEXT: 1 1 0.50 U 0 AscalonXGV VSETVLI vsetvli zero, zero, e64, m1, ta, ma
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFADD_VV vfadd.vv v8, v9, v10
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFMUL_VV vfmul.vv v8, v9, v10
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFMACC_VV vfmacc.vv v8, v9, v10
+# CHECK-NEXT: 1 8 4.00 8 AscalonXGFP,AscalonXGV[8] VFSQRT_V vfsqrt.v v8, v9
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFMIN_VV vfmin.vv v8, v9, v10
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFCVT_F_X_V vfcvt.f.x.v v8, v9
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFCVT_F_XU_V vfcvt.f.xu.v v8, v9
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFMIN_VF vfmin.vf v8, v9, fa0
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFMAX_VF vfmax.vf v8, v9, fa0
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFSGNJ_VV vfsgnj.vv v8, v9, v10
+# CHECK-NEXT: 1 9 2.00 9 AscalonXGFP,AscalonXGFP64V[4],AscalonXGV VFSGNJ_VF vfsgnj.vf v8, v9, fa0
+# CHECK-NEXT: 1 1 0.50 1 AscalonXGFX,AscalonXGV VFREDUSUM_VS vfredusum.vs v8, v9, v10
+# CHECK-NEXT: 1 1 0.50 U 0 AscalonXGV VSETVLI vsetvli zero, zero, e64, m4, ta, ma
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFADD_VV vfadd.vv v8, v12, v16
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFMUL_VV vfmul.vv v8, v12, v16
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFMACC_VV vfmacc.vv v8, v12, v16
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFCVT_F_X_V vfcvt.f.x.v v8, v12
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFMIN_VV vfmin.vv v8, v12, v16
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFMIN_VF vfmin.vf v8, v12, fa0
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFSGNJ_VV vfsgnj.vv v8, v12, v16
+# CHECK-NEXT: 1 12 8.00 12 AscalonXGFP,AscalonXGFP64V[16],AscalonXGV[4] VFSGNJ_VF vfsgnj.vf v8, v12, fa0
+
+# CHECK: Resources:
+# CHECK-NEXT: [0.0] - AscalonXGFP64V
+# CHECK-NEXT: [0.1] - AscalonXGFP64V
+# CHECK-NEXT: [1] - AscalonXGFPA
+# CHECK-NEXT: [2] - AscalonXGFPB
+# CHECK-NEXT: [3] - AscalonXGFXA
+# CHECK-NEXT: [4] - AscalonXGFXB
+# CHECK-NEXT: [5.0] - AscalonXGFXC
+# CHECK-NEXT: [5.1] - AscalonXGFXC
+# CHECK-NEXT: [6.0] - AscalonXGFXD
+# CHECK-NEXT: [6.1] - AscalonXGFXD
+# CHECK-NEXT: [7.0] - AscalonXGLS
+# CHECK-NEXT: [7.1] - AscalonXGLS
+# CHECK-NEXT: [7.2] - AscalonXGLS
+# CHECK-NEXT: [8] - AscalonXGVA
+# CHECK-NEXT: [9] - AscalonXGVB
+
+# CHECK: Resource pressure per iteration:
+# CHECK-NEXT: [0.0] [0.1] [1] [2] [3] [4] [5.0] [5.1] [6.0] [6.1] [7.0] [7.1] [7.2] [8] [9]
+# CHECK-NEXT: 94.00 94.00 17.00 17.00 0.17 1.17 0.17 0.17 0.17 0.17 - - - 37.00 37.00
+
+# CHECK: Resource pressure by instruction:
+# CHECK-NEXT: [0.0] [0.1] [1] [2] [3] [4] [5.0] [5.1] [6.0] [6.1] [7.0] [7.1] [7.2] [8] [9] Instructions:
+# CHECK-NEXT: - - - - - - - - - - - - - 0.50 0.50 vsetvli zero, zero, e32, m1, ta, ma
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfadd.vv v8, v9, v10
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmul.vv v8, v9, v10
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmacc.vv v8, v9, v10
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfwadd.vv v8, v10, v11
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfwmul.vv v8, v10, v11
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfwmacc.vv v8, v10, v11
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfwcvt.f.f.v v8, v10
+# CHECK-NEXT: 2.00 2.00 - - - 1.00 - - - - - - - 0.50 0.50 vfwcvt.f.x.v v8, v10
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 2.50 2.50 vfsqrt.v v8, v9
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmin.vv v8, v9, v10
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfcvt.f.x.v v8, v9
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfcvt.f.xu.v v8, v9
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmin.vf v8, v9, fa0
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmax.vf v8, v9, fa0
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfsgnj.vv v8, v9, v10
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 0.50 0.50 vfsgnj.vf v8, v9, fa0
+# CHECK-NEXT: - - - - - - - - - - - - - 0.50 0.50 vsetvli zero, zero, e64, m1, ta, ma
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfadd.vv v8, v9, v10
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmul.vv v8, v9, v10
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmacc.vv v8, v9, v10
+# CHECK-NEXT: - - 0.50 0.50 - - - - - - - - - 4.00 4.00 vfsqrt.v v8, v9
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmin.vv v8, v9, v10
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfcvt.f.x.v v8, v9
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfcvt.f.xu.v v8, v9
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmin.vf v8, v9, fa0
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfmax.vf v8, v9, fa0
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfsgnj.vv v8, v9, v10
+# CHECK-NEXT: 2.00 2.00 0.50 0.50 - - - - - - - - - 0.50 0.50 vfsgnj.vf v8, v9, fa0
+# CHECK-NEXT: - - - - 0.17 0.17 0.17 0.17 0.17 0.17 - - - 0.50 0.50 vfredusum.vs v8, v9, v10
+# CHECK-NEXT: - - - - - - - - - - - - - 0.50 0.50 vsetvli zero, zero, e64, m4, ta, ma
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfadd.vv v8, v12, v16
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfmul.vv v8, v12, v16
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfmacc.vv v8, v12, v16
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfcvt.f.x.v v8, v12
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfmin.vv v8, v12, v16
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfmin.vf v8, v12, fa0
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfsgnj.vv v8, v12, v16
+# CHECK-NEXT: 8.00 8.00 0.50 0.50 - - - - - - - - - 2.00 2.00 vfsgnj.vf v8, v12, fa0
>From d6900ca3938aaf387ae49817dc82f9d087d4eeca Mon Sep 17 00:00:00 2001
From: Rose Hudson <rose.hudson at sony.com>
Date: Wed, 7 Oct 2026 11:28:22 +0100
Subject: [PATCH 32/52] [Support] allow placement new with extended alignment
(#228507)
llvm/unittests/Support/AllocatorTest.cpp was failing under UBSan on
Windows because we were trying to allocate a 16-byte aligned type and
BumpPtrAllocatorImpl only guaranteed alignof(std::max_align_t). operator
new should guarantee __STDCPP_DEFAULT_NEW_ALIGNMENT__, not
alignof(std::max_align_t). The former is stronger than the latter on
some platforms (16 vs 8 on Windows, arm64 macOS, and s390x Linux)
meaning BumpPtrAllocatorImpl could misalign 16-byte allocations there
where the default allocator would not.
If placement new is used for a type that's aligned stronger than
__STDCPP_DEFAULT_NEW_ALIGNMENT__ and an overload of operator new() with
an align_val_t argument is available, that overload will be used. If the
overload is not available, the allocation can be misaligned regardless
of platform. Add the overload and correct the bound in the existing
operator new() implementation so that we can allocate types with any
alignment using placement new. Also add explicit alignment checks in the
test so that this behaviour is covered outside of UBSan.
---
llvm/include/llvm/Support/Allocator.h | 15 +++++++++++++--
llvm/unittests/Support/AllocatorTest.cpp | 8 ++++++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Support/Allocator.h b/llvm/include/llvm/Support/Allocator.h
index 79e2040fcd381..63b18ac077da2 100644
--- a/llvm/include/llvm/Support/Allocator.h
+++ b/llvm/include/llvm/Support/Allocator.h
@@ -28,6 +28,7 @@
#include <cstddef>
#include <cstdint>
#include <iterator>
+#include <new>
#include <optional>
#include <utility>
@@ -476,11 +477,21 @@ operator new(size_t Size,
llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize, SizeThreshold,
GrowthDelay, MinAlign> &Allocator) {
// alignof(T) is unknown but Size%alignof(T)==0, so the lowest set bit of Size
- // bounds it (capped at alignof(max_align_t)).
- auto S = Size | alignof(std::max_align_t);
+ // bounds it. Types with alignof(T) > __STDCPP_DEFAULT_NEW_ALIGNMENT__ use the
+ // align_val_t overload below.
+ auto S = Size | __STDCPP_DEFAULT_NEW_ALIGNMENT__;
return Allocator.Allocate(Size, llvm::Align(S & -S));
}
+template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold,
+ size_t GrowthDelay, size_t MinAlign>
+void *
+operator new(size_t Size, std::align_val_t Alignment,
+ llvm::BumpPtrAllocatorImpl<AllocatorT, SlabSize, SizeThreshold,
+ GrowthDelay, MinAlign> &Allocator) {
+ return Allocator.Allocate(Size, llvm::Align(static_cast<size_t>(Alignment)));
+}
+
template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold,
size_t GrowthDelay, size_t MinAlign>
void operator delete(
diff --git a/llvm/unittests/Support/AllocatorTest.cpp b/llvm/unittests/Support/AllocatorTest.cpp
index 2ecceb625a35d..dbc715b6abb6b 100644
--- a/llvm/unittests/Support/AllocatorTest.cpp
+++ b/llvm/unittests/Support/AllocatorTest.cpp
@@ -109,6 +109,9 @@ TEST(AllocatorTest, TestPlacementNew) {
struct S48 {
uint64_t X[6];
};
+ struct alignas(64) S64 {
+ uint64_t X[8];
+ };
BumpPtrAllocator Alloc;
Alloc.setRedZoneSize(0);
auto *A0 = new (Alloc) S24;
@@ -116,10 +119,15 @@ TEST(AllocatorTest, TestPlacementNew) {
EXPECT_EQ(uintptr_t(A0) + sizeof(S24), uintptr_t(A1));
auto *B0 = new (Alloc) S32;
auto *B1 = new (Alloc) S32;
+ EXPECT_EQ(uintptr_t(B0) % alignof(S32), 0u);
EXPECT_EQ(uintptr_t(B0) + sizeof(S32), uintptr_t(B1));
auto *C0 = new (Alloc) S48;
auto *C1 = new (Alloc) S48;
EXPECT_EQ(uintptr_t(C0) + sizeof(S48), uintptr_t(C1));
+ auto *D0 = new (Alloc) S64;
+ auto *D1 = new (Alloc) S64;
+ EXPECT_EQ(uintptr_t(D0) % alignof(S64), 0u);
+ EXPECT_EQ(uintptr_t(D0) + sizeof(S64), uintptr_t(D1));
}
// Test zero-sized allocations.
>From 306576cfef9311ff642b9e9e5e069a4e1516aa52 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 7 Oct 2026 12:29:26 +0200
Subject: [PATCH 33/52] [TwoAddressInstruction] Don't mark a use undef if it
reads a defined lane (#229554)
When a REG_SEQUENCE with undef sources is lowered with live intervals
available a use that reads the defined lane together with the undef lane
is marked undef as a whole. The copy defining that lane becomes dead and
the use reads an arbitrary register. AMDGPU started to hit this through
packed FP32 reads of a 64-bit subregister with one defined half since
#225174 .
Mark a use undef only when every lane it reads is undef.
---
.../lib/CodeGen/TwoAddressInstructionPass.cpp | 2 +-
...arly-lis-two-address-regsequence-undef.mir | 28 +++++++++++++++++++
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 13 ++++-----
3 files changed, 35 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index aa1116a2e6f61..5f6a165110e51 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -2065,7 +2065,7 @@ void TwoAddressInstructionImpl::eliminateRegSequence(
if (DefVN != VN)
continue;
LaneBitmask LaneMask = TRI->getSubRegIndexLaneMask(SubReg);
- if ((UndefLanes & LaneMask).any())
+ if ((LaneMask & UndefLanes) == LaneMask)
UseOp.setIsUndef(true);
}
LIS->removeInterval(DstReg);
diff --git a/llvm/test/CodeGen/AMDGPU/early-lis-two-address-regsequence-undef.mir b/llvm/test/CodeGen/AMDGPU/early-lis-two-address-regsequence-undef.mir
index f8033488e31ab..75e7cc15e6081 100644
--- a/llvm/test/CodeGen/AMDGPU/early-lis-two-address-regsequence-undef.mir
+++ b/llvm/test/CodeGen/AMDGPU/early-lis-two-address-regsequence-undef.mir
@@ -40,3 +40,31 @@ body: |
%3:vgpr_32 = nofpexcept V_FMAC_F32_e32 %1.sub1, %1.sub1, %1.sub1, implicit $mode, implicit $exec
SI_RETURN_TO_EPILOG implicit %3
...
+
+# The first add reads only the defined lane of its second source, through
+# a 64-bit read that also covers an undef lane, so the read stays defined.
+# The second add reads undef lanes only, so its read becomes undef.
+
+---
+name: read_of_defined_and_undef_lanes
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0, $vgpr2_vgpr3
+
+ ; GFX90A-LABEL: name: read_of_defined_and_undef_lanes
+ ; GFX90A: liveins: $vgpr0, $vgpr2_vgpr3
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX90A-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY $vgpr2_vgpr3
+ ; GFX90A-NEXT: undef [[COPY2:%[0-9]+]].sub0:vreg_128_align2 = COPY [[COPY]]
+ ; GFX90A-NEXT: [[V_PK_ADD_F32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY1]], 0, [[COPY2]].sub0_sub1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: [[V_PK_ADD_F32_1:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY1]], 0, undef [[COPY2]].sub2_sub3, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX90A-NEXT: SI_RETURN_TO_EPILOG implicit [[V_PK_ADD_F32_]], implicit [[V_PK_ADD_F32_1]]
+ %0:vgpr_32 = COPY $vgpr0
+ %1:vreg_64_align2 = COPY $vgpr2_vgpr3
+ %2:vreg_128_align2 = REG_SEQUENCE %0, %subreg.sub0, undef %3:vgpr_32, %subreg.sub1, undef %4:vgpr_32, %subreg.sub2, undef %5:vgpr_32, %subreg.sub3
+ %6:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, %1, 0, %2.sub0_sub1, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %7:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, %1, 0, %2.sub2_sub3, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ SI_RETURN_TO_EPILOG implicit %6, implicit %7
+...
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index 8c08f149e3055..4b04359796445 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -323,22 +323,21 @@ define amdgpu_kernel void @scalar_to_vector_v4f16() {
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: buffer_load_d16_u8 v0, off, s[0:3], 0
-; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: s_and_b32 s1, s0, 0xff00
; GFX11-NEXT: s_bfe_u32 s4, s0, 0x80008
; GFX11-NEXT: s_and_b32 s0, s0, 0xffff
; GFX11-NEXT: s_or_b32 s1, s4, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshl_b32 s4, s1, 16
; GFX11-NEXT: s_and_b32 s1, s1, 0xffff
; GFX11-NEXT: s_or_b32 s0, s0, s4
; GFX11-NEXT: s_or_b32 s1, s1, s4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v2, v1, v0
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: ; kill: def $vgpr2_lo16 killed $vgpr2 killed $exec
; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-NEXT: s_endpgm
bb:
>From 01c1d2c7ac2ae4a20bb176324619d54be0720ac5 Mon Sep 17 00:00:00 2001
From: Nathan Corbyn <n_corbyn at apple.com>
Date: Wed, 7 Oct 2026 11:37:58 +0100
Subject: [PATCH 34/52] [ScheduleDAG](NFC) Allocate worklist once in
`ScheduleDAGTopologicalSort` (#179256)
At the moment every reachability query that falls outside of the cache
allocates a large worklist. This brings a [modest compile time
improvement on
CTMark](https://llvm-compile-time-tracker.com/compare.php?from=94eb65e222efccf36ac36aec948d1979186bdcef&to=b7e3dacdd78aa9e1b05b5ca882b09ae885fd04dc&stat=instructions:u)
and bigger wins in heavier TUs.
---
llvm/include/llvm/CodeGen/ScheduleDAG.h | 3 +++
llvm/lib/CodeGen/ScheduleDAG.cpp | 13 ++++++-------
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ScheduleDAG.h b/llvm/include/llvm/CodeGen/ScheduleDAG.h
index 0617ef3c954b7..aab01ce42e7f3 100644
--- a/llvm/include/llvm/CodeGen/ScheduleDAG.h
+++ b/llvm/include/llvm/CodeGen/ScheduleDAG.h
@@ -749,6 +749,9 @@ class TargetRegisterInfo;
std::vector<int> Node2Index;
/// a set of nodes visited during a DFS traversal.
BitVector Visited;
+ /// A worklist for use during traversals. Retained after traversals so must
+ /// be cleared before use.
+ std::vector<const SUnit *> WorkList;
/// Cache of reachability queries. {A, B} -> true if B is reachable from A.
/// The keys are SUnit NodeNums.
DenseMap<std::pair<int, int>, bool> Reachable;
diff --git a/llvm/lib/CodeGen/ScheduleDAG.cpp b/llvm/lib/CodeGen/ScheduleDAG.cpp
index 7ac5a816b1368..db0fcea4b379e 100644
--- a/llvm/lib/CodeGen/ScheduleDAG.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAG.cpp
@@ -476,12 +476,13 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() {
Reachable.clear();
unsigned DAGSize = SUnits.size();
- std::vector<SUnit*> WorkList;
- WorkList.reserve(DAGSize);
Index2Node.resize(DAGSize);
Node2Index.resize(DAGSize);
+ WorkList.reserve(DAGSize);
+ WorkList.clear();
+
// Initialize the data structures.
if (ExitSU)
WorkList.push_back(ExitSU);
@@ -501,7 +502,7 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() {
int Id = DAGSize;
while (!WorkList.empty()) {
- SUnit *SU = WorkList.back();
+ const SUnit *SU = WorkList.back();
WorkList.pop_back();
if (SU->NodeNum < DAGSize)
Allocate(SU->NodeNum, --Id);
@@ -578,8 +579,7 @@ void ScheduleDAGTopologicalSort::RemovePred(SUnit *M, SUnit *N) {
void ScheduleDAGTopologicalSort::DFS(const SUnit *SU, int UpperBound,
bool &HasLoop) {
- std::vector<const SUnit*> WorkList;
- WorkList.reserve(SUnits.size());
+ WorkList.clear();
WorkList.push_back(SU);
do {
@@ -606,7 +606,6 @@ void ScheduleDAGTopologicalSort::DFS(const SUnit *SU, int UpperBound,
std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
const SUnit &TargetSU,
bool &Success) {
- std::vector<const SUnit*> WorkList;
int LowerBound = Node2Index[StartSU.NodeNum];
int UpperBound = Node2Index[TargetSU.NodeNum];
bool Found = false;
@@ -618,7 +617,7 @@ std::vector<int> ScheduleDAGTopologicalSort::GetSubGraph(const SUnit &StartSU,
return Nodes;
}
- WorkList.reserve(SUnits.size());
+ WorkList.clear();
Visited.reset();
// Starting from StartSU, visit all successors up
>From 448badfab126bdee384360a9355fa707068203c2 Mon Sep 17 00:00:00 2001
From: Andre Sun <andsun9999 at gmail.com>
Date: Wed, 7 Oct 2026 06:40:08 -0400
Subject: [PATCH 35/52] [clang-tidy] Fix redundant-parentheses diagnosing
synthetic parentheses (#228666)
Skip synthetic parentheses with identical source locations in
`readability-redundant-parentheses`. This also stops
`__builtin_dump_struct(&s, printf)` from being rewritten to `(&s,
printf)`
and stops variables being deleted from OpenMP `linear` clauses.
Fixes #224705.
---------
Co-authored-by: Baranov Victor <bar.victor.2002 at gmail.com>
---
.../readability/RedundantParenthesesCheck.cpp | 5 +++++
clang-tools-extra/docs/ReleaseNotes.md | 10 ++++++++--
.../readability/redundant-parentheses-openmp.cpp | 8 ++++++++
.../checkers/readability/redundant-parentheses.cpp | 14 ++++++++++++++
4 files changed, 35 insertions(+), 2 deletions(-)
create mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses-openmp.cpp
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantParenthesesCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantParenthesesCheck.cpp
index bc1d10586da8a..09dd53e83b547 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantParenthesesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantParenthesesCheck.cpp
@@ -34,6 +34,10 @@ AST_MATCHER(ParenExpr, isInMacro) {
E->getBeginLoc().isMacroID() || E->getEndLoc().isMacroID();
}
+AST_MATCHER(ParenExpr, isWritten) {
+ return Node.getLParen() != Node.getRParen();
+}
+
AST_MATCHER(TypeLoc, isTypeOfExprTypeLoc) {
return !Node.getUnqualifiedLoc().getAs<TypeOfExprTypeLoc>().isNull();
}
@@ -82,6 +86,7 @@ void RedundantParenthesesCheck::registerMatchers(MatchFinder *Finder) {
callExpr(unless(cxxOperatorCallExpr(
unless(hasAnyOperatorName("()", "[]"))))),
arraySubscriptExpr())),
+ isWritten(),
unless(anyOf(isInMacro(),
// sizeof(...) is common used.
hasParent(unaryExprOrTypeTraitExpr()),
diff --git a/clang-tools-extra/docs/ReleaseNotes.md b/clang-tools-extra/docs/ReleaseNotes.md
index 20b612509d7f7..b973c3f1f4a0e 100644
--- a/clang-tools-extra/docs/ReleaseNotes.md
+++ b/clang-tools-extra/docs/ReleaseNotes.md
@@ -328,8 +328,14 @@ infrastructure are described first, followed by tool-specific sections.
`atomic_compare_exchange_strong()`.
- Improved {doc}`readability-redundant-parentheses
- <clang-tidy/checks/readability/redundant-parentheses>` check by fixing a false
- positive on the required parentheses of `typeof` and `typeof_unqual` operands.
+ <clang-tidy/checks/readability/redundant-parentheses>` check:
+
+ - Fixed a false positive on the required parentheses of `typeof` and
+ `typeof_unqual` operands.
+
+ - Fixed false positives and incorrect fixes caused by synthetic parentheses
+ in reference non-type template parameter uses, `__builtin_dump_struct` calls,
+ and OpenMP `linear` clauses.
- Fixed {doc}`readability-simplify-boolean-expr
<clang-tidy/checks/readability/simplify-boolean-expr>` producing invalid
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses-openmp.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses-openmp.cpp
new file mode 100644
index 0000000000000..98e9c36fb0a86
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses-openmp.cpp
@@ -0,0 +1,8 @@
+// RUN: %check_clang_tidy %s readability-redundant-parentheses %t -- -- -fopenmp=libomp
+
+void linearClause(int *a, int n) {
+ int i = 0;
+#pragma omp simd linear(i)
+ for (int k = 0; k < n; ++k)
+ a[k] = i;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses.cpp
index c6275be9f2498..12286cd513f85 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-parentheses.cpp
@@ -74,6 +74,20 @@ void ignoreStdMaxMin() {
(std::min)(1,2);
}
+int global = 0;
+template <int &R> int &referenceTemplateArgument() {
+ return R;
+}
+int &useReferenceTemplateArgument() {
+ return referenceTemplateArgument<global>();
+}
+
+extern "C" int printf(const char *, ...);
+struct Dumped { int x; };
+void dumpStruct(Dumped *d) {
+ __builtin_dump_struct(d, printf);
+}
+
struct Foo
{
bool x;
>From db9b95948d4ef3b119911b8991496c821455c3e3 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 12:48:29 +0200
Subject: [PATCH 36/52] RuntimeLibcalls: Dedup dispatch calls to the same
library function (#229561)
Referencing a provider both as a base opt-out and as a same-name re-add
variant emitted the same setAvailableLibFuncs_<name> call twice. Collapse
references with the same (Name, FuncSuffix) into one dispatch, and drop the
unused dispatch-side exclusion list.
Co-authored-by: Claude (Opus 4.8) <noreply at anthropic.com>
---
llvm/include/llvm/IR/RuntimeLibcallsImpl.td | 7 ++--
.../TableGen/Basic/RuntimeLibcallsEmitter.cpp | 32 ++++++++-----------
2 files changed, 17 insertions(+), 22 deletions(-)
diff --git a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
index c8255913a9d21..5828f7d4695fb 100644
--- a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
+++ b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
@@ -133,9 +133,10 @@ class LibcallLibrary<string name, dag impls,
}
// Reference a shared LibcallLibrary from a SystemRuntimeLibrary, dropping the
-// impls in `exclude`. Lets an override target pull a provider library (e.g.
-// compiler-rt) but keep its own versions of a few entries.
-class LibraryRef<LibcallLibrary lib, list<RuntimeLibcallImpl> exclude> {
+// impls in `exclude`, so a target can pull a provider (e.g. compiler-rt) but
+// keep its own versions of a few entries. A bare LibcallLibrary member is
+// equivalent to LibraryRef<Lib>.
+class LibraryRef<LibcallLibrary lib, list<RuntimeLibcallImpl> exclude = []> {
LibcallLibrary Library = lib;
list<RuntimeLibcallImpl> Exclude = exclude;
}
diff --git a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
index 6dfe4b9b0ac50..f4b6a10198b1a 100644
--- a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
@@ -846,44 +846,38 @@ void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
}
}
- // Split the top-level member list into named LibcallLibrary references
- // (dispatched to their own setAvailableLibFuncs_<name> under an
- // isLibraryAvailable guard) and the remaining bare impl / LibcallImpls
- // members (emitted inline below). A LibraryRef also records impls to drop.
- // Name is the linker library (the isLibraryAvailable guard); FuncSuffix is
- // the function suffix, which differs from Name only when isolated.
+ // Split the member list into library references, each dispatched to
+ // setAvailableLibFuncs_<FuncSuffix> under an isLibraryAvailable(Name)
+ // guard, and the remaining members, emitted inline below. LibraryRef
+ // exclusions are applied inside the library function.
struct DispatchLib {
StringRef Name;
StringRef FuncSuffix;
- std::vector<const RuntimeLibcallImpl *> Exclude;
};
const DagInit *MemberDag =
R->getValueAsDef("MemberList")->getValueAsDag("MemberList");
SmallVector<DispatchLib, 4> DispatchLibs;
SmallVector<const Init *, 16> InlineArgs;
SmallVector<const StringInit *, 16> InlineArgNames;
+ // A provider referenced both as a base opt-out and a same-name re-add
+ // variant resolves to the same function; dispatch it once.
+ DenseSet<std::pair<StringRef, StringRef>> SeenDispatch;
+ auto AddDispatch = [&](StringRef Name, StringRef FuncSuffix) {
+ if (SeenDispatch.insert({Name, FuncSuffix}).second)
+ DispatchLibs.push_back({Name, FuncSuffix});
+ };
for (auto [Arg, ArgName] :
zip_equal(MemberDag->getArgs(), MemberDag->getArgNames())) {
if (const auto *DI = dyn_cast<DefInit>(Arg)) {
const Record *Def = DI->getDef();
if (Def->isSubClassOf("LibcallLibrary")) {
- DispatchLibs.push_back(
- {Def->getValueAsString("LibraryName"), libFuncKey(Def), {}});
+ AddDispatch(Def->getValueAsString("LibraryName"), libFuncKey(Def));
continue;
}
if (Def->isSubClassOf("LibraryRef")) {
const Record *Lib = Def->getValueAsDef("Library");
- DispatchLib DL{
- Lib->getValueAsString("LibraryName"), libFuncKey(Lib), {}};
- for (const Record *ExcludeRec :
- Def->getValueAsListOfDefs("Exclude")) {
- if (const RuntimeLibcallImpl *Impl =
- Libcalls.getRuntimeLibcallImpl(ExcludeRec))
- DL.Exclude.push_back(Impl);
- }
-
- DispatchLibs.push_back(std::move(DL));
+ AddDispatch(Lib->getValueAsString("LibraryName"), libFuncKey(Lib));
continue;
}
}
>From 2c8899609919bbc8afd59b69424ea8487f9396db Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 7 Oct 2026 12:49:21 +0200
Subject: [PATCH 37/52] [SPIR-V][NewPM] Port SPIRVModuleAnalysis (#227376)
This step enables us with e2e compilation with new pass manager
---
llvm/lib/Target/SPIRV/SPIRV.h | 2 +-
llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp | 20 ++-
.../Target/SPIRV/SPIRVCodeGenPassBuilder.cpp | 7 +
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp | 122 +++++++++++-------
llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h | 45 +++++--
llvm/lib/Target/SPIRV/SPIRVPassRegistry.def | 6 +
llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 2 +-
llvm/test/CodeGen/SPIRV/llc-pipeline.ll | 15 +--
8 files changed, 149 insertions(+), 70 deletions(-)
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index ccc7aa52ec52c..1c8e881c307b7 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -178,7 +178,7 @@ createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
const SPIRVSubtarget &Subtarget,
const RegisterBankInfo &RBI);
-void initializeSPIRVModuleAnalysisPass(PassRegistry &);
+void initializeSPIRVModuleAnalysisWrapperPassPass(PassRegistry &);
void initializeSPIRVAsmPrinterPass(PassRegistry &);
void initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PassRegistry &);
void initializeSPIRVPreLegalizerLegacyPass(PassRegistry &);
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 11f8674730a2e..685a4f553ca2a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -74,7 +74,11 @@ class SPIRVAsmPrinter : public AsmPrinter {
explicit SPIRVAsmPrinter(TargetMachine &TM,
std::unique_ptr<MCStreamer> Streamer)
: AsmPrinter(TM, std::move(Streamer), ID), ModuleSectionsEmitted(false),
- ST(nullptr), TII(nullptr), MAI(nullptr) {}
+ ST(nullptr), TII(nullptr), MAI(nullptr) {
+ GetMAI = [this] {
+ return &getAnalysis<SPIRVModuleAnalysisWrapperPass>().MAI;
+ };
+ }
static char ID;
bool ModuleSectionsEmitted;
const SPIRVSubtarget *ST;
@@ -129,6 +133,7 @@ class SPIRVAsmPrinter : public AsmPrinter {
void getAnalysisUsage(AnalysisUsage &AU) const override;
SPIRV::ModuleAnalysisInfo *MAI;
+ std::function<SPIRV::ModuleAnalysisInfo *()> GetMAI;
// Non-owning pointer to the NSDI handler registered via addAsmPrinterHandler.
// The handler's lifetime is managed by AsmPrinter (the base class of this
@@ -143,8 +148,8 @@ class SPIRVAsmPrinter : public AsmPrinter {
} // namespace
void SPIRVAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
- AU.addRequired<SPIRVModuleAnalysis>();
- AU.addPreserved<SPIRVModuleAnalysis>();
+ AU.addRequired<SPIRVModuleAnalysisWrapperPass>();
+ AU.addPreserved<SPIRVModuleAnalysisWrapperPass>();
AsmPrinter::getAnalysisUsage(AU);
}
@@ -890,7 +895,7 @@ void SPIRVAsmPrinter::outputModuleSections() {
// Get the global subtarget to output module-level info.
ST = static_cast<const SPIRVTargetMachine &>(TM).getSubtargetImpl();
TII = ST->getInstrInfo();
- MAI = &getAnalysis<SPIRVModuleAnalysis>().MAI;
+ MAI = GetMAI();
assert(ST && TII && MAI && M && "Module analysis is required");
if (!AuxDataHandler) {
@@ -998,6 +1003,10 @@ SPIRVAsmPrinterPass::run(MachineFunction &MF,
.getCachedResult<AsmPrinterAnalysis>(*MF.getFunction().getParent())
->getPrinter());
setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
+ AsmPrinter.GetMAI = [&MFAM, &MF] {
+ return MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(MF)
+ .getCachedResult<SPIRVModuleAnalysis>(*MF.getFunction().getParent());
+ };
AsmPrinter.runOnMachineFunction(MF);
return PreservedAnalyses::all();
}
@@ -1007,6 +1016,9 @@ PreservedAnalyses SPIRVAsmPrinterEndPass::run(Module &M,
SPIRVAsmPrinter &AsmPrinter = static_cast<SPIRVAsmPrinter &>(
MAM.getResult<AsmPrinterAnalysis>(M).getPrinter());
setupModuleAsmPrinter(M, MAM, AsmPrinter);
+ AsmPrinter.GetMAI = [&MAM, &M] {
+ return MAM.getCachedResult<SPIRVModuleAnalysis>(M);
+ };
AsmPrinter.doFinalization(M);
return PreservedAnalyses::all();
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVCodeGenPassBuilder.cpp b/llvm/lib/Target/SPIRV/SPIRVCodeGenPassBuilder.cpp
index 4feb8ee2a4699..6481a20217342 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCodeGenPassBuilder.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCodeGenPassBuilder.cpp
@@ -9,6 +9,7 @@
#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
#include "SPIRV.h"
#include "SPIRVAsmPrinter.h"
+#include "SPIRVModuleAnalysis.h"
#include "SPIRVTargetMachine.h"
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/BranchFoldingPass.h"
@@ -193,6 +194,12 @@ void SPIRVCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
}
void SPIRVCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
+ // SPIRVModuleAnalysis needs every MachineFunction before printing starts.
+ if (TargetPassConfig::willCompleteCodeGenPipeline()) {
+ flushFPMsToMPM(PMW);
+ addModulePass(RequireAnalysisPass<SPIRVModuleAnalysis, Module>(), PMW,
+ /*Force=*/true);
+ }
addMachineFunctionPass(SPIRVAsmPrinterPass(), PMW);
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index bbf457ea78224..0ba4174615b56 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -26,6 +26,7 @@
#include "SPIRVTargetMachine.h"
#include "SPIRVUtils.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
@@ -51,10 +52,10 @@ struct AvoidCapabilitiesSet {
AvoidCapabilitiesSet() { S.insert_range(AvoidCapabilities); }
};
-char llvm::SPIRVModuleAnalysis::ID = 0;
+char llvm::SPIRVModuleAnalysisWrapperPass::ID = 0;
-INITIALIZE_PASS(SPIRVModuleAnalysis, DEBUG_TYPE, "SPIRV module analysis", true,
- true)
+INITIALIZE_PASS(SPIRVModuleAnalysisWrapperPass, DEBUG_TYPE,
+ "SPIRV module analysis", true, true)
static void reportUnsupported(const MachineInstr &MI, const char *Msg) {
const Function &Func = MI.getMF()->getFunction();
@@ -141,7 +142,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
return {false, {}, {}, VersionTuple(), VersionTuple()};
}
-void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
+void SPIRVModuleAnalysisImpl::setBaseInfo(const Module &M) {
MAI.MaxID = 0;
for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
MAI.MS[i].clear();
@@ -348,8 +349,8 @@ static unsigned getInvokeOperandIdx(unsigned Opcode) {
}
}
-bool SPIRVModuleAnalysis::isDeclSection(const MachineRegisterInfo &MRI,
- const MachineInstr &MI) {
+bool SPIRVModuleAnalysisImpl::isDeclSection(const MachineRegisterInfo &MRI,
+ const MachineInstr &MI) {
unsigned Opcode = MI.getOpcode();
switch (Opcode) {
case SPIRV::OpTypeForwardPointer:
@@ -402,7 +403,7 @@ bool SPIRVModuleAnalysis::isDeclSection(const MachineRegisterInfo &MRI,
// FunPtrOp is the MachineOperand previously recorded via
// SPIRVGlobalRegistry::recordFunctionPointer, identifying which Function
// this placeholder refers to.
-void SPIRVModuleAnalysis::visitFunPtrUse(
+void SPIRVModuleAnalysisImpl::visitFunPtrUse(
Register OpReg, const MachineOperand *FunPtrOp,
InstrGRegsMap &SignatureToGReg,
std::map<const Value *, unsigned> &GlobalToGReg,
@@ -429,7 +430,7 @@ void SPIRVModuleAnalysis::visitFunPtrUse(
// Depth first recursive traversal of dependencies. Repeated visits are guarded
// by MAI.hasRegisterAlias().
-void SPIRVModuleAnalysis::visitDecl(
+void SPIRVModuleAnalysisImpl::visitDecl(
const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
const MachineInstr &MI) {
@@ -511,7 +512,7 @@ void SPIRVModuleAnalysis::visitDecl(
MAI.setSkipEmission(&MI);
}
-MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(
+MCRegister SPIRVModuleAnalysisImpl::handleFunctionOrParameter(
const MachineFunction *MF, const MachineInstr &MI,
std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());
@@ -531,9 +532,8 @@ MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(
return GReg;
}
-MCRegister
-SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,
- InstrGRegsMap &SignatureToGReg) {
+MCRegister SPIRVModuleAnalysisImpl::handleTypeDeclOrConstant(
+ const MachineInstr &MI, InstrGRegsMap &SignatureToGReg) {
InstrSignature MISign = instrToSignature(MI, MAI, false);
auto [It, Inserted] = SignatureToGReg.try_emplace(MISign);
if (!Inserted)
@@ -544,7 +544,7 @@ SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,
return GReg;
}
-MCRegister SPIRVModuleAnalysis::handleVariable(
+MCRegister SPIRVModuleAnalysisImpl::handleVariable(
const MachineFunction *MF, const MachineInstr &MI,
std::map<const Value *, unsigned> &GlobalToGReg) {
MAI.GlobalVarList.push_back(&MI);
@@ -561,11 +561,11 @@ MCRegister SPIRVModuleAnalysis::handleVariable(
return GReg;
}
-void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {
+void SPIRVModuleAnalysisImpl::collectDeclarations(const Module &M) {
InstrGRegsMap SignatureToGReg;
std::map<const Value *, unsigned> GlobalToGReg;
for (const Function &F : M) {
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
if (!MF)
continue;
const MachineRegisterInfo &MRI = MF->getRegInfo();
@@ -633,8 +633,8 @@ void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {
// to the register defining that variable (which will usually be the result of
// an OpFunction). This lets us call externally imported functions using
// the correct ID registers.
-void SPIRVModuleAnalysis::collectFuncNames(MachineInstr &MI,
- const Function *F) {
+void SPIRVModuleAnalysisImpl::collectFuncNames(MachineInstr &MI,
+ const Function *F) {
if (MI.getOpcode() == SPIRV::OpDecorate) {
// If it's got Import linkage.
auto Dec = MI.getOperand(1).getImm();
@@ -723,12 +723,12 @@ static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
// Some global instructions make reference to function-local ID regs, so cannot
// be correctly collected until these registers are globally numbered.
-void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
+void SPIRVModuleAnalysisImpl::processOtherInstrs(const Module &M) {
InstrTraces IS;
for (const Function &F : M) {
if (F.isDeclaration())
continue;
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
assert(MF);
for (MachineBasicBlock &MBB : *MF)
@@ -802,11 +802,11 @@ void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
// Number registers in all functions globally from 0 onwards and store
// the result in global register alias table. Some registers are already
// numbered.
-void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) {
+void SPIRVModuleAnalysisImpl::numberRegistersGlobally(const Module &M) {
for (const Function &F : M) {
if (F.isDeclaration())
continue;
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
assert(MF);
for (MachineBasicBlock &MBB : *MF) {
for (MachineInstr &MI : MBB) {
@@ -2680,10 +2680,10 @@ void addInstrRequirements(const MachineInstr &MI,
}
static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
- MachineModuleInfo *MMI, const SPIRVSubtarget &ST) {
+ MachineFunctionGetter GetMF, const SPIRVSubtarget &ST) {
// Collect requirements for existing instructions.
for (const Function &F : M) {
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
if (!MF)
continue;
for (const MachineBasicBlock &MBB : *MF)
@@ -2947,11 +2947,12 @@ static void handleMIFlagDecoration(
// Walk all functions and add decorations related to MI flags.
static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,
- MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
+ MachineFunctionGetter GetMF,
+ const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI,
const SPIRVGlobalRegistry *GR) {
for (const Function &F : M) {
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
if (!MF)
continue;
@@ -2963,10 +2964,10 @@ static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,
}
static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,
- MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
+ MachineFunctionGetter GetMF, const SPIRVSubtarget &ST,
SPIRV::ModuleAnalysisInfo &MAI) {
for (const Function &F : M) {
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
if (!MF)
continue;
if (MF->getFunction()
@@ -2989,9 +2990,9 @@ static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,
// patching Instruction::PHI to SPIRV::OpPhi
static void patchPhis(const Module &M, SPIRVGlobalRegistry *GR,
- const SPIRVInstrInfo &TII, MachineModuleInfo *MMI) {
+ const SPIRVInstrInfo &TII, MachineFunctionGetter GetMF) {
for (const Function &F : M) {
- MachineFunction *MF = MMI->getMachineFunction(F);
+ MachineFunction *MF = GetMF(F);
if (!MF)
continue;
for (auto &MBB : *MF) {
@@ -3113,29 +3114,22 @@ static void collectFPFastMathDefaults(const Module &M,
}
}
-void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
- AU.addRequired<TargetPassConfig>();
- AU.addRequired<MachineModuleInfoWrapperPass>();
-}
-
-bool SPIRVModuleAnalysis::runOnModule(Module &M) {
- SPIRVTargetMachine &TM =
- getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
- ST = TM.getSubtargetImpl();
- GR = ST->getSPIRVGlobalRegistry();
- TII = ST->getInstrInfo();
-
- MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+SPIRVModuleAnalysisImpl::SPIRVModuleAnalysisImpl(const SPIRVSubtarget &ST,
+ SPIRV::ModuleAnalysisInfo &MAI,
+ MachineFunctionGetter GetMF)
+ : ST(&ST), GR(ST.getSPIRVGlobalRegistry()), TII(ST.getInstrInfo()),
+ MAI(MAI), GetMF(GetMF) {}
+void SPIRVModuleAnalysisImpl::run(const Module &M) {
setBaseInfo(M);
- patchPhis(M, GR, *TII, MMI);
+ patchPhis(M, GR, *TII, GetMF);
- addMBBNames(M, *TII, MMI, *ST, MAI);
+ addMBBNames(M, *TII, GetMF, *ST, MAI);
collectFPFastMathDefaults(M, MAI, *ST);
- addDecorations(M, *TII, MMI, *ST, MAI, GR);
+ addDecorations(M, *TII, GetMF, *ST, MAI, GR);
- collectReqs(M, MAI, MMI, *ST);
+ collectReqs(M, MAI, GetMF, *ST);
// Process type/const/global var/func decl instructions, number their
// destination registers from 0 to N, collect Extensions and Capabilities.
@@ -3153,6 +3147,42 @@ bool SPIRVModuleAnalysis::runOnModule(Module &M) {
// Set maximum ID used.
GR->setBound(MAI.MaxID);
+}
+void SPIRVModuleAnalysisWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<TargetPassConfig>();
+ AU.addRequired<MachineModuleInfoWrapperPass>();
+}
+
+bool SPIRVModuleAnalysisWrapperPass::runOnModule(Module &M) {
+ SPIRVTargetMachine &TM =
+ getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
+ MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+ SPIRVModuleAnalysisImpl(
+ *TM.getSubtargetImpl(), MAI,
+ [&MMI](const Function &F) { return MMI.getMachineFunction(F); })
+ .run(M);
return false;
}
+
+AnalysisKey SPIRVModuleAnalysis::Key;
+
+SPIRVModuleAnalysis::Result
+SPIRVModuleAnalysis::run(Module &M, ModuleAnalysisManager &MAM) {
+ const auto &TM = static_cast<const SPIRVTargetMachine &>(
+ MAM.getResult<MachineModuleAnalysis>(M).getMMI().getTarget());
+ FunctionAnalysisManager &FAM =
+ MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+ Result MAI;
+ SPIRVModuleAnalysisImpl(*TM.getSubtargetImpl(), MAI,
+ [&FAM](const Function &F) -> MachineFunction * {
+ MachineFunctionAnalysis::Result *MFA =
+ FAM.getCachedResult<MachineFunctionAnalysis>(
+ const_cast<Function &>(F));
+ assert((MFA || F.isDeclaration()) &&
+ "Missing MachineFunction for definition");
+ return MFA ? &MFA->getMF() : nullptr;
+ })
+ .run(M);
+ return MAI;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index 6559b5cfc7457..6f939894337b0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -20,11 +20,11 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/PassManager.h"
namespace llvm {
class SPIRVSubtarget;
class MachineFunction;
-class MachineModuleInfo;
namespace SPIRV {
// The enum contains logical module sections for the instruction collection.
@@ -211,23 +211,26 @@ struct ModuleAnalysisInfo {
It->second = getNextIDRegister();
return It->second;
}
+ // Must stay alive until the AsmPrinter consumes it.
+ bool invalidate(Module &, const PreservedAnalyses &,
+ ModuleAnalysisManager::Invalidator &) {
+ return false;
+ }
};
} // namespace SPIRV
using InstrSignature = SmallVector<size_t>;
using InstrTraces = std::set<InstrSignature>;
using InstrGRegsMap = std::map<SmallVector<size_t>, unsigned>;
+using MachineFunctionGetter = function_ref<MachineFunction *(const Function &)>;
-struct SPIRVModuleAnalysis : public ModulePass {
- static char ID;
-
+class SPIRVModuleAnalysisImpl {
public:
- SPIRVModuleAnalysis()
- : ModulePass(ID), ST(nullptr), GR(nullptr), TII(nullptr), MMI(nullptr) {}
+ SPIRVModuleAnalysisImpl(const SPIRVSubtarget &ST,
+ SPIRV::ModuleAnalysisInfo &MAI,
+ MachineFunctionGetter GetMF);
- bool runOnModule(Module &M) override;
- void getAnalysisUsage(AnalysisUsage &AU) const override;
- SPIRV::ModuleAnalysisInfo MAI;
+ void run(const Module &M);
private:
void setBaseInfo(const Module &M);
@@ -257,7 +260,29 @@ struct SPIRVModuleAnalysis : public ModulePass {
const SPIRVSubtarget *ST;
SPIRVGlobalRegistry *GR;
const SPIRVInstrInfo *TII;
- MachineModuleInfo *MMI;
+ SPIRV::ModuleAnalysisInfo &MAI;
+ MachineFunctionGetter GetMF;
+};
+
+struct SPIRVModuleAnalysisWrapperPass : public ModulePass {
+ static char ID;
+
+public:
+ SPIRVModuleAnalysisWrapperPass() : ModulePass(ID) {}
+
+ bool runOnModule(Module &M) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+ SPIRV::ModuleAnalysisInfo MAI;
+};
+
+class SPIRVModuleAnalysis : public AnalysisInfoMixin<SPIRVModuleAnalysis> {
+ friend AnalysisInfoMixin<SPIRVModuleAnalysis>;
+ static AnalysisKey Key;
+
+public:
+ using Result = SPIRV::ModuleAnalysisInfo;
+
+ Result run(Module &M, ModuleAnalysisManager &MAM);
};
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRVMODULEANALYSIS_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def b/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def
index 4cd3a94b017b4..6236c03f5cbef 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def
+++ b/llvm/lib/Target/SPIRV/SPIRVPassRegistry.def
@@ -13,6 +13,12 @@
// NOTE: NO INCLUDE GUARD DESIRED!
+#ifndef MODULE_ANALYSIS
+#define MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+MODULE_ANALYSIS("spirv-module-analysis", SPIRVModuleAnalysis())
+#undef MODULE_ANALYSIS
+
#ifndef MODULE_PASS
#define MODULE_PASS(NAME, CREATE_PASS)
#endif
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index 7df3f9881fe73..ddf9889c5c521 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -46,7 +46,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTarget() {
PassRegistry &PR = *PassRegistry::getPassRegistry();
initializeGlobalISel(PR);
- initializeSPIRVModuleAnalysisPass(PR);
+ initializeSPIRVModuleAnalysisWrapperPassPass(PR);
initializeSPIRVAsmPrinterPass(PR);
initializeSPIRVConvergenceRegionAnalysisWrapperPassPass(PR);
initializeSPIRVStructurizerPass(PR);
diff --git a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
index 406641b2aa753..2aa5a18fdab94 100644
--- a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
+++ b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
@@ -7,9 +7,9 @@
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=SPIRV-Opt %s
; RUN:llc -O3 -mtriple=spirv-- -disable-verify -debug-pass=Structure < %s 2>&1 \
; RUN: | FileCheck -match-full-lines -strict-whitespace -check-prefix=SPIRV-Opt %s
-; RUN:llc -O0 -mtriple=spirv-- -enable-new-pm -stop-before=spirv-asm-printer < %s \
+; RUN:llc -O0 -mtriple=spirv-- -enable-new-pm < %s \
; RUN: | FileCheck -check-prefix=SPIRV-NPM %s
-; RUN:llc -O3 -mtriple=spirv-- -enable-new-pm -stop-before=spirv-asm-printer < %s \
+; RUN:llc -O3 -mtriple=spirv-- -enable-new-pm < %s \
; RUN: | FileCheck -check-prefix=SPIRV-NPM %s
; REQUIRES:asserts
@@ -230,13 +230,12 @@
; SPIRV-Opt-NEXT: SPIRV Assembly Printer
; SPIRV-Opt-NEXT: Free MachineFunction
-; SPIR-V NewPM skips RegBankSelect, so instruction selection must not require it.
-; SPIRV-NPM:legalized: true
-; SPIRV-NPM-NEXT:regBankSelected: false
-; SPIRV-NPM-NEXT:selected: true
-; SPIRV-NPM:%[[#VOID:]]:type = OpTypeVoid
-; SPIRV-NPM:%[[#]]:iid = OpFunction %[[#VOID]], 0, %[[#]]
+; SPIRV-NPM:OpCapability Linkage
+; SPIRV-NPM:OpName %[[#FN:]] "empty"
+; SPIRV-NPM:%[[#VOID:]] = OpTypeVoid
+; SPIRV-NPM:%[[#FN]] = OpFunction %[[#VOID]] None %[[#]]
; SPIRV-NPM:OpReturn
+; SPIRV-NPM:OpFunctionEnd
define void @empty() {
ret void
>From f26eef0b8447630f3ff4c02180faa59141b682cb Mon Sep 17 00:00:00 2001
From: Lucas Mellone <github.snugness349 at passinbox.com>
Date: Wed, 7 Oct 2026 12:50:12 +0200
Subject: [PATCH 38/52] [libc++][ranges][enumerate_view] Update sentinel
`equal` test (#222331)
Completes the [range.enumerate.sentinel] `equal` test by addressing the
review comment
https://github.com/llvm/llvm-project/pull/73617#discussion_r1416644183
from the original implementation.
---
.../range.enumerate/sentinel/equal.pass.cpp | 209 +++++++++++++++---
1 file changed, 177 insertions(+), 32 deletions(-)
diff --git a/libcxx/test/std/ranges/range.adaptors/range.enumerate/sentinel/equal.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.enumerate/sentinel/equal.pass.cpp
index 7bf2898bcd158..ed0ccb9608a76 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.enumerate/sentinel/equal.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.enumerate/sentinel/equal.pass.cpp
@@ -18,56 +18,201 @@
// requires sentinel_for<sentinel_t<Base>, iterator_t<maybe-const<OtherConst, V>>>
// friend constexpr bool operator==(const iterator<OtherConst>& x, const sentinel& y);
-#include <ranges>
-
-#include <array>
#include <cassert>
#include <concepts>
-#include <memory>
+#include <ranges>
+#include <type_traits>
#include <utility>
+#include "../../range_adaptor_types.h"
+#include "../types.h"
#include "test_iterators.h"
+#include "test_range.h"
+
+using Iterator = random_access_iterator<int*>;
+using ConstIterator = contiguous_iterator<const int*>;
+
+template <bool Const>
+struct ComparableSentinel {
+ using Iter = std::conditional_t<Const, ConstIterator, Iterator>;
+ Iter iter_;
+
+ explicit ComparableSentinel() = default;
+ constexpr explicit ComparableSentinel(const Iter& it) : iter_(it) {}
+
+ constexpr friend bool operator==(const Iterator& i, const ComparableSentinel& s) { return base(i) == base(s.iter_); }
+
+ constexpr friend bool operator==(const ConstIterator& i, const ComparableSentinel& s) {
+ return base(i) == base(s.iter_);
+ }
+};
+
+struct ComparableView : IntBufferView {
+ using IntBufferView::IntBufferView;
-#include "../types.h"
+ constexpr auto begin() { return Iterator(buffer_); }
+ constexpr auto begin() const { return ConstIterator(buffer_); }
+ constexpr auto end() { return ComparableSentinel<false>(Iterator(buffer_ + size_)); }
+ constexpr auto end() const { return ComparableSentinel<true>(ConstIterator(buffer_ + size_)); }
+};
+
+struct ConstIncompatibleView : IntBufferView {
+ using IntBufferView::IntBufferView;
+
+ constexpr random_access_iterator<int*> begin() { return random_access_iterator<int*>(buffer_); }
+ constexpr contiguous_iterator<const int*> begin() const { return contiguous_iterator<const int*>(buffer_); }
+ constexpr sentinel_wrapper<random_access_iterator<int*>> end() {
+ return sentinel_wrapper<random_access_iterator<int*>>(random_access_iterator<int*>(buffer_ + size_));
+ }
+ constexpr sentinel_wrapper<contiguous_iterator<const int*>> end() const {
+ return sentinel_wrapper<contiguous_iterator<const int*>>(contiguous_iterator<const int*>(buffer_ + size_));
+ }
+};
+
+constexpr bool test() {
+ int buffer[] = {0, 1, 2, 3, 85};
+ {
+ // simple-view: const and non-const have the same iterator/sentinel type
+ using View = std::ranges::enumerate_view<SimpleNonCommon>;
+ static_assert(!std::ranges::common_range<View>);
+ static_assert(simple_view<View>);
+
+ static_assert(weakly_equality_comparable_with<std::ranges::iterator_t<View>, std::ranges::sentinel_t<View>>);
+
+ View ev{SimpleNonCommon(buffer)};
-template <class Iterator, class Sentinel = sentinel_wrapper<Iterator>>
-constexpr void test() {
- using View = MinimalView<Iterator, Sentinel>;
+ assert(ev.begin() != ev.end());
+ assert(ev.begin() + 1 != ev.end());
+ assert(ev.begin() + 2 != ev.end());
+ assert(ev.begin() + 3 != ev.end());
+ assert(ev.begin() + 5 == ev.end());
+ }
+
+ {
+ // !simple-view: const and non-const have different iterator/sentinel types
+ using View = std::ranges::enumerate_view<NonSimpleNonCommon>;
+ static_assert(!std::ranges::common_range<View>);
+ static_assert(!simple_view<View>);
+
+ using Iter = std::ranges::iterator_t<View>;
+ using ConstIter = std::ranges::iterator_t<const View>;
+ static_assert(!std::is_same_v<Iter, ConstIter>);
+
+ using Sentinel = std::ranges::sentinel_t<View>;
+ using ConstSentinel = std::ranges::sentinel_t<const View>;
+ static_assert(!std::is_same_v<Sentinel, ConstSentinel>);
+
+ static_assert(weakly_equality_comparable_with<Iter, Sentinel>);
+ static_assert(!weakly_equality_comparable_with<ConstIter, Sentinel>);
+ static_assert(weakly_equality_comparable_with<Iter, ConstSentinel>);
+ static_assert(weakly_equality_comparable_with<ConstIter, ConstSentinel>);
+
+ View ev{NonSimpleNonCommon(buffer)};
+
+ assert(ev.begin() != ev.end());
+ assert(ev.begin() + 5 == ev.end());
+
+ assert(ev.begin() != std::as_const(ev).end());
+ assert(ev.begin() + 5 == std::as_const(ev).end());
+ // the above works because
+ static_assert(std::convertible_to<Iter, ConstIter>);
+
+ assert(std::as_const(ev).begin() != std::as_const(ev).end());
+ assert(std::as_const(ev).begin() + 5 == std::as_const(ev).end());
+ }
+
+ {
+ // underlying const/non-const sentinel can be compared with both const/non-const iterator
+ using View = std::ranges::enumerate_view<ComparableView>;
+ static_assert(!std::ranges::common_range<View>);
+ static_assert(!simple_view<View>);
+
+ using Iter = std::ranges::iterator_t<View>;
+ using ConstIter = std::ranges::iterator_t<const View>;
+ static_assert(!std::is_same_v<Iter, ConstIter>);
+
+ using Sentinel = std::ranges::sentinel_t<View>;
+ using ConstSentinel = std::ranges::sentinel_t<const View>;
+ static_assert(!std::is_same_v<Sentinel, ConstSentinel>);
+
+ static_assert(weakly_equality_comparable_with<Iter, Sentinel>);
+ static_assert(weakly_equality_comparable_with<ConstIter, Sentinel>);
+ static_assert(weakly_equality_comparable_with<Iter, ConstSentinel>);
+ static_assert(weakly_equality_comparable_with<ConstIter, ConstSentinel>);
+
+ View ev{ComparableView(buffer)};
+
+ assert(ev.begin() != ev.end());
+ assert(ev.begin() + 5 == ev.end());
+
+ static_assert(!std::convertible_to<Iter, ConstIter>);
+
+ assert(ev.begin() != std::as_const(ev).end());
+ assert(ev.begin() + 5 == std::as_const(ev).end());
+
+ assert(std::as_const(ev).begin() != ev.end());
+ assert(std::as_const(ev).begin() + 5 == ev.end());
+
+ assert(std::as_const(ev).begin() != std::as_const(ev).end());
+ assert(std::as_const(ev).begin() + 5 == std::as_const(ev).end());
+ }
+
+ {
+ // underlying const/non-const sentinel cannot be compared with non-const/const iterator
+ using View = std::ranges::enumerate_view<ComparableView>;
+ static_assert(!std::ranges::common_range<View>);
+ static_assert(!simple_view<View>);
+
+ using Iter = std::ranges::iterator_t<View>;
+ using ConstIter = std::ranges::iterator_t<const View>;
+ static_assert(!std::is_same_v<Iter, ConstIter>);
- std::array array{0, 1, 2, 3, 84};
+ using Sentinel = std::ranges::sentinel_t<View>;
+ using ConstSentinel = std::ranges::sentinel_t<const View>;
+ static_assert(!std::is_same_v<Sentinel, ConstSentinel>);
- View mv{Iterator(std::to_address(base(array.begin()))), Sentinel(Iterator(std::to_address(base(array.end()))))};
- std::ranges::enumerate_view view(std::move(mv));
+ static_assert(weakly_equality_comparable_with<Iter, Sentinel>);
+ static_assert(weakly_equality_comparable_with<ConstIter, Sentinel>);
+ static_assert(weakly_equality_comparable_with<Iter, ConstSentinel>);
+ static_assert(weakly_equality_comparable_with<ConstIter, ConstSentinel>);
- auto const it = view.begin();
- auto const s = view.end();
+ View ev{ComparableView(buffer)};
- std::same_as<bool> decltype(auto) eqItSResult = (it == s);
- assert(!eqItSResult);
- std::same_as<bool> decltype(auto) eqSItResult = (s == it);
- assert(!eqSItResult);
+ assert(ev.begin() != ev.end());
+ assert(ev.begin() + 5 == ev.end());
- std::same_as<bool> decltype(auto) neqItSResult = (it != s);
- assert(neqItSResult);
- std::same_as<bool> decltype(auto) neqSItResult = (s != it);
- assert(neqSItResult);
-}
+ assert(std::as_const(ev).begin() != std::as_const(ev).end());
+ assert(std::as_const(ev).begin() + 5 == std::as_const(ev).end());
+ }
+
+ {
+ // sentinel comparison with input move-only iterator must work without copying the iterator
+ using InputIterator = cpp20_input_iterator<int*>;
+ using Sentinel = sentinel_wrapper<InputIterator>;
+ using View = MinimalView<InputIterator, Sentinel>;
+ static_assert(simple_view<View>);
+
+ View mv{InputIterator(buffer), Sentinel(InputIterator(buffer + 5))};
+ std::ranges::enumerate_view ev(std::move(mv));
+
+ auto it = ev.begin();
+ assert(it != ev.end());
+ assert(ev.end() != it);
+
+ // enumerate_view iterator only has operator+ when the underlying range is random-access.
+ // for input_iterator we increment it explicitly.
+ std::ranges::advance(it, 5);
-constexpr bool tests() {
- test<cpp17_input_iterator<int*>>();
- test<cpp20_input_iterator<int*>>();
- test<forward_iterator<int*>>();
- test<bidirectional_iterator<int*>>();
- test<random_access_iterator<int*>>();
- test<contiguous_iterator<int*>>();
- test<int*>();
+ assert(it == ev.end());
+ assert(ev.end() == it);
+ }
return true;
}
int main(int, char**) {
- tests();
- static_assert(tests());
+ test();
+ static_assert(test());
return 0;
}
>From abbf6958bb8449ea38dca4c6b843aad76e82670f Mon Sep 17 00:00:00 2001
From: keinflue <keinflue at posteo.de>
Date: Wed, 7 Oct 2026 10:53:28 +0000
Subject: [PATCH 39/52] [clang] Implement constexpr heap allocation alignments
(#174549)
Previously the old constant interpreter would crash on
`__builtin_assume_aligned` applied to dynamic allocations and the new
interpreter would incorrectly always assume an alignment of 8.
This implements computation of the alignments of dynamic allocations for
the builtin by using the minimum alignment guarantees made in the
standard for each kind of possible dynamic allocation during constant
evaluation.
The kind of allocation depends on the source expression causing it to be
created, but the alignment also depends on the size of the allocation
which can be retrieved from the allocated type, but is decided
dynamically.
For the old constant expression evaluator, the allocated type is always
available from the LValue, but the source expression is only available
from the `DynAlloc` representing the allocation.
The `DynAlloc` object does not continue to exist when the lifetime of
the dynamic allocation ends. Because `__builtin_assume_aligned` should
still be valid out-of-lifetime, it is therefore not possible to retrieve
the source expression from the `DynAlloc` object.
Instead this keeps track of the allocation kind during the creation of the
allocation and stores it as part of the `LValueBase` by appropriating
3 bits of the allocation identifier.
The new constant expression interpreter does not need additional state
to store the alignment, because it already has both the source
expression and allocated size available.
Fixes #173767
---------
Co-authored-by: Nikolas Klauser <nikolasklauser at berlin.de>
---
clang/docs/ReleaseNotes.md | 10 +-
clang/include/clang/AST/APValue.h | 55 ++++++--
clang/include/clang/AST/PropertiesBase.td | 7 +-
.../clang/Serialization/ASTRecordReader.h | 4 +
.../clang/Serialization/ASTRecordWriter.h | 4 +
clang/lib/AST/ByteCode/Descriptor.cpp | 36 +++--
clang/lib/AST/ByteCode/Descriptor.h | 13 ++
clang/lib/AST/ByteCode/Interp.cpp | 20 +--
clang/lib/AST/ByteCode/Interp.h | 3 +-
clang/lib/AST/ByteCode/InterpBlock.h | 7 +-
clang/lib/AST/ByteCode/InterpBuiltin.cpp | 6 +-
clang/lib/AST/ByteCode/Pointer.cpp | 6 +-
clang/lib/AST/ExprConstShared.h | 3 +
clang/lib/AST/ExprConstant.cpp | 127 +++++++++++++++---
clang/lib/Sema/SemaExprCXX.cpp | 1 +
clang/test/AST/ByteCode/new-delete.cpp | 8 +-
clang/test/SemaCXX/builtin-assume-aligned.cpp | 94 ++++++++++++-
.../SemaCXX/constant-expression-cxx2a.cpp | 3 +-
18 files changed, 332 insertions(+), 75 deletions(-)
diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 08442f74879af..efbcc198a28e3 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -642,8 +642,10 @@ features cannot lower the translation-unit ABI level;
reference to a vector type; `vec_step` (in C++ for OpenCL) and
`__builtin_ptrauth_type_discriminator` similarly no longer accept reference
types that their evaluation silently mishandled. (#GH216997)
+- Fix a crash when using `__builtin_assume_aligned` with dynamic allocations
+ during constant evaluation. (#GH173767)
- Fixed a crash when constant-evaluating `__builtin_align_up`, `__builtin_align_down`,
- or `__builtin_is_aligned` with pointers without an underlying object. Null pointers
+ or `__builtin_is_aligned` with pointers without an underlying object. Null pointers
are handled as aligned values, while other base-less pointers are rejected during constant
evaluation.
@@ -844,9 +846,9 @@ features cannot lower the translation-unit ABI level;
- Fixed ambiguous overload where two non-static member functions with
different signatures could be incorrectly considered equivalent. (#GH224499)
-- Fixed an assertion failure when explicitly instantiating a nested member with
- an ill-formed template argument. Clang now checks for a failed declaration
- lookup before asserting that the name is not dependent, avoiding an assertion
+- Fixed an assertion failure when explicitly instantiating a nested member with
+ an ill-formed template argument. Clang now checks for a failed declaration
+ lookup before asserting that the name is not dependent, avoiding an assertion
after an earlier diagnostic has caused the declaration to be unavailable. (#GH220525)
- Fixed a crash in constant evaluation when a new-expression selects a
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index 0a338f805a665..35576a4b6403e 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -22,6 +22,7 @@
#include "llvm/ADT/FoldingSet.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/bit.h"
#include "llvm/Support/AlignOf.h"
#include "llvm/Support/Compiler.h"
@@ -63,33 +64,65 @@ class TypeInfoLValue {
void print(llvm::raw_ostream &Out, const PrintingPolicy &Policy) const;
};
+/// Kind of source for a dynamic allocation.
+enum class DynAllocKind {
+ New, // new expression
+ ArrayNew, // new[] expression
+ StdAllocator, // std::allocator::allocate call
+ None, // not a dynamic allocation
+ BuiltinOperatorNew, // __operator_builtin_new call
+ ALLOC_KIND_MAX = BuiltinOperatorNew
+};
+
/// Symbolic representation of a dynamic allocation.
class DynamicAllocLValue {
- unsigned Index;
+public:
+ static constexpr int NumLowBitsAvailable = 2;
+ static constexpr int NumAllocKindBits = 3;
+ static_assert((1 << NumAllocKindBits) - 1 >=
+ static_cast<int>(DynAllocKind::ALLOC_KIND_MAX));
+
+private:
+ // lower NumAlignmentBits: alignment exponent
+ // remaining bits: allocation index incremented by one
+ // value of zero indicates distinct empty state
+ LLVM_PREFERRED_TYPE(DynAllocKind)
+ uintptr_t AllocKind : NumAllocKindBits;
+ uintptr_t Index : sizeof(uintptr_t) * CHAR_BIT - NumAllocKindBits;
public:
- DynamicAllocLValue() : Index(0) {}
- explicit DynamicAllocLValue(unsigned Index) : Index(Index + 1) {}
- unsigned getIndex() { return Index - 1; }
+ DynamicAllocLValue() : AllocKind(0), Index(0) {}
+ explicit DynamicAllocLValue(unsigned Idx, DynAllocKind AllocKind)
+ : AllocKind(llvm::to_underlying(AllocKind)), Index(Idx + 1) {
+ assert(Idx <= getMaxIndex() && "Index is out of range");
+ }
+ unsigned getIndex() const { return Index - 1; }
+ DynAllocKind getAllocKind() const {
+ return static_cast<DynAllocKind>(AllocKind);
+ }
explicit operator bool() const { return Index != 0; }
const void *getOpaqueValue() const {
- return reinterpret_cast<const void *>(static_cast<uintptr_t>(Index)
- << NumLowBitsAvailable);
+ return reinterpret_cast<const void *>(
+ (Index << NumAllocKindBits | AllocKind) << NumLowBitsAvailable);
}
static DynamicAllocLValue getFromOpaqueValue(const void *Value) {
DynamicAllocLValue V;
- V.Index = reinterpret_cast<uintptr_t>(Value) >> NumLowBitsAvailable;
+ uintptr_t Combined =
+ reinterpret_cast<uintptr_t>(Value) >> NumLowBitsAvailable;
+ V.AllocKind = Combined & (1 << NumAllocKindBits) - 1;
+ V.Index = Combined >> NumAllocKindBits;
return V;
}
- static unsigned getMaxIndex() {
- return (std::numeric_limits<unsigned>::max() >> NumLowBitsAvailable) - 1;
+ static uintptr_t getMaxIndex() {
+ return (std::numeric_limits<uintptr_t>::max() >>
+ (NumLowBitsAvailable + NumAllocKindBits)) -
+ 1;
}
-
- static constexpr int NumLowBitsAvailable = 3;
};
+static_assert(sizeof(DynamicAllocLValue) == sizeof(uintptr_t));
}
namespace llvm {
diff --git a/clang/include/clang/AST/PropertiesBase.td b/clang/include/clang/AST/PropertiesBase.td
index 88ece140e826a..bb572d40d6c8c 100644
--- a/clang/include/clang/AST/PropertiesBase.td
+++ b/clang/include/clang/AST/PropertiesBase.td
@@ -77,6 +77,7 @@ def APValueKind : EnumPropertyType<"APValue::ValueKind">;
def ArraySizeModifier : EnumPropertyType<"ArraySizeModifier">;
def AttrKind : EnumPropertyType<"attr::Kind">;
def Attr : PropertyType<"const Attr *">;
+def DynAllocKind : PropertyType<"DynAllocKind">;
def DeducedKind : EnumPropertyType;
def AutoTypeKeyword : EnumPropertyType;
def Bool : PropertyType<"bool">;
@@ -544,6 +545,10 @@ let Class = PropertyTypeCase<APValue, "LValue"> in {
let Conditional = [{ hasBase && isDynamicAlloc }];
let Read = [{ node.getLValueBase().get<DynamicAllocLValue>().getIndex() }];
}
+ def : Property<"dynamicAlign", DynAllocKind> {
+ let Conditional = [{ hasBase && isDynamicAlloc }];
+ let Read = [{ node.getLValueBase().get<DynamicAllocLValue>().getAllocKind() }];
+ }
def : Property<"type", QualType> {
let Conditional = [{ hasBase && (isTypeInfo || isDynamicAlloc) }];
let Read = [{
@@ -586,7 +591,7 @@ let Class = PropertyTypeCase<APValue, "LValue"> in {
TypeInfoLValue(typeInfo->getTypePtr()), *type);
} else if (isDynamicAlloc) {
base = APValue::LValueBase::getDynamicAlloc(
- DynamicAllocLValue(*dynamicAlloc), *type);
+ DynamicAllocLValue(*dynamicAlloc, *dynamicAlign), *type);
} else if (isExpr) {
base = APValue::LValueBase(cast<Expr>(*stmt),
*callIndex, *version);
diff --git a/clang/include/clang/Serialization/ASTRecordReader.h b/clang/include/clang/Serialization/ASTRecordReader.h
index aed1b7d309001..c3482f146374f 100644
--- a/clang/include/clang/Serialization/ASTRecordReader.h
+++ b/clang/include/clang/Serialization/ASTRecordReader.h
@@ -320,6 +320,10 @@ class ASTRecordReader
return readInt();
}
+ DynAllocKind readDynAllocKind() {
+ return static_cast<DynAllocKind>(readInt());
+ }
+
UnsignedOrNone readUnsignedOrNone() {
return UnsignedOrNone::fromInternalRepresentation(unsigned(readInt()));
}
diff --git a/clang/include/clang/Serialization/ASTRecordWriter.h b/clang/include/clang/Serialization/ASTRecordWriter.h
index 9849ea6b395ab..8fc60338c4c72 100644
--- a/clang/include/clang/Serialization/ASTRecordWriter.h
+++ b/clang/include/clang/Serialization/ASTRecordWriter.h
@@ -189,6 +189,10 @@ class ASTRecordWriter
Record->push_back(Value);
}
+ void writeDynAllocKind(DynAllocKind Value) {
+ Record->push_back(llvm::to_underlying(Value));
+ }
+
void writeUnsignedOrNone(UnsignedOrNone Value) {
Record->push_back(Value.toInternalRepresentation());
}
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 0137adf003a30..2c8e3c0da7f9d 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -9,6 +9,7 @@
#include "Descriptor.h"
#include "Boolean.h"
#include "Char.h"
+#include "ExprConstShared.h"
#include "FixedPoint.h"
#include "Floating.h"
#include "Integral.h"
@@ -20,6 +21,8 @@
#include "Reflect.h"
#include "Source.h"
#include "clang/AST/ExprCXX.h"
+#include "clang/Basic/TargetInfo.h"
+#include "llvm/Support/ErrorHandling.h"
using namespace clang;
using namespace clang::interp;
@@ -456,17 +459,8 @@ QualType Descriptor::getDataType(const ASTContext &Ctx) const {
return ElemType;
};
- if (const auto *E = asExpr()) {
- if (isa<CXXNewExpr>(E))
- return MakeArrayType(E->getType()->getPointeeType());
-
- // std::allocator.allocate() call.
- if (const auto *ME = dyn_cast<CXXMemberCallExpr>(E);
- ME && ME->getRecordDecl()->getName() == "allocator" &&
- ME->getMethodDecl()->getName() == "allocate")
- return MakeArrayType(E->getType()->getPointeeType());
- return E->getType();
- }
+ if (isDynAlloc())
+ return MakeArrayType(asExpr()->getType()->getPointeeType());
return getType();
}
@@ -505,3 +499,23 @@ unsigned Descriptor::getElemDataSize() const {
}
return ElemSize;
}
+
+DynAllocKind Descriptor::getDynAllocKindForExpr(const Expr *E) {
+ // new or new[] expression
+ if (const auto *NE = dyn_cast<CXXNewExpr>(E))
+ return NE->isArray() ? DynAllocKind::ArrayNew : DynAllocKind::New;
+ // std::allocator::allocate call
+ if (const auto *ME = dyn_cast<CXXMemberCallExpr>(E);
+ ME && ME->getRecordDecl()->getName() == "allocator" &&
+ ME->getMethodDecl()->getName() == "allocate")
+ return DynAllocKind::StdAllocator;
+ // __builtin_operator_new call
+ if (const auto *CE = dyn_cast<CallExpr>(E);
+ CE && CE->getBuiltinCallee() == Builtin::BI__builtin_operator_new)
+ return DynAllocKind::BuiltinOperatorNew;
+ return DynAllocKind::None;
+}
+
+CharUnits Descriptor::computeAlignForDynamicAlloc(const ASTContext &Ctx) const {
+ return GetAlignOfDynamicAlloc(Ctx, getDataType(Ctx), getDynAllocKind());
+}
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index bec7ca653e055..567d08c4507d7 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -13,6 +13,7 @@
#ifndef LLVM_CLANG_AST_INTERP_DESCRIPTOR_H
#define LLVM_CLANG_AST_INTERP_DESCRIPTOR_H
+#include "../ExprConstShared.h"
#include "DeclOrExpr.h"
#include "InitMap.h"
#include "PrimType.h"
@@ -269,6 +270,18 @@ struct Descriptor final {
/// Whether variables of this descriptor need their destructor called or not.
bool hasTrivialDtor() const;
+ /// Returns the kind of dynamic allocation source of this block.
+ static DynAllocKind getDynAllocKindForExpr(const Expr *E);
+ /// Returns the kind of dynamic allocation source of this block.
+ DynAllocKind getDynAllocKind() const {
+ return asExpr() ? getDynAllocKindForExpr(asExpr()) : DynAllocKind::None;
+ }
+ /// Checks if the descriptor is of a dynamic allocation.
+ bool isDynAlloc() const { return getDynAllocKind() != DynAllocKind::None; }
+
+ /// Compute the alignment for a dynamic allocation.
+ CharUnits computeAlignForDynamicAlloc(const ASTContext &Ctx) const;
+
void dump() const;
void dump(llvm::raw_ostream &OS) const;
void dumpFull(unsigned Offset = 0, unsigned Indent = 0) const;
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index eae2698e29db8..4bcabd73a4e26 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1384,20 +1384,8 @@ bool CheckNewDeleteForms(InterpState &S, CodePtr OpPC,
return false;
}
-bool CheckDeleteSource(InterpState &S, CodePtr OpPC, const Expr *Source,
- const Pointer &Ptr) {
- if (!Ptr.isBlockPointer() && !Ptr.isOpaquePointer())
- return false;
- // Regular new type(...) call.
- if (isa_and_nonnull<CXXNewExpr>(Source))
- return true;
- // operator new.
- if (const auto *CE = dyn_cast_if_present<CallExpr>(Source);
- CE && CE->getBuiltinCallee() == Builtin::BI__builtin_operator_new)
- return true;
- // std::allocator.allocate() call
- if (const auto *MCE = dyn_cast_if_present<CXXMemberCallExpr>(Source);
- MCE && MCE->getMethodDecl()->getIdentifier()->isStr("allocate"))
+bool CheckDeleteSource(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
+ if (Ptr.isBlockPointer() && Ptr.block()->isDynamic())
return true;
// Whatever this is, we didn't heap allocate it.
@@ -1563,7 +1551,7 @@ bool Free(InterpState &S, CodePtr OpPC, bool DeleteIsArrayForm,
return true;
if (!Ptr.isBlockPointer())
- return CheckDeleteSource(S, OpPC, nullptr, Ptr);
+ return CheckDeleteSource(S, OpPC, Ptr);
// Remove base casts.
QualType InitialType = Ptr.getType();
@@ -1602,7 +1590,7 @@ bool Free(InterpState &S, CodePtr OpPC, bool DeleteIsArrayForm,
return false;
}
- if (!CheckDeleteSource(S, OpPC, Source, Ptr))
+ if (!CheckDeleteSource(S, OpPC, Ptr))
return false;
// For a class type with a virtual destructor, the selected operator delete
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 62f179ebaf7b8..b82600ca5db86 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -105,8 +105,7 @@ bool CheckDynamicMemoryAllocation(InterpState &S, CodePtr OpPC);
/// Check the source of the pointer passed to delete/delete[] has actually
/// been heap allocated by us.
-bool CheckDeleteSource(InterpState &S, CodePtr OpPC, const Expr *Source,
- const Pointer &Ptr);
+bool CheckDeleteSource(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
/// Sets the given integral value to the pointer, which is of
/// a std::{weak,partial,strong}_ordering type.
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 4d6a2ecf81321..8aaff6eb9464e 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -80,7 +80,12 @@ class Block final {
/// Checks if the block is temporary.
bool isTemporary() const { return Desc->IsTemporary; }
bool isWeak() const { return AccessFlags & WeakFlag; }
- bool isDynamic() const { return (DynAllocId != std::nullopt); }
+ bool isDynamic() const {
+ bool Result = (DynAllocId != std::nullopt);
+ assert((Result == Desc->isDynAlloc()) &&
+ "Inconsistent block/descriptor dynamic alloc state");
+ return Result;
+ }
bool isDead() const { return AccessFlags & DeadFlag; }
/// Returns the size of the block, including metadata.
unsigned getSize() const { return Desc->getAllocSize() + MDSize; }
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 5ea3cc6bd2920..5f3f59ae24245 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -14,6 +14,8 @@
#include "InterpHelpers.h"
#include "PrimType.h"
#include "Program.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/ExprCXX.h"
#include "clang/AST/InferAlloc.h"
#include "clang/AST/OSLog.h"
#include "clang/AST/RecordLayout.h"
@@ -1454,7 +1456,9 @@ static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
// If there is a base object, then it must have the correct alignment.
if (Ptr.isBlockPointer() || Ptr.isOpaquePointer()) {
CharUnits BaseAlignment;
- if (const auto *VD = Ptr.getRootVarDecl())
+ if (Ptr.isBlockPointer() && Ptr.block()->isDynamic())
+ BaseAlignment = Ptr.getDeclDesc()->computeAlignForDynamicAlloc(ASTCtx);
+ else if (const auto *VD = Ptr.getRootVarDecl())
BaseAlignment = ASTCtx.getDeclAlign(VD);
else if (const auto *E = Ptr.getRootExpr())
BaseAlignment = GetAlignOfExpr(ASTCtx, E, UETT_AlignOf);
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index c7e00cc964f82..6ea2cae2a02ca 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -346,9 +346,9 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
Base = VD;
else if (const auto *E = Desc->asExpr()) {
if (block()->isDynamic()) {
- QualType AllocatedType = getDeclPtr().getFieldDesc()->getDataType(ASTCtx);
- DynamicAllocLValue DA(*block()->DynAllocId);
- Base = APValue::LValueBase::getDynamicAlloc(DA, AllocatedType);
+ DynamicAllocLValue DA(*block()->DynAllocId, Desc->getDynAllocKind());
+ Base =
+ APValue::LValueBase::getDynamicAlloc(DA, Desc->getDataType(ASTCtx));
} else {
Base = E;
}
diff --git a/clang/lib/AST/ExprConstShared.h b/clang/lib/AST/ExprConstShared.h
index 7693ea25592f1..b8682bcd3e44d 100644
--- a/clang/lib/AST/ExprConstShared.h
+++ b/clang/lib/AST/ExprConstShared.h
@@ -32,6 +32,9 @@ class CharUnits;
class Expr;
class CallExpr;
class CXXRecordDecl;
+
+CharUnits GetAlignOfDynamicAlloc(const ASTContext &Ctx, QualType AllocType,
+ DynAllocKind AllocKind);
} // namespace clang
using namespace clang;
/// Values returned by __builtin_classify_type, chosen to match the values
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 34c3e39807c7f..d5b8d0ed889d1 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -62,6 +62,7 @@
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/bit.h"
#include "llvm/Support/CRC.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
@@ -731,20 +732,16 @@ namespace {
/// std::allocator<T>::allocate).
const Expr *AllocExpr = nullptr;
- enum Kind {
- New,
- ArrayNew,
- StdAllocator
- };
-
/// Get the kind of the allocation. This must match between allocation
/// and deallocation.
- Kind getKind() const {
+ static DynAllocKind kindOfExpr(const Expr *AllocExpr) {
if (auto *NE = dyn_cast<CXXNewExpr>(AllocExpr))
- return NE->isArray() ? ArrayNew : New;
+ return NE->isArray() ? DynAllocKind::ArrayNew : DynAllocKind::New;
assert(isa<CallExpr>(AllocExpr));
- return StdAllocator;
+ return DynAllocKind::StdAllocator;
}
+
+ DynAllocKind getKind() const { return kindOfExpr(AllocExpr); }
};
struct DynAllocOrder {
@@ -823,7 +820,7 @@ namespace {
std::map<DynamicAllocLValue, DynAlloc, DynAllocOrder> HeapAllocs;
/// The number of heap allocations performed so far in this evaluation.
- unsigned NumHeapAllocs = 0;
+ uintptr_t NumHeapAllocs = 0;
struct EvaluatingConstructorRAII {
EvalInfo &EI;
@@ -1823,13 +1820,109 @@ APValue &CallStackFrame::createLocal(APValue::LValueBase Base, const void *Key,
return Result;
}
+CharUnits clang::GetAlignOfDynamicAlloc(const ASTContext &Ctx,
+ QualType AllocType,
+ DynAllocKind AllocKind) {
+ assert((AllocKind != DynAllocKind::None) &&
+ "should only be called on dynamically allocated blocks");
+ assert((AllocKind != DynAllocKind::BuiltinOperatorNew) &&
+ "__builtin_operator_new should have been allowed only from "
+ "std::allocator::allocate");
+
+ const TargetInfo &TI = Ctx.getTargetInfo();
+ uint64_t DefaultNewAlign = TI.getNewAlign();
+
+ uint64_t TypeAlignment = Ctx.getTypeAlign(AllocType);
+ assert(TypeAlignment > 0 && "Unknown alignment for allocated type!");
+
+ uint64_t AllocSize = Ctx.getTypeSize(AllocType);
+
+ if (AllocSize == 0) {
+ switch (AllocKind) {
+ // Allocating a zero-sized array is allowed, however it doesn't have
+ // any alignment guarantees.
+ case DynAllocKind::ArrayNew:
+ case DynAllocKind::StdAllocator:
+ return CharUnits::One();
+
+ // Flexible array members are allowed as only member as an extension.
+ // In this case the size of the type will be zero, but the allocation
+ // should still be suitable for the array element type.
+ case DynAllocKind::New:
+ return Ctx.toCharUnitsFromBits(TypeAlignment);
+
+ default:
+ llvm_unreachable("Unhandled DynAllocKind");
+ }
+ }
+
+ assert(TypeAlignment <= AllocSize && "Invalid alignment/size for type!");
+ assert(AllocSize % TypeAlignment == 0 && "Invalid alignment/size for type!");
+
+ // For new-extended alignment the ::operator new overload with
+ // std::align_val_t parameter is used. According to C++
+ // [basic.stc.dynamic.allocation]p3.1 this overload returns memory
+ // according to the requested alignment. No stricter guarantees are
+ // made.
+ if (TypeAlignment > DefaultNewAlign)
+ return Ctx.toCharUnitsFromBits(TypeAlignment);
+
+ switch (AllocKind) {
+ // According to C++ [allocator.members]p5 it is unspecified how the
+ // memory obtained from ::operator new is used by
+ // std::allocator::allocate, therefore be conservative here.
+ case DynAllocKind::StdAllocator:
+ return Ctx.toCharUnitsFromBits(TypeAlignment);
+
+ // The non-array form of new does not permit allocation overhead and
+ // therefore provides alignment as guaranteed by ::operator new.
+ // According to C++ [basic.stc.dynamic.allocation]p3.3 the allocation
+ // is suitably aligned for all objects without new-extended alignment
+ // with the exact size of the allocation. An object of the exact size
+ // AllocSize can have alignment of at most the lowest bit set in
+ // AllocSize.
+ case DynAllocKind::New:
+ return Ctx.toCharUnitsFromBits(
+ std::min(DefaultNewAlign, uint64_t(1) << llvm::countr_zero(AllocSize)));
+
+ case DynAllocKind::ArrayNew: {
+ const Type *ET = AllocType.getTypePtr()
+ ->getArrayElementTypeNoTypeQual()
+ ->getCanonicalTypeUnqualified()
+ .getTypePtr();
+ // According to C++ [expr.new]p17, unless the element type of an
+ // array new expression is char, unsigned char or std::byte, the
+ // allocation may be offset into the allocation returned by
+ // ::operator new[]. Therefore no stricter alignment than the type's
+ // alignment is guaranteed. For char, unsigned char and std::byte
+ if (!ET->isSpecificBuiltinType(BuiltinType::UChar) &&
+ !ET->isSpecificBuiltinType(BuiltinType::Char_U) &&
+ !ET->isSpecificBuiltinType(BuiltinType::Char_S) && !ET->isStdByteType())
+ return Ctx.toCharUnitsFromBits(TypeAlignment);
+
+ // Otherwise, the allocation is offset from the result of ::operator
+ // new[] by a multiple of the strictest fundamental alignment.
+ // According C++ [basic.stc.dynamic.allocation]p3.2 the allocation
+ // returned by ::operator new[] is suitably aligned for all objects
+ // without new-extended alignment and size up to the allocated size.
+ uint64_t MaxFundamentalAlign =
+ std::max(TI.getLongLongAlign(), TI.getLongDoubleAlign());
+ return Ctx.toCharUnitsFromBits(std::min(
+ {DefaultNewAlign, MaxFundamentalAlign, llvm::bit_floor(AllocSize)}));
+ }
+
+ default:
+ llvm_unreachable("Unhandled DynAllocKind");
+ }
+}
+
APValue *EvalInfo::createHeapAlloc(const Expr *E, QualType T, LValue &LV) {
if (NumHeapAllocs > DynamicAllocLValue::getMaxIndex()) {
FFDiag(E, diag::note_constexpr_heap_alloc_limit_exceeded);
return nullptr;
}
- DynamicAllocLValue DA(NumHeapAllocs++);
+ DynamicAllocLValue DA(NumHeapAllocs++, DynAlloc::kindOfExpr(E));
LV.set(APValue::LValueBase::getDynamicAlloc(DA, T));
auto Result = HeapAllocs.emplace(std::piecewise_construct,
std::forward_as_tuple(DA), std::tuple<>());
@@ -7798,7 +7891,7 @@ static const FunctionDecl *getVirtualOperatorDelete(QualType T) {
/// a diagnostic and returns std::nullopt.
static std::optional<DynAlloc *> CheckDeleteKind(EvalInfo &Info, const Expr *E,
const LValue &Pointer,
- DynAlloc::Kind DeallocKind) {
+ DynAllocKind DeallocKind) {
auto PointerAsString = [&] {
return Pointer.toString(Info.Ctx, Info.Ctx.VoidPtrTy);
};
@@ -7827,7 +7920,7 @@ static std::optional<DynAlloc *> CheckDeleteKind(EvalInfo &Info, const Expr *E,
}
bool Subobject = false;
- if (DeallocKind == DynAlloc::New) {
+ if (DeallocKind == DynAllocKind::New) {
Subobject = Pointer.Designator.MostDerivedPathLength != 0 ||
Pointer.Designator.isOnePastTheEnd();
} else {
@@ -7871,7 +7964,7 @@ static bool HandleOperatorDeleteCall(EvalInfo &Info, const CallExpr *E) {
return true;
}
- if (!CheckDeleteKind(Info, E, Pointer, DynAlloc::StdAllocator))
+ if (!CheckDeleteKind(Info, E, Pointer, DynAllocKind::StdAllocator))
return false;
Info.HeapAllocs.erase(Pointer.Base.get<DynamicAllocLValue>());
@@ -10546,6 +10639,9 @@ static CharUnits getBaseAlignment(EvalInfo &Info, const LValue &Value) {
return Info.Ctx.getDeclAlign(VD);
if (const auto *E = Value.Base.dyn_cast<const Expr *>())
return GetAlignOfExpr(Info.Ctx, E, UETT_AlignOf);
+ if (const auto &DA = Value.Base.dyn_cast<DynamicAllocLValue>())
+ return GetAlignOfDynamicAlloc(Info.getASTContext(), Value.Base.getType(),
+ DA.getAllocKind());
return GetAlignOfType(Info.Ctx, Value.Base.getTypeInfoType(), UETT_AlignOf);
}
@@ -21771,7 +21867,8 @@ bool VoidExprEvaluator::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
}
std::optional<DynAlloc *> Alloc = CheckDeleteKind(
- Info, E, Pointer, E->isArrayForm() ? DynAlloc::ArrayNew : DynAlloc::New);
+ Info, E, Pointer,
+ E->isArrayForm() ? DynAllocKind::ArrayNew : DynAllocKind::New);
if (!Alloc)
return false;
QualType AllocType = Pointer.Base.getDynamicAllocType();
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index b89c97f2b8900..0d65db6f4523c 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -2633,6 +2633,7 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
} else {
Diag(TypeRange.getEnd(), diag::err_new_array_size_unknown_from_init)
<< Initializer->getSourceRange();
+ return ExprError();
}
}
}
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index e5ddd8eb2a037..a9a297a10e76c 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -1154,7 +1154,7 @@ namespace NewNegSizeNothrow {
constexpr bool test_nothrow_neg_size() {
int x = get_neg_size();
- int* p = new (std::nothrow) int[x];
+ int* p = new (std::nothrow) int[x];
return p == nullptr;
}
@@ -1291,13 +1291,13 @@ namespace FreeNonBlockPointer {
extern int f();
#define fold(x) (__builtin_constant_p(x) ? (x) : (x))
- constexpr int foo() {
+ constexpr int foo() { // expected-error {{constexpr function never produces a constant expression}}
int *p;
p = fold((int*)(void*)f);
- delete p;
+ delete p; // expected-note 2 {{delete of pointer '&f' that does not point to a heap-allocated object}}
return 10;
}
- static_assert(foo() == 10); // both-error {{not an integral constant expression}}
+ static_assert(foo() == 10); // both-error {{not an integral constant expression}} expected-note {{in call to 'foo()'}}
}
namespace NonPrimitiveImplicitValueInitExpr {
diff --git a/clang/test/SemaCXX/builtin-assume-aligned.cpp b/clang/test/SemaCXX/builtin-assume-aligned.cpp
index 30296c72c6be8..1131cbbb32966 100644
--- a/clang/test/SemaCXX/builtin-assume-aligned.cpp
+++ b/clang/test/SemaCXX/builtin-assume-aligned.cpp
@@ -1,14 +1,18 @@
-// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 -triple x86_64-linux-gnu %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx11 -std=c++11 -triple x86_64-linux-gnu %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx11 -std=c++11 -triple x86_64-linux-gnu -fexperimental-new-constant-interpreter %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx26 -std=c++26 -triple x86_64-linux-gnu %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected,cxx26 -std=c++26 -triple x86_64-linux-gnu -fexperimental-new-constant-interpreter %s
int n;
constexpr int *p = 0;
-// expected-error at +1 {{must be initialized by a constant expression}}
+// expected-error at +2 {{must be initialized by a constant expression}}
+// cxx26-note at +1 {{a constant expression cannot modify an object that is visible outside that expression}}
constexpr int *k = (int *) __builtin_assume_aligned(p, 16, n = 5);
constexpr void *l = __builtin_assume_aligned(p, 16);
-// expected-error at +2 {{must be initialized by a constant expression}}
-// expected-note at +1 {{cast from 'void *' is not allowed in a constant expression}}
+// cxx11-error at +2 {{must be initialized by a constant expression}}
+// cxx11-note at +1 {{cast from 'void *' is not allowed in a constant expression}}
constexpr int *c = (int *) __builtin_assume_aligned(p, 16);
// expected-error at +2 {{must be initialized by a constant expression}}
@@ -60,3 +64,85 @@ void AllocateAlignedBytes() {
void *m = __builtin_assume_aligned(
reinterpret_cast<void *>(AllocateAlignedBytes_payload), kAlignment);
}
+
+namespace std {
+ enum class byte : unsigned char {};
+} // namespace std
+
+namespace GH173767 {
+#if __cplusplus > 202302L
+ constexpr int a1a = (delete[] (unsigned char*)__builtin_assume_aligned(new unsigned char[65], __STDCPP_DEFAULT_NEW_ALIGNMENT__), 0);
+ constexpr int a1b = (delete[] (char*)__builtin_assume_aligned(new char[65], __STDCPP_DEFAULT_NEW_ALIGNMENT__), 0);
+ constexpr int a1c = (delete[] (std::byte*)__builtin_assume_aligned(new std::byte[65], __STDCPP_DEFAULT_NEW_ALIGNMENT__), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (16 bytes) is less than the asserted 32 bytes}}
+ constexpr int a2a = (delete[] (unsigned char*)__builtin_assume_aligned(new unsigned char[65], 2*__STDCPP_DEFAULT_NEW_ALIGNMENT__), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (16 bytes) is less than the asserted 32 bytes}}
+ constexpr int a2b = (delete[] (char*)__builtin_assume_aligned(new char[65], 2*__STDCPP_DEFAULT_NEW_ALIGNMENT__), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (16 bytes) is less than the asserted 32 bytes}}
+ constexpr int a2c = (delete[] (std::byte*)__builtin_assume_aligned(new std::byte[65], 2*__STDCPP_DEFAULT_NEW_ALIGNMENT__), 0);
+
+ constexpr int b1 = (delete (int*)__builtin_assume_aligned(new int, alignof(int)), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (4 bytes) is less than the asserted 8 bytes}}
+ constexpr int b2 = (delete (int*)__builtin_assume_aligned(new int, 2*alignof(int)), 0);
+
+ constexpr int c1 = (delete[] (int*)__builtin_assume_aligned(new int[4], alignof(int)), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (4 bytes) is less than the asserted 8 bytes}}
+ constexpr int c2 = (delete[] (int*)__builtin_assume_aligned(new int[4], 2*alignof(int)), 0);
+
+ struct D {
+ alignas(2*__STDCPP_DEFAULT_NEW_ALIGNMENT__) int x[2];
+ };
+
+ constexpr int d1 = (delete (D*)__builtin_assume_aligned(new D, alignof(D)), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (32 bytes) is less than the asserted 64 bytes}}
+ constexpr int d2 = (delete (D*)__builtin_assume_aligned(new D, 2*alignof(D)), 0);
+
+ constexpr int d3 = []{
+ auto p = new D;
+ (void)__builtin_assume_aligned(p->x + 1, alignof(int));
+ delete p;
+ return 0;
+ }();
+
+ // expected-error at +3 {{must be initialized by a constant expression}}
+ // expected-note at +2 {{in call to}}
+ // expected-note at +3 {{offset of the aligned pointer from the base pointee object (4 bytes) is not a multiple of the asserted 8 bytes}}
+ constexpr int d4 = []{
+ auto p = new D;
+ (void)__builtin_assume_aligned(p->x + 1, 2*alignof(int));
+ delete p;
+ return 0;
+ }();
+
+ struct E {
+ unsigned char x[65];
+ };
+
+ constexpr int e1 = (delete (E*)__builtin_assume_aligned(new E, 1), 0);
+ // expected-error at +2 {{must be initialized by a constant expression}}
+ // expected-note at +1 {{alignment of the base pointee object (1 byte) is less than the asserted 2 bytes}}
+ constexpr int e2 = (delete (E*)__builtin_assume_aligned(new E, 2), 0);
+
+ constexpr int f = []{
+ auto p = new int;
+ delete p;
+ (void)__builtin_assume_aligned(p, alignof(int));
+ return 0;
+ }();
+
+ // flexible array members as only member are allowed as an extension
+ // and cause the type to have size zero, but should still be aligned
+ // properly for the element type
+ struct G {
+ int x[];
+ };
+
+ constexpr int g1 = (delete (G*)__builtin_assume_aligned(new G, alignof(int)), 0);
+#endif
+} // namespace GH173767
diff --git a/clang/test/SemaCXX/constant-expression-cxx2a.cpp b/clang/test/SemaCXX/constant-expression-cxx2a.cpp
index 870865646a61e..fcc982182c06e 100644
--- a/clang/test/SemaCXX/constant-expression-cxx2a.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx2a.cpp
@@ -901,8 +901,7 @@ namespace dynamic_alloc {
// An array new-expression whose bound is neither given nor deducible from the
// initializer is ill-formed; the constant evaluator must reject it gracefully
// rather than crash. See GH200139.
- static_assert((new int[]())[0] == 0); // expected-error {{cannot determine allocated array size from initializer}} \
- // expected-error {{static assertion expression is not an integral constant expression}}
+ static_assert((new int[]())[0] == 0); // expected-error {{cannot determine allocated array size from initializer}}
constexpr bool erroneous_array_bound_nothrow(long long n) {
int *p = new (std::nothrow) int[n];
>From 9b989cbefc778d4c321677642931f88c5b79e110 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames at gmail.com>
Date: Wed, 7 Oct 2026 21:56:18 +1100
Subject: [PATCH 40/52] [examples] Set target triple on modules in
Kaleidoscope. (#229666)
If no triple is set then the object format component of the Module's
triple will be set by getDefaultFormat in Triple.cpp, typically to ELF.
In AArch64AsmPrinter::emitStartOfAsmFile, the "ELF" default will cause
control to fall through the following check:
if (!TT.isOSBinFormatELF())
return;
and into a region that assumes an MCTargetStreamer. On Darwin/arm64,
which does not set an MCTargetStreamer, this will cause a nullptr
access.
Fix the issue by setting the Module's triple to the host triple as
reported by the KaleidoscopeJIT object.
rdar://189147321
---
.../Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h | 2 ++
llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp | 1 +
.../Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h | 2 ++
llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp | 1 +
.../Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h | 2 ++
llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp | 1 +
.../Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h | 2 ++
llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp | 1 +
llvm/examples/Kaleidoscope/Chapter4/toy.cpp | 1 +
llvm/examples/Kaleidoscope/Chapter5/toy.cpp | 1 +
llvm/examples/Kaleidoscope/Chapter6/toy.cpp | 1 +
llvm/examples/Kaleidoscope/Chapter7/toy.cpp | 1 +
llvm/examples/Kaleidoscope/Chapter9/toy.cpp | 1 +
llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h | 2 ++
14 files changed, 19 insertions(+)
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
index 7800dca5fd375..59f131a7d542d 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
@@ -82,6 +82,8 @@ class KaleidoscopeJIT {
std::move(*DL));
}
+ const Triple &getTargetTriple() const { return ES->getTargetTriple(); }
+
const DataLayout &getDataLayout() const { return DL; }
JITDylib &getMainJITDylib() { return MainJD; }
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp
index 92e64b34c2150..ec19e17fa3bcf 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/toy.cpp
@@ -1105,6 +1105,7 @@ static void InitializeModule() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
index 1789ab1e20fef..0772f389eed54 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
@@ -89,6 +89,8 @@ class KaleidoscopeJIT {
std::move(*DL));
}
+ const Triple &getTargetTriple() const { return ES->getTargetTriple(); }
+
const DataLayout &getDataLayout() const { return DL; }
JITDylib &getMainJITDylib() { return MainJD; }
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp
index 92e64b34c2150..ec19e17fa3bcf 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/toy.cpp
@@ -1105,6 +1105,7 @@ static void InitializeModule() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
index 028c3afe8e9c6..5c0a0359bfa90 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
@@ -134,6 +134,8 @@ class KaleidoscopeJIT {
std::move(*MemAccess), std::move(JTMB), std::move(*DL));
}
+ const Triple &getTargetTriple() const { return ES->getTargetTriple(); }
+
const DataLayout &getDataLayout() const { return DL; }
JITDylib &getMainJITDylib() { return MainJD; }
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp
index 92e64b34c2150..ec19e17fa3bcf 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/toy.cpp
@@ -1105,6 +1105,7 @@ static void InitializeModule() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
index e4933c0ef6870..1db29dd0385c1 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
@@ -217,6 +217,8 @@ class KaleidoscopeJIT {
std::move(*MemAccess), std::move(JTMB), std::move(*DL));
}
+ const Triple &getTargetTriple() const { return ES->getTargetTriple(); }
+
const DataLayout &getDataLayout() const { return DL; }
JITDylib &getMainJITDylib() { return MainJD; }
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp
index a4cfc4b9d5b7b..28d19ff96d0d0 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/toy.cpp
@@ -1096,6 +1096,7 @@ static void InitializeModule() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
index b73dea67511a3..d21cb82d01796 100644
--- a/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter4/toy.cpp
@@ -563,6 +563,7 @@ static void InitializeModuleAndManagers() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
index c33e4da68d875..4c6cd258b5878 100644
--- a/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter5/toy.cpp
@@ -837,6 +837,7 @@ static void InitializeModuleAndManagers() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
index 18cba9447cf04..9fc8864a4df1a 100644
--- a/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter6/toy.cpp
@@ -956,6 +956,7 @@ static void InitializeModuleAndManagers() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
index 6a8c4840bf4f5..5ebec1799ad4b 100644
--- a/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter7/toy.cpp
@@ -1124,6 +1124,7 @@ static void InitializeModuleAndManagers() {
// Open a new context and module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("KaleidoscopeJIT", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
// Create a new builder for the module.
diff --git a/llvm/examples/Kaleidoscope/Chapter9/toy.cpp b/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
index 1bd7f8bc0ae49..f939c9bd43284 100644
--- a/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
@@ -1323,6 +1323,7 @@ static void InitializeModule() {
// Open a new module.
TheContext = std::make_unique<LLVMContext>();
TheModule = std::make_unique<Module>("my cool jit", *TheContext);
+ TheModule->setTargetTriple(TheJIT->getTargetTriple());
TheModule->setDataLayout(TheJIT->getDataLayout());
Builder = std::make_unique<IRBuilder<>>(*TheModule);
diff --git a/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
index 1fc89fca1e1da..853144429518e 100644
--- a/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
@@ -87,6 +87,8 @@ class KaleidoscopeJIT {
std::move(*DL));
}
+ const Triple &getTargetTriple() const { return ES->getTargetTriple(); }
+
const DataLayout &getDataLayout() const { return DL; }
JITDylib &getMainJITDylib() { return MainJD; }
>From 440cb1bc0b89620449cc0df761d3fd954840c5cc Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 7 Oct 2026 12:01:47 +0100
Subject: [PATCH 41/52] [LV] Support fcmp in uncountable exit loops with stores
(#228451)
Small addition to allow vectorization of more loops.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
.../AArch64/early_exit_with_stores.ll | 49 +++++++++++++---
.../RISCV/early_exit_with_stores.ll | 37 ++++++++----
.../early_exit_store_legality.ll | 5 +-
.../LoopVectorize/early_exit_with_stores.ll | 57 +++++++++++++------
5 files changed, 111 insertions(+), 39 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b4906d8ea879b..1ca6c754b5136 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3233,7 +3233,7 @@ getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
VPValue *Op1, *Op2;
// Walk back through recipes until we find at least one load from memory.
- if (match(V, m_ICmp(m_VPValue(Op1), m_VPValue(Op2)))) {
+ if (match(V, m_Cmp(m_VPValue(Op1), m_VPValue(Op2)))) {
Worklist.push_back(Op1);
Worklist.push_back(Op2);
Recipes.push_back(cast<VPInstruction>(V->getDefiningRecipe()));
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index faba88cee2203..09fb90912a608 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -129,9 +129,40 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-LABEL: define void @loop_contains_store_fcmp_condition(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP1]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp ugt <vscale x 8 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <vscale x 8 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP5]])
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP7]], ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 20
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -139,11 +170,11 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load half, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = fcmp ugt half [[EE_VAL]], 5.000000e+02
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -356,7 +387,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
@@ -378,7 +409,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -479,7 +510,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -500,7 +531,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1151,7 +1182,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -1172,7 +1203,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index beee8ddce5f13..ca4d4f5fc770f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -126,10 +126,27 @@ exit:
define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_fcmp_condition(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x half>, ptr [[PRED]], align 2
+; CHECK-NEXT: [[TMP0:%.*]] = fcmp ugt <16 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <16 x i1> [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP1]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[TMP2]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p0(ptr align 2 [[ARRAY]], <16 x i1> [[UNCOUNTABLE_EXIT_MASK]], <16 x i16> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <16 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v16i16.p0(<16 x i16> [[TMP3]], ptr align 2 [[ARRAY]], <16 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[TMP2]], 20
+; CHECK-NEXT: br i1 [[TMP4]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP2]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -137,11 +154,11 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load half, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = fcmp ugt half [[EE_VAL]], 5.000000e+02
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -354,7 +371,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
@@ -376,7 +393,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -522,7 +539,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -543,7 +560,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1194,7 +1211,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP1]]
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -1215,7 +1232,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 714f172d08fbf..4e376392d7d30 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -140,11 +140,10 @@ exit:
ret void
}
-;; Vectorizeable, needs work on exit condition recipe collection.
define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) !dbg !18 {
; CHECK-DEBUG-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp_condition'
-; CHECK-DEBUG: LV: Not vectorizing: Unable to determine early exit condition for loop with side effects
-; CHECK-REMARK: foo.c:50:3: loop not vectorized: Unable to determine early exit condition for loop with side effects
+; CHECK-DEBUG: LV: Loop passed LoopVectorizationLegality checks!
+; CHECK-DEBUG: LV: VPlan created successfully. Loop can be vectorized.
entry:
br label %for.body, !dbg !19
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index f02553b69daa7..f4eaaba1bebcf 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -120,10 +120,35 @@ exit:
define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_fcmp_condition(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp ugt <4 x half> [[WIDE_LOAD]], splat (half 5.000000e+02)
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP3]])
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP4]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP5]], ptr align 2 [[TMP4]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
+; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[TMP9]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -131,11 +156,11 @@ define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load half, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = fcmp ugt half [[EE_VAL]], 5.000000e+02
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -343,7 +368,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
@@ -364,7 +389,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -460,7 +485,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
@@ -480,7 +505,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1036,7 +1061,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i16> [[TMP30]], i64 3
; CHECK-NEXT: [[TMP45:%.*]] = add i64 [[TMP3]], [[TMP14]]
@@ -1057,7 +1082,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[DATA_LCSSA:%.*]] = phi i16 [ [[DATA1]], %[[FOR_INC]] ], [ [[DATA1]], %[[FOR_BODY1]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[DATA_LCSSA]]
@@ -1108,7 +1133,7 @@ define i64 @uncountable_exit_with_live_out_iv(ptr dereferenceable(40) noalias %a
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP2]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
@@ -1128,7 +1153,7 @@ define i64 @uncountable_exit_with_live_out_iv(ptr dereferenceable(40) noalias %a
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[FOR_INC]] ], [ [[IV]], %[[FOR_BODY]] ], [ 19, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
@@ -1269,7 +1294,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
@@ -1289,7 +1314,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
@@ -1412,7 +1437,7 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP3]]
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 20
@@ -1432,7 +1457,7 @@ define void @swapped_cmp_operands(ptr dereferenceable(40) noalias %array, ptr al
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
>From 3e85cb1ebc56e895c9efecf04eaad79058dbfc34 Mon Sep 17 00:00:00 2001
From: Raphael Isemann <rise at apple.com>
Date: Wed, 7 Oct 2026 12:08:28 +0100
Subject: [PATCH 42/52] [lldb] Keep the inferior's terminal open until LLDB
read its output (#229387)
lldb gives the inferior the secondary side of a pseudo terminal for its
stdio and drains the primary side (what we get back from from
posix_openpt) via ThreadedCommunication's read thread.
On Darwin, it seems that closing the last secondary descriptor clears
the terminal's contents. Any output the inferior wrote but that the read
thread has not consumed yet is then destroyed alongside the inferior.
There isn't really any guarantee that we read the inferior's output
before it dies (and closes its secondary side of the terminal). So
reading stdout is currently racy in LLDB. This is also why
TestTargetAPI.py randomly fails to read the output:
```
File ".../lldb/test/API/python_api/target/TestTargetAPI.py", line 189
self.assertIn("arg: foo", output)
AssertionError: 'arg: foo' not found in ''
```
The easiest way to make this race more likely is to sleep at the start
of ThreadedCommunication::ReadThread's main loop like this:
```
static std::chrono::milliseconds g_stdio_read_delay(800);
// in ThreadedCommunication::ReadThrea
while (!done && m_read_thread_enabled) {
// sleep if we're the stdio reader thread.
if (GetBroadcasterName() == "process.stdio")
std::this_thread::sleep_for(g_stdio_read_delay);
```
This patch fixes this by also making LLDB open the secondary side of the
terminal so that the the pty outlives the inferior. Then, LLDB can read
all stdout contents and close the secondary (and the pty).
assisted-by: claude
---
lldb/include/lldb/Target/Process.h | 32 +++++++++-
.../MacOSX/PlatformAppleSimulator.cpp | 6 +-
.../Plugins/Platform/POSIX/PlatformPOSIX.cpp | 6 +-
.../Platform/QemuUser/PlatformQemuUser.cpp | 3 +-
.../Platform/WebAssembly/PlatformWasm.cpp | 3 +-
.../Process/gdb-remote/ProcessGDBRemote.cpp | 4 ++
lldb/source/Target/Platform.cpp | 6 +-
lldb/source/Target/Process.cpp | 64 ++++++++++++++++---
8 files changed, 102 insertions(+), 22 deletions(-)
diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index 029f7f422caee..08defc57d253a 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -53,6 +53,7 @@
#include "lldb/Utility/Broadcaster.h"
#include "lldb/Utility/Event.h"
#include "lldb/Utility/Listener.h"
+#include "lldb/Utility/Locked.h"
#include "lldb/Utility/NameMatches.h"
#include "lldb/Utility/Policy.h"
#include "lldb/Utility/ProcessAddress.h"
@@ -2668,10 +2669,27 @@ void PruneThreadPlans();
/// assumed to be valid and will be managed by the newly created
/// connection.
///
+ /// \param[in] secondary_fd A descriptor for the secondary side of that same
+ /// terminal as file_descriptor. This is used to keep the terminal (and
+ /// its output buffer alive). The process takes ownership of it and keeps
+ /// it open until it stops monitoring fd.
+ ///
+ /// Pass std::nullopt if fd is not a pseudo terminal or no such
+ /// descriptor is available.
+ ///
/// \see lldb_private::Process::STDIOReadThreadBytesReceived()
/// \see lldb_private::IOHandlerProcessSTDIO
/// \see lldb_private::ConnectionFileDescriptor
- void SetSTDIOFileDescriptor(int file_descriptor);
+ void SetSTDIOFileDescriptor(int file_descriptor,
+ std::optional<int> secondary_fd = std::nullopt);
+
+#if !defined(_WIN32)
+ /// Associates the primary side of \a pty with the process' STDIO handling.
+ /// The process takes ownership of both of \a pty's descriptors.
+ ///
+ /// \see SetSTDIOFileDescriptor()
+ void SetSTDIOPseudoTerminal(PseudoTerminal &pty);
+#endif
// Add a permanent region of memory that should never be read or written to.
// This can be used to ensure that memory reads or writes to certain areas of
@@ -3525,6 +3543,9 @@ void PruneThreadPlans();
mutable std::mutex m_process_input_reader_mutex;
ThreadedCommunication m_stdio_communication;
std::recursive_mutex m_stdio_communication_mutex;
+ /// The secondary side of the pseudo terminal the inferior uses for stdio.
+ /// std::nullopt if no pseudo terminal is open.
+ Guarded<std::optional<int>, std::mutex> m_stdio_secondary_fd;
bool m_stdin_forward; /// Remember if stdin must be forwarded to remote debug
/// server
std::string m_stdout_data;
@@ -3688,6 +3709,15 @@ void PruneThreadPlans();
static void STDIOReadThreadBytesReceived(void *baton, const void *src,
size_t src_len);
+ /// Stop monitoring the process' stdio.
+ ///
+ /// \param drain
+ /// Whether to read any remaining stdio buffers before.
+ void StopSTDIOMonitoring(bool drain);
+
+ /// Close m_stdio_secondary_fd if it is still open.
+ void CloseSTDIOSecondaryFileDescriptor();
+
bool PushProcessIOHandler();
bool PopProcessIOHandler();
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
index 6a04a9a3840cc..fb8784ed8919c 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformAppleSimulator.cpp
@@ -215,9 +215,9 @@ PlatformAppleSimulator::DebugProcess(ProcessLaunchInfo &launch_info,
// been used where the secondary side was given as the file to open for
// stdin/out/err after we have already opened the primary so we can
// read/write stdin/out/err.
- int pty_fd = launch_info.GetPTY().ReleasePrimaryFileDescriptor();
- if (pty_fd != PseudoTerminal::invalid_fd) {
- process_sp->SetSTDIOFileDescriptor(pty_fd);
+ if (launch_info.GetPTY().GetPrimaryFileDescriptor() !=
+ PseudoTerminal::invalid_fd) {
+ process_sp->SetSTDIOPseudoTerminal(launch_info.GetPTY());
}
}
}
diff --git a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
index b52d4da817f49..7114588d1293a 100644
--- a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
+++ b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
@@ -493,9 +493,9 @@ lldb::ProcessSP PlatformPOSIX::DebugProcess(ProcessLaunchInfo &launch_info,
// Hook up process PTY if we have one (which we should for local debugging
// with llgs).
#ifndef _WIN32 // TODO: Implement on Windows
- int pty_fd = launch_info.GetPTY().ReleasePrimaryFileDescriptor();
- if (pty_fd != PseudoTerminal::invalid_fd) {
- process_sp->SetSTDIOFileDescriptor(pty_fd);
+ if (launch_info.GetPTY().GetPrimaryFileDescriptor() !=
+ PseudoTerminal::invalid_fd) {
+ process_sp->SetSTDIOPseudoTerminal(launch_info.GetPTY());
LLDB_LOG(log, "hooked up STDIO pty to process");
} else
LLDB_LOG(log, "not using process STDIO pty");
diff --git a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
index ea446f67e0fca..cf6450a1ba171 100644
--- a/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
+++ b/lldb/source/Plugins/Platform/QemuUser/PlatformQemuUser.cpp
@@ -238,8 +238,7 @@ lldb::ProcessSP PlatformQemuUser::DebugProcess(ProcessLaunchInfo &launch_info,
#ifndef _WIN32 // TODO: Implement on Windows
if (launch_info.GetPTY().GetPrimaryFileDescriptor() !=
PseudoTerminal::invalid_fd)
- process_sp->SetSTDIOFileDescriptor(
- launch_info.GetPTY().ReleasePrimaryFileDescriptor());
+ process_sp->SetSTDIOPseudoTerminal(launch_info.GetPTY());
#endif
return process_sp;
diff --git a/lldb/source/Plugins/Platform/WebAssembly/PlatformWasm.cpp b/lldb/source/Plugins/Platform/WebAssembly/PlatformWasm.cpp
index 2dee06e414c5b..8259a63f70dea 100644
--- a/lldb/source/Plugins/Platform/WebAssembly/PlatformWasm.cpp
+++ b/lldb/source/Plugins/Platform/WebAssembly/PlatformWasm.cpp
@@ -266,8 +266,7 @@ lldb::ProcessSP PlatformWasm::DebugProcess(ProcessLaunchInfo &launch_info,
#ifndef _WIN32
if (launch_info.GetPTY().GetPrimaryFileDescriptor() !=
PseudoTerminal::invalid_fd)
- process_sp->SetSTDIOFileDescriptor(
- launch_info.GetPTY().ReleasePrimaryFileDescriptor());
+ process_sp->SetSTDIOPseudoTerminal(launch_info.GetPTY());
#endif
return process_sp;
}
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 5d1722868cfff..ed269936d15bf 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -945,7 +945,11 @@ Status ProcessGDBRemote::DoLaunch(lldb_private::Module *exe_module,
if (!disable_stdio) {
if (pty.GetPrimaryFileDescriptor() != PseudoTerminal::invalid_fd) {
+#ifdef _WIN32
SetSTDIOFileDescriptor(pty.ReleasePrimaryFileDescriptor());
+#else
+ SetSTDIOPseudoTerminal(pty);
+#endif
}
#ifdef _WIN32
else if (m_stdin_forward) {
diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index 22dfec490ac43..4b488195ad14c 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -1155,9 +1155,9 @@ lldb::ProcessSP Platform::DebugProcess(ProcessLaunchInfo &launch_info,
// stdin/out/err after we have already opened the primary so we can
// read/write stdin/out/err.
#ifndef _WIN32
- int pty_fd = launch_info.GetPTY().ReleasePrimaryFileDescriptor();
- if (pty_fd != PseudoTerminal::invalid_fd) {
- process_sp->SetSTDIOFileDescriptor(pty_fd);
+ if (launch_info.GetPTY().GetPrimaryFileDescriptor() !=
+ PseudoTerminal::invalid_fd) {
+ process_sp->SetSTDIOPseudoTerminal(launch_info.GetPTY());
}
#endif
} else {
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 546b676f72f63..0be2d927d349a 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -12,6 +12,7 @@
#include <optional>
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/Process.h"
#include "llvm/Support/ScopedPrinter.h"
#include "llvm/Support/Threading.h"
@@ -579,6 +580,9 @@ Process::~Process() {
LLDB_LOGF(log, "%p Process::~Process()", static_cast<void *>(this));
StopPrivateStateThread();
+ // Close the secondary to make sure the pty is not retained.
+ CloseSTDIOSecondaryFileDescriptor();
+
// ThreadList::Clear() will try to acquire this process's mutex, so
// explicitly clear the thread list here to ensure that the mutex is not
// destroyed before the thread list.
@@ -3889,9 +3893,9 @@ Status Process::DestroyImpl(bool force_kill) {
DidDestroy();
StopPrivateStateThread();
}
- m_stdio_communication.StopReadThread();
- m_stdio_communication.Disconnect();
- m_stdin_forward = false;
+ // Not draining: We are tearing the process down, so we don't care about its
+ // remaining output as no one could read it.
+ StopSTDIOMonitoring(/*drain=*/false);
{
std::lock_guard<std::mutex> guard(m_process_input_reader_mutex);
@@ -3960,10 +3964,9 @@ bool Process::ShouldBroadcastEvent(Event *event_ptr) {
case eStateDetached:
case eStateExited:
case eStateUnloaded:
- m_stdio_communication.SynchronizeWithReadThread();
- m_stdio_communication.StopReadThread();
- m_stdio_communication.Disconnect();
- m_stdin_forward = false;
+ // The inferior is gone, so this is our last chance to pick up whatever it
+ // wrote before it went away.
+ StopSTDIOMonitoring(/*drain=*/true);
[[fallthrough]];
case eStateConnected:
@@ -4995,7 +4998,13 @@ void Process::STDIOReadThreadBytesReceived(void *baton, const void *src,
process->AppendSTDOUT(static_cast<const char *>(src), src_len);
}
-void Process::SetSTDIOFileDescriptor(int fd) {
+void Process::SetSTDIOFileDescriptor(int fd, std::optional<int> secondary_fd) {
+ {
+ auto stdio_secondary_fd = m_stdio_secondary_fd.Lock();
+ assert(!*stdio_secondary_fd && "process stdio is already set up");
+ *stdio_secondary_fd = secondary_fd;
+ }
+
// First set up the Read Thread for reading/handling process I/O
m_stdio_communication.SetConnection(
std::make_unique<ConnectionFileDescriptor>(fd, true));
@@ -5011,7 +5020,46 @@ void Process::SetSTDIOFileDescriptor(int fd) {
m_process_input_reader =
std::make_shared<IOHandlerProcessSTDIO>(this, fd);
}
+ } else {
+ // Nobody is going to read the primary side, so there is no point in keeping
+ // the terminal alive.
+ CloseSTDIOSecondaryFileDescriptor();
+ }
+}
+
+#if !defined(_WIN32)
+void Process::SetSTDIOPseudoTerminal(PseudoTerminal &pty) {
+ // Take a descriptor for the secondary side so it is kept alive.
+ if (llvm::Error err = pty.OpenSecondary(O_RDWR | O_NOCTTY | O_CLOEXEC))
+ LLDB_LOG_ERROR(GetLog(LLDBLog::Process), std::move(err),
+ "failed to open the secondary side of the inferior's "
+ "terminal: {0}");
+
+ std::optional<int> secondary_fd;
+ if (int fd = pty.ReleaseSecondaryFileDescriptor();
+ fd != PseudoTerminal::invalid_fd)
+ secondary_fd = fd;
+ SetSTDIOFileDescriptor(pty.ReleasePrimaryFileDescriptor(), secondary_fd);
+}
+#endif
+
+void Process::StopSTDIOMonitoring(bool drain) {
+ if (drain) {
+ // Collect everything the inferior has written.
+ m_stdio_communication.SynchronizeWithReadThread();
}
+
+ // Nothing is left to read, so let go of the terminal.
+ CloseSTDIOSecondaryFileDescriptor();
+
+ m_stdio_communication.StopReadThread();
+ m_stdio_communication.Disconnect();
+ m_stdin_forward = false;
+}
+
+void Process::CloseSTDIOSecondaryFileDescriptor() {
+ if (std::optional<int> fd = std::exchange(*m_stdio_secondary_fd.Lock(), {}))
+ llvm::sys::Process::SafelyCloseFileDescriptor(*fd);
}
bool Process::ProcessIOHandlerIsActive() {
>From 669d41170786def9f4188ea872e65471b88929e5 Mon Sep 17 00:00:00 2001
From: Raphael Isemann <rise at apple.com>
Date: Wed, 7 Oct 2026 12:11:44 +0100
Subject: [PATCH 43/52] [lldb] Move DemangledNameInfo cache out of Mangled
class (#225332)
This patch moves the per-Mangled DemangledInfoCache into a global
cache with a size limit. The motivation for this is:
1. It makes `Mangled` a smaller which saves memory as its a frequently
heap-allocated object.
2. The current Mangled class is sometimes accessed via the SB API from
multiple threads, and the stateful cache in Symbol causes crashes due
to the inherent race conditions of this approach.
3. It avoids potentially large memory usage in case many symbols have
their DemangledNameInfo computed and cached.
The size of the global cache can be changed via the new
'symbols.demangled-name-info-cache-size' setting. Setting this to 0
disables the cache.
For the caching strategy, the current implementation drops a random
entry whenever the cache is full. Note that we can't use an LRU/LFU
cache as this won't allow us to use a read-write lock (as we would
need to update the use counter on a read).
Also, this patch doesn't make Symbol or Mangled fully thread-safe. It
just avoid the racy allocations.
The default size of 4096 is arbitrarily chosen. I don't expect the
average user to fancy-print more than a few hundred unique
functions in a debug session.
---
lldb/include/lldb/Core/DemangledNameInfo.h | 45 +++++
lldb/include/lldb/Core/Mangled.h | 58 ++-----
lldb/include/lldb/Core/ModuleList.h | 1 +
lldb/include/lldb/Symbol/Symbol.h | 2 +-
lldb/source/Core/CoreProperties.td | 4 +
lldb/source/Core/DemangledNameInfo.cpp | 76 +++++++++
lldb/source/Core/Mangled.cpp | 48 +++---
lldb/source/Core/ModuleList.cpp | 6 +
.../Language/CPlusPlus/CPlusPlusLanguage.cpp | 11 +-
lldb/source/Symbol/Symbol.cpp | 2 +-
lldb/source/Symbol/Symtab.cpp | 2 +-
lldb/unittests/Core/MangledTest.cpp | 159 +++++++++++++++---
12 files changed, 309 insertions(+), 105 deletions(-)
diff --git a/lldb/include/lldb/Core/DemangledNameInfo.h b/lldb/include/lldb/Core/DemangledNameInfo.h
index ac08e091d902b..a9b5572d279ec 100644
--- a/lldb/include/lldb/Core/DemangledNameInfo.h
+++ b/lldb/include/lldb/Core/DemangledNameInfo.h
@@ -9,14 +9,22 @@
#ifndef LLDB_CORE_DEMANGLEDNAMEINFO_H
#define LLDB_CORE_DEMANGLEDNAMEINFO_H
+#include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/Locked.h"
+#include "llvm/ADT/DenseMap.h"
#include "llvm/Demangle/ItaniumDemangle.h"
#include "llvm/Demangle/Utility.h"
#include <cstddef>
+#include <optional>
+#include <random>
#include <utility>
+#include <vector>
namespace lldb_private {
+class Mangled;
+
/// Stores information about where certain portions of a demangled
/// function name begin and end.
struct DemangledNameInfo {
@@ -132,6 +140,43 @@ struct DemangledNameInfo {
bool hasSuffix() const { return SuffixRange.second >= SuffixRange.first; }
};
+/// A cache from mangled names to DemangledNameInfo.
+class DemangledNameInfoCache {
+public:
+ /// Calculates the DemangledNameInfo and caches the result.
+ ///
+ /// \return std::nullopt if no info could be computed for \c mangled.
+ std::optional<DemangledNameInfo> Get(const Mangled &mangled);
+
+ void Clear();
+
+ size_t GetMaxEntries() const;
+
+ size_t GetNumEntries() const;
+
+ /// Changes the maximum number of entries. A limit of 0 disables the cache.
+ void SetMaxEntries(size_t max_entries);
+
+private:
+ struct State {
+ /// Drops a random entry from the cache.
+ void EvictRandomEntry();
+
+ /// Maps a ConstString to its DemangledNameInfo.
+ /// Values can be empty if no name info could be computed for a name.
+ llvm::DenseMap<ConstString, std::optional<DemangledNameInfo>> infos;
+
+ /// The mangled names.
+ std::vector<ConstString> mangled_names;
+
+ std::minstd_rand rng;
+
+ size_t max_entries = 0;
+ };
+
+ Guarded<State> m_state;
+};
+
/// An OutputBuffer which keeps a record of where certain parts of a
/// demangled name begin/end (e.g., basename, scope, argument list, etc.).
/// The tracking occurs during printing of the Itanium demangle tree.
diff --git a/lldb/include/lldb/Core/Mangled.h b/lldb/include/lldb/Core/Mangled.h
index 1c7bbeb05a37d..622c0f82f49f7 100644
--- a/lldb/include/lldb/Core/Mangled.h
+++ b/lldb/include/lldb/Core/Mangled.h
@@ -17,7 +17,7 @@
#include "llvm/ADT/StringRef.h"
#include <cstddef>
-#include <memory>
+#include <optional>
namespace lldb_private {
@@ -53,28 +53,6 @@ class Mangled {
/// Initialize with both mangled and demangled names empty.
Mangled() = default;
- Mangled(const Mangled &other)
- : m_mangled(other.m_mangled), m_demangled(other.m_demangled),
- m_demangled_info(
- other.m_demangled_info
- ? std::make_unique<DemangledNameInfo>(*other.m_demangled_info)
- : nullptr) {}
-
- Mangled &operator=(const Mangled &other) {
- if (this != &other) {
- m_mangled = other.m_mangled;
- m_demangled = other.m_demangled;
- m_demangled_info =
- other.m_demangled_info
- ? std::make_unique<DemangledNameInfo>(*other.m_demangled_info)
- : nullptr;
- }
- return *this;
- }
-
- Mangled(Mangled &&) = default;
- Mangled &operator=(Mangled &&) = default;
-
/// Construct with name.
///
/// Constructor with an optional string and auto-detect if \a name is
@@ -157,15 +135,9 @@ class Mangled {
/// A const reference to the display demangled name string object.
ConstString GetDisplayDemangledName() const;
- void SetDemangledName(ConstString name) {
- m_demangled = name;
- m_demangled_info.reset();
- }
+ void SetDemangledName(ConstString name) { m_demangled = name; }
- void SetMangledName(ConstString name) {
- m_mangled = name;
- m_demangled_info.reset();
- }
+ void SetMangledName(ConstString name) { m_mangled = name; }
/// Mangled name get accessor.
///
@@ -290,8 +262,15 @@ class Mangled {
/// table offsets in the cache data.
void Encode(DataEncoder &encoder, ConstStringTable &strtab) const;
- /// Retrieve \c DemangledNameInfo of the demangled name held by this object.
- const DemangledNameInfo *GetDemangledInfo() const;
+ /// Compute the \c DemangledNameInfo of the demangled name.
+ ///
+ /// Note that this always re-runs the demangler. \see DemangledNameInfoCache
+ /// for caching the result.
+ ///
+ /// \return
+ /// std::nullopt if no info could be computed (for example because the
+ /// name is mangled with a scheme that doesn't provide any info).
+ std::optional<DemangledNameInfo> ComputeDemangledInfo() const;
/// Compute the base name (without namespace/class qualifiers) from the
/// demangled name.
@@ -305,25 +284,14 @@ class Mangled {
ConstString GetBaseName() const;
private:
- /// If \c force is \c false, this function will re-use the previously
- /// demangled name (if any). If \c force is \c true (or the mangled name
- /// on this object was not previously demangled), demangle and cache the
- /// name.
- ConstString GetDemangledNameImpl(bool force) const;
-
/// The mangled version of the name.
ConstString m_mangled;
/// Mutable so we can get it on demand with
/// a const version of this object.
mutable ConstString m_demangled;
-
- /// If available, holds information about where in \c m_demangled certain
- /// parts of the name (e.g., basename, arguments, etc.) begin and end.
- mutable std::unique_ptr<DemangledNameInfo> m_demangled_info;
};
-static_assert(sizeof(Mangled) <= 2 * sizeof(ConstString) +
- sizeof(std::unique_ptr<DemangledNameInfo>),
+static_assert(sizeof(Mangled) <= 2 * sizeof(ConstString),
"High-volume object, size of object must be increased with care");
Stream &operator<<(Stream &s, const Mangled &obj);
diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h
index aa0e7c674c1d2..4e815e8f32666 100644
--- a/lldb/include/lldb/Core/ModuleList.h
+++ b/lldb/include/lldb/Core/ModuleList.h
@@ -113,6 +113,7 @@ class ModuleListProperties : public Properties {
FileSpec GetLLDBIndexCachePath() const;
bool SetLLDBIndexCachePath(const FileSpec &path);
bool GetLoadSymbolOnDemand() const;
+ uint64_t GetDemangledNameInfoCacheSize() const;
lldb::SymbolDownload GetSymbolAutoDownload() const;
PathMappingList GetSymlinkMappings() const;
};
diff --git a/lldb/include/lldb/Symbol/Symbol.h b/lldb/include/lldb/Symbol/Symbol.h
index 184e1ca9bdb63..700b29b23a747 100644
--- a/lldb/include/lldb/Symbol/Symbol.h
+++ b/lldb/include/lldb/Symbol/Symbol.h
@@ -458,7 +458,7 @@ class Symbol : public SymbolContextScope {
#if __SIZEOF_POINTER__ == 8
static_assert(
- sizeof(lldb_private::Symbol) == 80,
+ sizeof(lldb_private::Symbol) == 72,
"Symbol is a high volume data type, size must be increased with care");
#endif
diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td
index 268d5da19fadd..2a9398ed8325a 100644
--- a/lldb/source/Core/CoreProperties.td
+++ b/lldb/source/Core/CoreProperties.td
@@ -51,6 +51,10 @@ let Definition = "modulelist", Path = "symbols" in {
Global,
DefaultFalse,
Desc<"Enable on demand symbol loading in LLDB. LLDB will load debug info on demand for each module based on various conditions (e.g. matched breakpoint, resolved stack frame addresses and matched global variables/function symbols in symbol table) to improve performance. Please refer to docs/use/ondemand.rst for details.">;
+ def DemangledNameInfoCacheSize: Property<"demangled-name-info-cache-size", "UInt64">,
+ Global,
+ DefaultUnsignedValue<4096>,
+ Desc<"The maximum number of formatted function names to cache. A value of 0 disables the cache.">;
}
let Definition = "diagnostics", Path = "diagnostics" in {
diff --git a/lldb/source/Core/DemangledNameInfo.cpp b/lldb/source/Core/DemangledNameInfo.cpp
index 16fbfda299b21..0f69947548d52 100644
--- a/lldb/source/Core/DemangledNameInfo.cpp
+++ b/lldb/source/Core/DemangledNameInfo.cpp
@@ -7,11 +7,87 @@
//===----------------------------------------------------------------------===//
#include "lldb/Core/DemangledNameInfo.h"
+#include "lldb/Core/Mangled.h"
using namespace llvm::itanium_demangle;
namespace lldb_private {
+std::optional<DemangledNameInfo>
+DemangledNameInfoCache::Get(const Mangled &mangled) {
+ ConstString mangled_name = mangled.GetMangledName();
+ if (!mangled_name)
+ return std::nullopt;
+
+ bool cache_enabled = false;
+ {
+ auto state = m_state.LockShared();
+ cache_enabled = state->max_entries != 0;
+ auto it = state->infos.find(mangled_name);
+ if (it != state->infos.end())
+ return it->second;
+ }
+
+ // Demangle outside of the lock. Two threads asking for the same name at the
+ // same time just compute the same info twice, which is cheaper than making
+ // every other thread wait for one demangle to finish.
+ std::optional<DemangledNameInfo> info = mangled.ComputeDemangledInfo();
+
+ // Caching is disabled.
+ if (!cache_enabled)
+ return info;
+
+ auto state = m_state.Lock();
+ // Another thread might have cached this name while we were demangling.
+ if (state->infos.contains(mangled_name))
+ return info;
+
+ // Make room for the new entry by dropping a random one.
+ if (state->infos.size() >= state->max_entries)
+ state->EvictRandomEntry();
+
+ state->infos.insert({mangled_name, info});
+ state->mangled_names.push_back(mangled_name);
+ return info;
+}
+
+void DemangledNameInfoCache::State::EvictRandomEntry() {
+ assert(!mangled_names.empty() && "evicting from an empty cache");
+ std::uniform_int_distribution<size_t> dist(0, mangled_names.size() - 1);
+ size_t index = dist(rng);
+ infos.erase(mangled_names[index]);
+ mangled_names[index] = mangled_names.back();
+ mangled_names.pop_back();
+}
+
+void DemangledNameInfoCache::Clear() {
+ auto state = m_state.Lock();
+ state->infos.clear();
+ state->mangled_names.clear();
+}
+
+size_t DemangledNameInfoCache::GetMaxEntries() const {
+ return m_state.LockShared()->max_entries;
+}
+
+size_t DemangledNameInfoCache::GetNumEntries() const {
+ return m_state.LockShared()->infos.size();
+}
+
+void DemangledNameInfoCache::SetMaxEntries(size_t max_entries) {
+ // This is called before every lookup, so avoid taking the exclusive lock
+ // when the limit didn't change.
+ if (GetMaxEntries() == max_entries)
+ return;
+
+ auto state = m_state.Lock();
+ state->max_entries = max_entries;
+ if (state->infos.size() > state->max_entries) {
+ state->infos.clear();
+ state->mangled_names.clear();
+ }
+}
+
bool TrackingOutputBuffer::shouldTrack() const {
if (!isPrintingTopLevelFunctionType())
return false;
diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp
index 3f278bfa40335..06aa1a97309c1 100644
--- a/lldb/source/Core/Mangled.cpp
+++ b/lldb/source/Core/Mangled.cpp
@@ -26,6 +26,7 @@
#include "llvm/Support/Compiler.h"
#include <mutex>
+#include <optional>
#include <string>
#include <string_view>
#include <utility>
@@ -112,7 +113,6 @@ Mangled::operator bool() const { return m_mangled || m_demangled; }
void Mangled::Clear() {
m_mangled.Clear();
m_demangled.Clear();
- m_demangled_info.reset();
}
// Compare the string values.
@@ -126,16 +126,13 @@ void Mangled::SetValue(ConstString name) {
if (IsMangledName(name.GetStringRef())) {
m_demangled.Clear();
m_mangled = name;
- m_demangled_info.reset();
} else {
m_demangled = name;
m_mangled.Clear();
- m_demangled_info.reset();
}
} else {
m_demangled.Clear();
m_mangled.Clear();
- m_demangled_info.reset();
}
}
@@ -281,30 +278,19 @@ bool Mangled::GetRichManglingInfo(RichManglingContext &context,
llvm_unreachable("Fully covered switch above!");
}
-ConstString Mangled::GetDemangledName() const {
- return GetDemangledNameImpl(/*force=*/false);
-}
-
-const DemangledNameInfo *Mangled::GetDemangledInfo() const {
- if (!m_demangled_info)
- GetDemangledNameImpl(/*force=*/true);
- return m_demangled_info.get();
-}
-
// Generate the demangled name on demand using this accessor. Code in this
// class will need to use this accessor if it wishes to decode the demangled
// name. The result is cached and will be kept until a new string value is
// supplied to this object, or until the end of the object's lifetime.
-ConstString Mangled::GetDemangledNameImpl(bool force) const {
+ConstString Mangled::GetDemangledName() const {
if (!m_mangled)
return m_demangled;
// Re-use previously demangled names.
- if (!force && !m_demangled.IsNull())
+ if (!m_demangled.IsNull())
return m_demangled;
- if (!force && m_mangled.GetMangledCounterpart(m_demangled) &&
- !m_demangled.IsNull())
+ if (m_mangled.GetMangledCounterpart(m_demangled) && !m_demangled.IsNull())
return m_demangled;
// We didn't already mangle this name, demangle it and if all goes well
@@ -314,14 +300,9 @@ ConstString Mangled::GetDemangledNameImpl(bool force) const {
case eManglingSchemeMSVC:
demangled_name = GetMSVCDemangledStr(m_mangled);
break;
- case eManglingSchemeItanium: {
- std::pair<char *, DemangledNameInfo> demangled =
- GetItaniumDemangledStr(m_mangled.GetCString());
- demangled_name = demangled.first;
- m_demangled_info =
- std::make_unique<DemangledNameInfo>(std::move(demangled.second));
+ case eManglingSchemeItanium:
+ demangled_name = GetItaniumDemangledStr(m_mangled.GetCString()).first;
break;
- }
case eManglingSchemeRustV0:
demangled_name = GetRustV0DemangledStr(m_mangled);
break;
@@ -351,6 +332,18 @@ ConstString Mangled::GetDemangledNameImpl(bool force) const {
return m_demangled;
}
+std::optional<DemangledNameInfo> Mangled::ComputeDemangledInfo() const {
+ // Only supported for itanium.
+ if (!m_mangled ||
+ GetManglingScheme(m_mangled.GetStringRef()) != eManglingSchemeItanium)
+ return std::nullopt;
+
+ std::pair<char *, DemangledNameInfo> demangled =
+ GetItaniumDemangledStr(m_mangled.GetCString());
+ free(demangled.first);
+ return std::move(demangled.second);
+}
+
ConstString Mangled::GetDisplayDemangledName() const {
if (Language *lang = Language::FindPlugin(GuessLanguage()))
return lang->GetDisplayDemangledName(*this);
@@ -469,7 +462,6 @@ bool Mangled::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
const StringTableReader &strtab) {
m_mangled.Clear();
m_demangled.Clear();
- m_demangled_info.reset();
MangledEncoding encoding = (MangledEncoding)data.GetU8(offset_ptr);
switch (encoding) {
case Empty:
@@ -549,8 +541,8 @@ void Mangled::Encode(DataEncoder &file, ConstStringTable &strtab) const {
}
ConstString Mangled::GetBaseName() const {
- const auto *demangled_info = GetDemangledInfo();
- if (demangled_info == nullptr)
+ std::optional<DemangledNameInfo> demangled_info = ComputeDemangledInfo();
+ if (!demangled_info)
return {};
ConstString demangled_name = GetDemangledName();
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index 5b7e15f71c26c..c6a05757af6ac 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -198,6 +198,12 @@ bool ModuleListProperties::GetLoadSymbolOnDemand() const {
idx, g_modulelist_properties[idx].default_uint_value != 0);
}
+uint64_t ModuleListProperties::GetDemangledNameInfoCacheSize() const {
+ const uint32_t idx = ePropertyDemangledNameInfoCacheSize;
+ return GetPropertyAtIndexAs<uint64_t>(
+ idx, g_modulelist_properties[idx].default_uint_value);
+}
+
ModuleList::ModuleList() : m_modules(), m_modules_mutex() {}
ModuleList::ModuleList(const ModuleList &rhs) : m_modules(), m_modules_mutex() {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index d453f9c9c4ae0..ea8e94c116a11 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -23,6 +23,7 @@
#include "lldb/Core/DemangledNameInfo.h"
#include "lldb/Core/Mangled.h"
#include "lldb/Core/Module.h"
+#include "lldb/Core/ModuleList.h"
#include "lldb/Core/PluginManager.h"
#include "lldb/Core/UniqueCStringMap.h"
#include "lldb/DataFormatters/CXXFunctionPointer.h"
@@ -257,6 +258,14 @@ static bool PrettyPrintFunctionNameWithArgs(Stream &out_stream,
return true;
}
+static DemangledNameInfoCache &GetDemangledInfoCache() {
+ static DemangledNameInfoCache g_cache;
+ // Pick up any changes to the size the user made since the last call.
+ g_cache.SetMaxEntries(ModuleList::GetGlobalModuleListProperties()
+ .GetDemangledNameInfoCacheSize());
+ return g_cache;
+}
+
static llvm::Expected<std::pair<llvm::StringRef, DemangledNameInfo>>
GetAndValidateInfo(const SymbolContext &sc) {
Mangled mangled = sc.GetPossiblyInlinedFunctionName();
@@ -269,7 +278,7 @@ GetAndValidateInfo(const SymbolContext &sc) {
"function '{0}' does not have a demangled name",
mangled.GetMangledName());
- const DemangledNameInfo *info = mangled.GetDemangledInfo();
+ std::optional<DemangledNameInfo> info = GetDemangledInfoCache().Get(mangled);
if (!info)
return llvm::createStringErrorV(
"function '{0}' does not have demangled info", demangled_name);
diff --git a/lldb/source/Symbol/Symbol.cpp b/lldb/source/Symbol/Symbol.cpp
index 99be6e5186101..e96d6d371798c 100644
--- a/lldb/source/Symbol/Symbol.cpp
+++ b/lldb/source/Symbol/Symbol.cpp
@@ -771,7 +771,7 @@ bool Symbol::Decode(const DataExtractor &data, lldb::offset_t *offset_ptr,
// the DataFileCache version number in Symtab::Encode
// will need to be incremented as well.
#if __SIZEOF_POINTER__ == 8
-static_assert(sizeof(lldb_private::Symbol) == 80,
+static_assert(sizeof(lldb_private::Symbol) == 72,
"Symbol size has changed, Symbol::Encode and Decode likely need "
"to be updated");
#endif
diff --git a/lldb/source/Symbol/Symtab.cpp b/lldb/source/Symbol/Symtab.cpp
index 0b151d7497f3a..d79b8dc33219d 100644
--- a/lldb/source/Symbol/Symtab.cpp
+++ b/lldb/source/Symbol/Symtab.cpp
@@ -1234,7 +1234,7 @@ constexpr uint32_t CURRENT_CACHE_VERSION = 3;
// CURRENT_CACHE_VERSION number will need to be incremented.
#if __SIZEOF_POINTER__ == 8
static_assert(
- sizeof(lldb_private::Symbol) == 80,
+ sizeof(lldb_private::Symbol) == 72,
"Symbol size has changed, DataFileCache version likely needs updating");
#endif
diff --git a/lldb/unittests/Core/MangledTest.cpp b/lldb/unittests/Core/MangledTest.cpp
index dc0ecde94b174..8a7c46d91d944 100644
--- a/lldb/unittests/Core/MangledTest.cpp
+++ b/lldb/unittests/Core/MangledTest.cpp
@@ -353,78 +353,181 @@ static bool NameInfoEquals(const DemangledNameInfo &lhs,
rhs.QualifiersRange);
}
-TEST(MangledTest, DemangledNameInfo_SetMangledResets) {
+TEST(MangledTest, DemangledNameInfo_SetMangled) {
Mangled mangled;
- EXPECT_EQ(mangled.GetDemangledInfo(), nullptr);
+ EXPECT_EQ(mangled.ComputeDemangledInfo(), std::nullopt);
mangled.SetMangledName(ConstString("_Z3foov"));
ASSERT_TRUE(mangled);
- ASSERT_NE(mangled.GetDemangledInfo(), nullptr);
// Keep a copy of the original demangled info.
- DemangledNameInfo info1 = *mangled.GetDemangledInfo();
- EXPECT_TRUE(info1.hasBasename());
+ std::optional<DemangledNameInfo> info1 = mangled.ComputeDemangledInfo();
+ ASSERT_NE(info1, std::nullopt);
+ EXPECT_TRUE(info1->hasBasename());
mangled.SetMangledName(ConstString("_Z4funcv"));
// Should have re-calculated demangled-info since mangled name changed.
- ASSERT_NE(mangled.GetDemangledInfo(), nullptr);
- DemangledNameInfo info2 = *mangled.GetDemangledInfo();
- EXPECT_TRUE(info2.hasBasename());
+ std::optional<DemangledNameInfo> info2 = mangled.ComputeDemangledInfo();
+ ASSERT_NE(info2, std::nullopt);
+ EXPECT_TRUE(info2->hasBasename());
- EXPECT_FALSE(NameInfoEquals(info1, info2));
+ EXPECT_FALSE(NameInfoEquals(*info1, *info2));
EXPECT_EQ(mangled.GetDemangledName(), "func()");
}
-TEST(MangledTest, DemangledNameInfo_SetDemangledResets) {
+TEST(MangledTest, DemangledNameInfo_SetDemangled) {
Mangled mangled("_Z3foov");
ASSERT_TRUE(mangled);
+ std::optional<DemangledNameInfo> info = mangled.ComputeDemangledInfo();
+ ASSERT_NE(info, std::nullopt);
+
mangled.SetDemangledName(ConstString(""));
- // Mangled name hasn't changed, so GetDemangledInfo causes re-demangling
- // of previously set mangled name.
- EXPECT_NE(mangled.GetDemangledInfo(), nullptr);
- EXPECT_EQ(mangled.GetDemangledName(), "foo()");
+ // The info is computed from the mangled name, which hasn't changed.
+ std::optional<DemangledNameInfo> new_info = mangled.ComputeDemangledInfo();
+ ASSERT_NE(new_info, std::nullopt);
+ EXPECT_TRUE(NameInfoEquals(*new_info, *info));
}
TEST(MangledTest, DemangledNameInfo_Clear) {
Mangled mangled("_Z3foov");
ASSERT_TRUE(mangled);
- EXPECT_NE(mangled.GetDemangledInfo(), nullptr);
+ EXPECT_NE(mangled.ComputeDemangledInfo(), std::nullopt);
mangled.Clear();
- EXPECT_EQ(mangled.GetDemangledInfo(), nullptr);
+ EXPECT_EQ(mangled.ComputeDemangledInfo(), std::nullopt);
}
TEST(MangledTest, DemangledNameInfo_SetValue) {
Mangled mangled("_Z4funcv");
ASSERT_TRUE(mangled);
- ASSERT_NE(mangled.GetDemangledInfo(), nullptr);
// Keep a copy of the original demangled info.
- DemangledNameInfo demangled_func = *mangled.GetDemangledInfo();
+ std::optional<DemangledNameInfo> demangled_func =
+ mangled.ComputeDemangledInfo();
+ ASSERT_NE(demangled_func, std::nullopt);
- // SetValue(mangled) resets demangled-info
+ // SetValue(mangled) re-computes demangled-info
mangled.SetValue(ConstString("_Z3foov"));
- ASSERT_NE(mangled.GetDemangledInfo(), nullptr);
- DemangledNameInfo demangled_foo = *mangled.GetDemangledInfo();
- EXPECT_FALSE(NameInfoEquals(demangled_foo, demangled_func));
+ std::optional<DemangledNameInfo> demangled_foo =
+ mangled.ComputeDemangledInfo();
+ ASSERT_NE(demangled_foo, std::nullopt);
+ EXPECT_FALSE(NameInfoEquals(*demangled_foo, *demangled_func));
- // SetValue(demangled) resets demangled-info
+ // SetValue(demangled) re-computes demangled-info
mangled.SetValue(ConstString("_Z4funcv"));
- EXPECT_TRUE(NameInfoEquals(*mangled.GetDemangledInfo(), demangled_func));
+ ASSERT_NE(mangled.ComputeDemangledInfo(), std::nullopt);
+ EXPECT_TRUE(NameInfoEquals(*mangled.ComputeDemangledInfo(), *demangled_func));
- // SetValue(empty) resets demangled-info
+ // SetValue(empty) leaves nothing to compute demangled-info from
mangled.SetValue(ConstString());
- EXPECT_EQ(mangled.GetDemangledInfo(), nullptr);
+ EXPECT_EQ(mangled.ComputeDemangledInfo(), std::nullopt);
// Demangling invalid mangled name will set demangled-info
// (without a valid basename).
mangled.SetValue(ConstString("_Zinvalid"));
- ASSERT_NE(mangled.GetDemangledInfo(), nullptr);
- EXPECT_FALSE(mangled.GetDemangledInfo()->hasBasename());
+ ASSERT_NE(mangled.ComputeDemangledInfo(), std::nullopt);
+ EXPECT_FALSE(mangled.ComputeDemangledInfo()->hasBasename());
+}
+
+/// Returns the mangled name of the function 'void foo<i>()'.
+static Mangled MakeUniqueFunctionName(unsigned i) {
+ std::string basename = "foo" + std::to_string(i);
+ return Mangled("_Z" + std::to_string(basename.size()) + basename + "v");
+}
+
+TEST(MangledTest, DemangledNameInfoCache_Get) {
+ DemangledNameInfoCache cache;
+
+ Mangled func("_Z4funcv");
+ Mangled foo("_Z3foov");
+
+ std::optional<DemangledNameInfo> func_info = cache.Get(func);
+ ASSERT_NE(func_info, std::nullopt);
+ ASSERT_TRUE(func_info->hasBasename());
+
+ // Repeated lookups (of both the same and a copied object) hand out the same
+ // info.
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(func), *func_info));
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(Mangled("_Z4funcv")), *func_info));
+
+ // Different names get their own info.
+ std::optional<DemangledNameInfo> foo_info = cache.Get(foo);
+ ASSERT_NE(foo_info, std::nullopt);
+ EXPECT_FALSE(NameInfoEquals(*foo_info, *func_info));
+
+ // Names that aren't mangled don't return any name info.
+ Mangled not_mangled("not_mangled");
+ EXPECT_EQ(cache.Get(not_mangled), std::nullopt);
+ EXPECT_EQ(cache.Get(not_mangled), std::nullopt);
+
+ cache.Clear();
+
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(func), *func_info));
+}
+
+TEST(MangledTest, DemangledNameInfoCache_MaxEntries) {
+ DemangledNameInfoCache cache;
+ cache.SetMaxEntries(4);
+
+ std::optional<DemangledNameInfo> expected = cache.Get(Mangled("_Z4funcv"));
+ ASSERT_NE(expected, std::nullopt);
+
+ // Overflow the cache a few times over. It drops entries to stay within its
+ // limit, but never hands out anything but the correct info.
+ for (unsigned i = 0; i < 4 * cache.GetMaxEntries(); ++i) {
+ cache.Get(MakeUniqueFunctionName(i));
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(Mangled("_Z4funcv")), *expected));
+ EXPECT_LE(cache.GetNumEntries(), cache.GetMaxEntries());
+ }
+
+ // Only a single entry is dropped to make room, so a full cache stays full.
+ EXPECT_EQ(cache.GetNumEntries(), cache.GetMaxEntries());
+
+ // Further lookups keep the cache at its limit.
+ cache.Get(Mangled("_Z4funcv"));
+ EXPECT_EQ(cache.GetNumEntries(), cache.GetMaxEntries());
+
+ cache.Clear();
+ EXPECT_EQ(cache.GetNumEntries(), 0u);
+}
+
+TEST(MangledTest, DemangledNameInfoCache_SetMaxEntries) {
+ DemangledNameInfoCache cache;
+ cache.SetMaxEntries(4);
+
+ std::optional<DemangledNameInfo> expected = cache.Get(Mangled("_Z4funcv"));
+ ASSERT_NE(expected, std::nullopt);
+
+ cache.SetMaxEntries(8);
+ EXPECT_EQ(cache.GetMaxEntries(), 8u);
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(Mangled("_Z4funcv")), *expected));
+
+ // Fill the cache with 8 unique functions.
+ for (unsigned i = 0; i < 8; ++i) {
+ cache.Get(MakeUniqueFunctionName(i));
+ }
+ EXPECT_EQ(cache.GetNumEntries(), 8u);
+
+ // Growing the limit keeps all entries.
+ cache.SetMaxEntries(16);
+ EXPECT_EQ(cache.GetNumEntries(), 8u);
+
+ // Shrinking the limit below the number of entries clears the cache.
+ cache.SetMaxEntries(2);
+ EXPECT_EQ(cache.GetNumEntries(), 0u);
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(Mangled("_Z4funcv")), *expected));
+
+ // A limit of 0 disables the cache, but lookups still compute the info.
+ cache.SetMaxEntries(0);
+ EXPECT_EQ(cache.GetMaxEntries(), 0u);
+ EXPECT_EQ(cache.GetNumEntries(), 0u);
+ EXPECT_TRUE(NameInfoEquals(*cache.Get(Mangled("_Z4funcv")), *expected));
+ EXPECT_EQ(cache.Get(Mangled("not_mangled")), std::nullopt);
+ EXPECT_EQ(cache.GetNumEntries(), 0u);
}
struct DemanglingPartsTestCase {
>From 21ea23e825432d183ceb43ff74cd1c08a26faf57 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac at arm.com>
Date: Wed, 7 Oct 2026 12:25:51 +0100
Subject: [PATCH 44/52] [AArch64][FMV] Skip unreachable versions before
emitting feature checks (#229120)
Currently there are checks emitted for unreachable features whose
results are then ignored. This results in a lot of unnecessary IR being
emitted. This patch removes that code, by moving the check for
unreachable version to the top of the loop.
---
clang/lib/CodeGen/CodeGenFunction.cpp | 8 ++++----
.../CodeGen/AArch64/fmv-unreachable-version.c | 18 +++---------------
2 files changed, 7 insertions(+), 19 deletions(-)
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 62dade6cf6f49..72a5808cb6229 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -3229,6 +3229,10 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
for (const FMVResolverOption &RO : Options) {
+ // Skip unreachable versions.
+ if (RO.Function == nullptr)
+ continue;
+
Builder.SetInsertPoint(CurBlock);
llvm::Value *Condition = FormAArch64ResolverCondition(RO);
@@ -3246,10 +3250,6 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
Builder.SetInsertPoint(CurBlock);
}
- // Skip unreachable versions.
- if (RO.Function == nullptr)
- continue;
-
llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
CGBuilderTy RetBuilder(CGM, RetBlock);
CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
diff --git a/clang/test/CodeGen/AArch64/fmv-unreachable-version.c b/clang/test/CodeGen/AArch64/fmv-unreachable-version.c
index ffb3cb986913f..ca6ce76929713 100644
--- a/clang/test/CodeGen/AArch64/fmv-unreachable-version.c
+++ b/clang/test/CodeGen/AArch64/fmv-unreachable-version.c
@@ -56,25 +56,13 @@ __attribute__((target_clones("sve2;priority=4", "aes+sve2;priority=3", "lse;prio
// CHECK-NEXT: ret ptr @unreachable_versions._Msve
// CHECK: resolver_else:
// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 69793284352
-// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 69793284352
+// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 128
+// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 128
// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 69793317632
-// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 69793317632
-// CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 128
-// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 128
-// CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]]
-// CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
// CHECK: resolver_return1:
// CHECK-NEXT: ret ptr @unreachable_versions._Mlse
// CHECK: resolver_else2:
-// CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 576460752303423616
-// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 576460752303423616
-// CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]]
// CHECK-NEXT: ret ptr @unreachable_versions.default
//
//.
>From c1e0a94abdaade63dc8503f0add566ac5c463dfe Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Wed, 7 Oct 2026 13:30:26 +0200
Subject: [PATCH 45/52] AMDGPU/GlobalISel: Fix selecting i16 sext atomic loads
on gfx6/gfx7 (#229723)
An atomic global sextload from i16 to i32 was incorrectly matched to
buffer_load_sbyte. The correct pattern to buffer_load_sshort is already
there, so remove the incorrect duplicate which was arbitrarily preferred.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/BUFInstructions.td | 1 -
llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 6cb0f77e0dbeb..307d3afdfd447 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1076,7 +1076,6 @@ defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, atomic_load_sext_8_global>;
-defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, atomic_load_sext_16_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
index f555a2073b46c..09c61795666bf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_load_global.ll
@@ -261,7 +261,7 @@ define i32 @atomic_load_global_monotonic_i16_sext_to_i32(ptr addrspace(1) %ptr)
; GFX6-NEXT: s_mov_b32 s6, 0
; GFX6-NEXT: s_mov_b32 s7, 0x100f000
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: buffer_load_sbyte v0, v[0:1], s[4:7], 0 addr64 glc
+; GFX6-NEXT: buffer_load_sshort v0, v[0:1], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
>From eecf8d525260e725df23848a777cf7bf662493c7 Mon Sep 17 00:00:00 2001
From: Sergey Semenov <sergey.semenov at intel.com>
Date: Wed, 7 Oct 2026 13:40:06 +0200
Subject: [PATCH 46/52] [libsycl] Fix prefetch pointer argument constness
(#228442)
---
libsycl/include/sycl/__impl/queue.hpp | 6 +++---
libsycl/src/detail/queue_impl.cpp | 2 +-
libsycl/src/detail/queue_impl.hpp | 2 +-
libsycl/src/queue.cpp | 2 +-
libsycl/unittests/queue/prefetch.cpp | 2 +-
5 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/libsycl/include/sycl/__impl/queue.hpp b/libsycl/include/sycl/__impl/queue.hpp
index cbcba218b104e..286f0d6d63672 100644
--- a/libsycl/include/sycl/__impl/queue.hpp
+++ b/libsycl/include/sycl/__impl/queue.hpp
@@ -637,7 +637,7 @@ class _LIBSYCL_EXPORT queue : private detail::KernelSubmissionBase<queue> {
/// \param ptr is a USM pointer to the memory to be prefetched to the device.
/// \param numBytes is a number of bytes to be prefetched.
/// \return an event representing prefetch operation.
- event prefetch(void *ptr, std::size_t numBytes) {
+ event prefetch(const void *ptr, std::size_t numBytes) {
return prefetch(ptr, numBytes, std::vector<event>{});
}
@@ -649,7 +649,7 @@ class _LIBSYCL_EXPORT queue : private detail::KernelSubmissionBase<queue> {
/// \param numBytes is a number of bytes to be prefetched.
/// \param depEvent is an event that specifies the kernel dependencies.
/// \return an event representing prefetch operation.
- event prefetch(void *ptr, std::size_t numBytes, event depEvent) {
+ event prefetch(const void *ptr, std::size_t numBytes, event depEvent) {
return prefetch(ptr, numBytes, std::vector<event>{depEvent});
}
@@ -662,7 +662,7 @@ class _LIBSYCL_EXPORT queue : private detail::KernelSubmissionBase<queue> {
/// \param depEvents is a vector of events that specify the kernel
/// dependencies.
/// \return an event representing prefetch operation.
- event prefetch(void *ptr, std::size_t numBytes,
+ event prefetch(const void *ptr, std::size_t numBytes,
const std::vector<event> &depEvents);
private:
diff --git a/libsycl/src/detail/queue_impl.cpp b/libsycl/src/detail/queue_impl.cpp
index aa9dca26038ff..8fb0eed67ffcd 100644
--- a/libsycl/src/detail/queue_impl.cpp
+++ b/libsycl/src/detail/queue_impl.cpp
@@ -231,7 +231,7 @@ EventImplPtr QueueImpl::fill(void *Ptr, const void *Pattern,
return createEvent();
}
-EventImplPtr QueueImpl::prefetch(void *Ptr, std::size_t NumBytes,
+EventImplPtr QueueImpl::prefetch(const void *Ptr, std::size_t NumBytes,
const std::vector<EventImplPtr> &DepEvents) {
assert(MContext && "Context impl ptr can't be nullptr");
checkEventsPlatformMatch(DepEvents, *MContext);
diff --git a/libsycl/src/detail/queue_impl.hpp b/libsycl/src/detail/queue_impl.hpp
index 34fc6391ae92b..c4dd79972aa54 100644
--- a/libsycl/src/detail/queue_impl.hpp
+++ b/libsycl/src/detail/queue_impl.hpp
@@ -167,7 +167,7 @@ class QueueImpl : public std::enable_shared_from_this<QueueImpl> {
/// \param NumBytes is a number of bytes to be prefetched.
/// \param DepEvents is a vector of dependencies for the operation.
/// \return an event impl object that represents the status of the operation.
- EventImplPtr prefetch(void *Ptr, std::size_t NumBytes,
+ EventImplPtr prefetch(const void *Ptr, std::size_t NumBytes,
const std::vector<EventImplPtr> &DepEvents);
private:
diff --git a/libsycl/src/queue.cpp b/libsycl/src/queue.cpp
index 99edb2fdb2ec6..4487bbddfdc2b 100644
--- a/libsycl/src/queue.cpp
+++ b/libsycl/src/queue.cpp
@@ -55,7 +55,7 @@ event queue::memcpy(void *dest, const void *src, std::size_t numBytes,
return detail::createSyclObjFromImpl<event>(Event);
}
-event queue::prefetch(void *ptr, std::size_t numBytes,
+event queue::prefetch(const void *ptr, std::size_t numBytes,
const std::vector<event> &depEvents) {
detail::EventImplPtr Event =
impl->prefetch(ptr, numBytes, detail::getSyclObjImpls(depEvents));
diff --git a/libsycl/unittests/queue/prefetch.cpp b/libsycl/unittests/queue/prefetch.cpp
index 32ffd2e1760ec..3990d50dd6947 100644
--- a/libsycl/unittests/queue/prefetch.cpp
+++ b/libsycl/unittests/queue/prefetch.cpp
@@ -25,7 +25,7 @@ TEST(Queue, TwoPrefetches) {
mock::MockWrapper Mock;
queue Q;
- void *Ptr = reinterpret_cast<void *>(1);
+ const void *Ptr = reinterpret_cast<void *>(1);
constexpr ol_mem_migration_flags_t ExpectedFlag =
OL_MEM_MIGRATION_FLAG_HOST_TO_DEVICE;
>From c51cabd0738f75c51cc3e4ddbd8e2766c586647b Mon Sep 17 00:00:00 2001
From: "forking-google-bazel-bot[bot]"
<265904573+forking-google-bazel-bot[bot]@users.noreply.github.com>
Date: Wed, 7 Oct 2026 13:42:07 +0200
Subject: [PATCH 47/52] [Bazel] Fixes abbf695 (#229726)
This fixes abbf6958bb8449ea38dca4c6b843aad76e82670f (#174549).
Buildkite error link:
https://buildkite.com/llvm-project/upstream-bazel/builds?commit=abbf6958bb8449ea38dca4c6b843aad76e82670f
Co-authored-by: Google Bazel Bot <google-bazel-bot at google.com>
---
utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 1 +
1 file changed, 1 insertion(+)
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index dafe982730a18..dd5639ec2e2e7 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1133,6 +1133,7 @@ cc_library(
# least bad approach. Using `includes` is *specifically* problematic for
# this library because it contains files that collide easily with system
# headers such as `CXXABI.h`.
+ "-I$(WORKSPACE_ROOT)/clang/lib/AST",
"-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST",
"-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST/ByteCode",
],
>From d427e395cd1b80ca4091e4da29cd3ab0354260b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vedran=20Mileti=C4=87?= <vedran at miletic.net>
Date: Wed, 7 Oct 2026 13:42:49 +0200
Subject: [PATCH 48/52] [Offload] Avoid global mutex destructors (#221471)
Replace namespace-scope std::mutex instances that trigger
-Wglobal-constructors with destruction-free storage.
This allows the offload libraries to build with
-Werror=global-constructors.
---
offload/liboffload/src/OffloadImpl.cpp | 13 ++++++++-----
offload/libomptarget/OffloadRTL.cpp | 9 ++++++---
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index cff0b53307579..2f937123075ca 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -301,9 +301,12 @@ struct OffloadContext;
// This pointer is non-null if and only if the context is valid and fully
// initialized
static std::atomic<OffloadContext *> OffloadContextVal;
-std::mutex OffloadContextValMutex;
+static std::mutex &getOffloadContextMutex() {
+ static std::mutex Mutex;
+ return Mutex;
+}
// Set once the vendor runtimes may have been torn down at process exit.
-// Guarded by OffloadContextValMutex.
+// Guarded by getOffloadContextMutex().
static bool ShutDownAtExit = false;
struct OffloadContext {
OffloadContext(OffloadContext &) = delete;
@@ -378,7 +381,7 @@ Error initPlugins(OffloadContext &Context, const ol_init_args_t *InitArgs) {
}
Error olInit_impl(const ol_init_args_t *InitArgs) {
- std::lock_guard<std::mutex> Lock(OffloadContextValMutex);
+ std::lock_guard<std::mutex> Lock(getOffloadContextMutex());
if (ShutDownAtExit)
return createOffloadError(ErrorCode::UNINITIALIZED,
@@ -419,7 +422,7 @@ static Error destroyContext(OffloadContext *Context) {
}
Error olShutDown_impl() {
- std::lock_guard<std::mutex> Lock(OffloadContextValMutex);
+ std::lock_guard<std::mutex> Lock(getOffloadContextMutex());
// The context may already have been released at process exit, calls that
// balance an earlier olInit are still valid.
@@ -440,7 +443,7 @@ Error olShutDown_impl() {
// In these cases we register a specific handler to shut it down with the proper
// ordering. If this is necessary than the shutdown implementation is skipped.
static void shutDownAtExit() {
- std::lock_guard<std::mutex> Lock(OffloadContextValMutex);
+ std::lock_guard<std::mutex> Lock(getOffloadContextMutex());
ShutDownAtExit = true;
if (OffloadContext *Context = OffloadContextVal.exchange(nullptr))
consumeError(destroyContext(Context));
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 96e4027dfc99f..74c4b77d141b8 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -18,7 +18,10 @@
using namespace llvm::omp::target::debug;
-static std::mutex PluginMtx;
+static std::mutex &getPluginMutex() {
+ static std::mutex Mutex;
+ return Mutex;
+}
static uint32_t RefCount = 0;
std::atomic<bool> RTLAlive{false};
std::atomic<int> RTLOngoingSyncs{0};
@@ -33,7 +36,7 @@ static void checkRuntimeEnvironment() {
}
void initRuntime() {
- std::scoped_lock<decltype(PluginMtx)> Lock(PluginMtx);
+ std::scoped_lock<std::mutex> Lock(getPluginMutex());
Profiler::get();
TIMESCOPE();
@@ -59,7 +62,7 @@ void initRuntime() {
}
void deinitRuntime() {
- std::scoped_lock<decltype(PluginMtx)> Lock(PluginMtx);
+ std::scoped_lock<std::mutex> Lock(getPluginMutex());
assert(PM && "Runtime not initialized");
if (RefCount == 1) {
>From 63e27cb75bb594cdb9ed8605165f062e6e3bd084 Mon Sep 17 00:00:00 2001
From: Daniel Petrovic <daniel-dev at hotmail.de>
Date: Wed, 7 Oct 2026 14:00:02 +0200
Subject: [PATCH 49/52] [clang-tidy] Report virtual-class-destructor at the
destructor location (#227198)
Fixes #224686
---
.../VirtualClassDestructorCheck.cpp | 20 +++---
clang-tools-extra/docs/ReleaseNotes.md | 7 +++
.../virtual-class-destructor.cpp | 62 +++++++++----------
3 files changed, 51 insertions(+), 38 deletions(-)
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
index e43a652ed9d0c..3b5564e3ae9a7 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/VirtualClassDestructorCheck.cpp
@@ -174,15 +174,21 @@ void VirtualClassDestructorCheck::check(
if (!Destructor)
return;
+ const bool HasUserDeclaredDtor =
+ MatchedClassOrStruct->hasUserDeclaredDestructor();
+
+ const SourceLocation DiagLoc = HasUserDeclaredDtor
+ ? Destructor->getLocation()
+ : MatchedClassOrStruct->getLocation();
+
if (Destructor->getAccess() == AccessSpecifier::AS_private) {
- diag(MatchedClassOrStruct->getLocation(),
- "destructor of %0 is private and prevents using the type")
+ diag(DiagLoc, "destructor of %0 is private and prevents using the type")
<< MatchedClassOrStruct;
- diag(MatchedClassOrStruct->getLocation(),
+ diag(DiagLoc,
/*Description=*/"make it public and virtual", DiagnosticIDs::Note)
<< changePrivateDestructorVisibilityTo(
"public", *Destructor, *Result.SourceManager, getLangOpts());
- diag(MatchedClassOrStruct->getLocation(),
+ diag(DiagLoc,
/*Description=*/"make it protected", DiagnosticIDs::Note)
<< changePrivateDestructorVisibilityTo(
"protected", *Destructor, *Result.SourceManager, getLangOpts());
@@ -194,7 +200,7 @@ void VirtualClassDestructorCheck::check(
bool ProtectedAndVirtual = false;
FixItHint Fix;
- if (MatchedClassOrStruct->hasUserDeclaredDestructor()) {
+ if (HasUserDeclaredDtor) {
if (Destructor->getAccess() == AccessSpecifier::AS_public) {
Fix = FixItHint::CreateInsertion(Destructor->getLocation(), "virtual ");
} else if (Destructor->getAccess() == AccessSpecifier::AS_protected) {
@@ -209,11 +215,11 @@ void VirtualClassDestructorCheck::check(
*Result.SourceManager);
}
- diag(MatchedClassOrStruct->getLocation(),
+ diag(DiagLoc,
"destructor of %0 is %select{public and non-virtual|protected and "
"virtual}1")
<< MatchedClassOrStruct << ProtectedAndVirtual;
- diag(MatchedClassOrStruct->getLocation(),
+ diag(DiagLoc,
"make it %select{public and virtual|protected and non-virtual}0",
DiagnosticIDs::Note)
<< ProtectedAndVirtual << Fix;
diff --git a/clang-tools-extra/docs/ReleaseNotes.md b/clang-tools-extra/docs/ReleaseNotes.md
index b973c3f1f4a0e..7bbf0a85d7088 100644
--- a/clang-tools-extra/docs/ReleaseNotes.md
+++ b/clang-tools-extra/docs/ReleaseNotes.md
@@ -209,6 +209,13 @@ infrastructure are described first, followed by tool-specific sections.
- Improved {doc}`cppcoreguidelines-use-enum-class
<clang-tidy/checks/cppcoreguidelines/use-enum-class>` check by omitting unnamed enums from the `enum class` requirement, as previously the check suggested users an ill-formed fix.
+- Improved {doc}`cppcoreguidelines-virtual-class-destructor
+ <clang-tidy/checks/cppcoreguidelines/virtual-class-destructor>` check by
+ emitting the diagnostic and its fix-it notes at the destructor's location
+ instead of the class name, whenever the destructor is user-declared. The
+ diagnostics are still emitted at the class name for implicitly declared
+ destructors.
+
- Improved {doc}`misc-const-correctness
<clang-tidy/checks/misc/const-correctness>` check:
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/virtual-class-destructor.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/virtual-class-destructor.cpp
index 725a7094a0f17..9cfde2e02b3d0 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/virtual-class-destructor.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/virtual-class-destructor.cpp
@@ -1,8 +1,8 @@
// RUN: %check_clang_tidy %s cppcoreguidelines-virtual-class-destructor %t -- --fix-notes
-// CHECK-MESSAGES: :[[@LINE+4]]:8: warning: destructor of 'PrivateVirtualBaseStruct' is private and prevents using the type [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+3]]:8: note: make it public and virtual
-// CHECK-MESSAGES: :[[@LINE+2]]:8: note: make it protected
+// CHECK-MESSAGES: :[[@LINE+8]]:11: warning: destructor of 'PrivateVirtualBaseStruct' is private and prevents using the type [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+7]]:11: note: make it public and virtual
+// CHECK-MESSAGES: :[[@LINE+6]]:11: note: make it protected
// As we have 2 conflicting fixes in notes, no fix is applied.
struct PrivateVirtualBaseStruct {
virtual void f();
@@ -16,8 +16,8 @@ struct PublicVirtualBaseStruct { // OK
virtual ~PublicVirtualBaseStruct() {}
};
-// CHECK-MESSAGES: :[[@LINE+2]]:8: warning: destructor of 'ProtectedVirtualBaseStruct' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:8: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+6]]:11: warning: destructor of 'ProtectedVirtualBaseStruct' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+5]]:11: note: make it protected and non-virtual
struct ProtectedVirtualBaseStruct {
virtual void f();
@@ -26,8 +26,8 @@ struct ProtectedVirtualBaseStruct {
// CHECK-FIXES: ~ProtectedVirtualBaseStruct() {}
};
-// CHECK-MESSAGES: :[[@LINE+2]]:8: warning: destructor of 'ProtectedVirtualDefaultBaseStruct' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:8: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+6]]:11: warning: destructor of 'ProtectedVirtualDefaultBaseStruct' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+5]]:11: note: make it protected and non-virtual
struct ProtectedVirtualDefaultBaseStruct {
virtual void f();
@@ -36,9 +36,9 @@ struct ProtectedVirtualDefaultBaseStruct {
// CHECK-FIXES: ~ProtectedVirtualDefaultBaseStruct() = default;
};
-// CHECK-MESSAGES: :[[@LINE+4]]:8: warning: destructor of 'PrivateNonVirtualBaseStruct' is private and prevents using the type [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+3]]:8: note: make it public and virtual
-// CHECK-MESSAGES: :[[@LINE+2]]:8: note: make it protected
+// CHECK-MESSAGES: :[[@LINE+8]]:3: warning: destructor of 'PrivateNonVirtualBaseStruct' is private and prevents using the type [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+7]]:3: note: make it public and virtual
+// CHECK-MESSAGES: :[[@LINE+6]]:3: note: make it protected
// As we have 2 conflicting fixes in notes, no fix is applied.
struct PrivateNonVirtualBaseStruct {
virtual void f();
@@ -47,8 +47,8 @@ struct PrivateNonVirtualBaseStruct {
~PrivateNonVirtualBaseStruct() {}
};
-// CHECK-MESSAGES: :[[@LINE+2]]:8: warning: destructor of 'PublicNonVirtualBaseStruct' is public and non-virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:8: note: make it public and virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:3: warning: destructor of 'PublicNonVirtualBaseStruct' is public and non-virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+3]]:3: note: make it public and virtual
struct PublicNonVirtualBaseStruct {
virtual void f();
~PublicNonVirtualBaseStruct() {}
@@ -85,9 +85,9 @@ struct ProtectedNonVirtualBaseStruct { // OK
~ProtectedNonVirtualBaseStruct() {}
};
-// CHECK-MESSAGES: :[[@LINE+4]]:7: warning: destructor of 'PrivateVirtualBaseClass' is private and prevents using the type [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+3]]:7: note: make it public and virtual
-// CHECK-MESSAGES: :[[@LINE+2]]:7: note: make it protected
+// CHECK-MESSAGES: :[[@LINE+6]]:11: warning: destructor of 'PrivateVirtualBaseClass' is private and prevents using the type [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+5]]:11: note: make it public and virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:11: note: make it protected
// As we have 2 conflicting fixes in notes, no fix is applied.
class PrivateVirtualBaseClass {
virtual void f();
@@ -101,8 +101,8 @@ class PublicVirtualBaseClass { // OK
virtual ~PublicVirtualBaseClass() {}
};
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'ProtectedVirtualBaseClass' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+6]]:11: warning: destructor of 'ProtectedVirtualBaseClass' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+5]]:11: note: make it protected and non-virtual
class ProtectedVirtualBaseClass {
virtual void f();
@@ -133,8 +133,8 @@ class PublicASImplicitNonVirtualBaseClass {
int foo = 42;
};
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'PublicNonVirtualBaseClass' is public and non-virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it public and virtual
+// CHECK-MESSAGES: :[[@LINE+6]]:3: warning: destructor of 'PublicNonVirtualBaseClass' is public and non-virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+5]]:3: note: make it public and virtual
class PublicNonVirtualBaseClass {
virtual void f();
@@ -275,44 +275,44 @@ namespace macro_tests {
#define MY_VIRTUAL virtual
#define CONCAT(x, y) x##y
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'FooBar1' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:28: warning: destructor of 'FooBar1' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+3]]:28: note: make it protected and non-virtual
class FooBar1 {
protected:
CONCAT(vir, tual) CONCAT(~Foo, Bar1()); // no-fixit
};
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'FooBar2' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:18: warning: destructor of 'FooBar2' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+3]]:18: note: make it protected and non-virtual
class FooBar2 {
protected:
virtual CONCAT(~Foo, Bar2()); // FIXME: We should have a fixit for this.
};
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'FooBar3' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:21: warning: destructor of 'FooBar3' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+3]]:21: note: make it protected and non-virtual
class FooBar3 {
protected:
CONCAT(vir, tual) ~FooBar3(); // FIXME: We should have a fixit for this.
};
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'FooBar4' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:21: warning: destructor of 'FooBar4' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+3]]:21: note: make it protected and non-virtual
class FooBar4 {
protected:
CONCAT(vir, tual) ~CONCAT(Foo, Bar4()); // FIXME: We should have a fixit for this.
};
-// CHECK-MESSAGES: :[[@LINE+3]]:7: warning: destructor of 'FooBar5' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+2]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+5]]:29: warning: destructor of 'FooBar5' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+4]]:29: note: make it protected and non-virtual
#define XMACRO(COLUMN1, COLUMN2) COLUMN1 COLUMN2
class FooBar5 {
protected:
XMACRO(CONCAT(vir, tual), ~CONCAT(Foo, Bar5());) // no-crash, no-fixit
};
-// CHECK-MESSAGES: :[[@LINE+2]]:7: warning: destructor of 'FooBar6' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
-// CHECK-MESSAGES: :[[@LINE+1]]:7: note: make it protected and non-virtual
+// CHECK-MESSAGES: :[[@LINE+4]]:14: warning: destructor of 'FooBar6' is protected and virtual [cppcoreguidelines-virtual-class-destructor]
+// CHECK-MESSAGES: :[[@LINE+3]]:14: note: make it protected and non-virtual
class FooBar6 {
protected:
MY_VIRTUAL ~FooBar6(); // FIXME: We should have a fixit for this.
>From 05559260278ec9d911ab9396f3258a313d229cbd Mon Sep 17 00:00:00 2001
From: Zahira Ammarguellat <zahira.ammarguellat at intel.com>
Date: Wed, 7 Oct 2026 05:02:17 -0700
Subject: [PATCH 50/52] [OpenMP] Tile/stripe followed by reverse (#222401)
`Clang` rejects `#pragma omp tile` or `#pragma omp stripe` immediately
followed by `#pragma omp reverse` on the same loop nest, emitting the
error:
`error: statement after '#pragma omp tile' must be a for loop`
See https://godbolt.org/z/d1fsfaMe9
The correct behavior is to accept it. Bare`reverse`, bare
`tile`/`stripe`, and `tile` followed by `stripe` all work. Only
`tile`/`stripe` and `reverse` fails.
This PR fixes the issue.
---
clang/include/clang/AST/StmtOpenMP.h | 42 ++++++---
clang/lib/AST/StmtOpenMP.cpp | 29 +++++-
clang/lib/Sema/SemaOpenMP.cpp | 35 ++++++-
clang/test/OpenMP/stripe_messages.cpp | 13 +++
clang/test/OpenMP/tile_messages.cpp | 13 +++
.../OpenMP/tile_stripe_reverse_ast_print.cpp | 79 ++++++++++++++++
.../test/transform/tile/tile-stripe-reverse.c | 94 +++++++++++++++++++
7 files changed, 283 insertions(+), 22 deletions(-)
create mode 100644 clang/test/OpenMP/tile_stripe_reverse_ast_print.cpp
create mode 100644 openmp/runtime/test/transform/tile/tile-stripe-reverse.c
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 773e1153ccdfc..a070d97a6db1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -905,6 +905,10 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
return ignoreIntraTileHint(const_cast<Stmt *>(S));
}
+ /// Returns the statements of \p Body that precede its last statement (the
+ /// nested loop), such as the user counter update `omp reverse` emits.
+ static ArrayRef<Stmt *> getStmtsBeforeNestedLoop(Stmt *Body);
+
/// Calls the specified callback function for all the loops in \p CurStmt,
/// from the outermost to the innermost.
///
@@ -913,6 +917,13 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
/// nullptr if the loop has no such hint. Callers that need the hint (see
/// checkOpenMPIterationSpace) can peel \p HintWrapper themselves; everyone
/// else can ignore it.
+ ///
+ /// \p RelaxNestForPeeledTransformation allows a loop generated by a peeled
+ /// loop-transformation directive to hold statements before the next loop,
+ /// such as the user counter update `omp reverse` emits, so e.g. `omp tile`
+ /// accepts a nested `omp reverse` on a non-innermost loop. Only set it if the
+ /// caller re-emits those statements in the loops it generates; otherwise they
+ /// are dropped.
static bool
doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
unsigned NumLoops,
@@ -920,25 +931,29 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
Stmt * /*HintWrapper*/)>
Callback,
llvm::function_ref<void(OMPLoopTransformationDirective *)>
- OnTransformationCallback);
+ OnTransformationCallback,
+ bool RelaxNestForPeeledTransformation = false);
static bool
doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
unsigned NumLoops,
llvm::function_ref<bool(unsigned, Stmt *)> Callback,
llvm::function_ref<void(OMPLoopTransformationDirective *)>
- OnTransformationCallback) {
+ OnTransformationCallback,
+ bool RelaxNestForPeeledTransformation = false) {
auto &&NewCallback = [Callback](unsigned Cnt, Stmt *Loop, Stmt *) {
return Callback(Cnt, Loop);
};
return doForAllLoops(CurStmt, TryImperfectlyNestedLoops, NumLoops,
- NewCallback, OnTransformationCallback);
+ NewCallback, OnTransformationCallback,
+ RelaxNestForPeeledTransformation);
}
static bool
doForAllLoops(const Stmt *CurStmt, bool TryImperfectlyNestedLoops,
unsigned NumLoops,
llvm::function_ref<bool(unsigned, const Stmt *)> Callback,
llvm::function_ref<void(const OMPLoopTransformationDirective *)>
- OnTransformationCallback) {
+ OnTransformationCallback,
+ bool RelaxNestForPeeledTransformation = false) {
auto &&NewCallback = [Callback](unsigned Cnt, Stmt *CurStmt) {
return Callback(Cnt, CurStmt);
};
@@ -947,28 +962,31 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
OnTransformationCallback(A);
};
return doForAllLoops(const_cast<Stmt *>(CurStmt), TryImperfectlyNestedLoops,
- NumLoops, NewCallback, NewTransformCb);
+ NumLoops, NewCallback, NewTransformCb,
+ RelaxNestForPeeledTransformation);
}
/// Calls the specified callback function for all the loops in \p CurStmt,
/// from the outermost to the innermost.
- static bool
- doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
- unsigned NumLoops,
- llvm::function_ref<bool(unsigned, Stmt *)> Callback) {
+ static bool doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
+ unsigned NumLoops,
+ llvm::function_ref<bool(unsigned, Stmt *)> Callback,
+ bool RelaxNestForPeeledTransformation = false) {
auto &&TransformCb = [](OMPLoopTransformationDirective *) {};
return doForAllLoops(CurStmt, TryImperfectlyNestedLoops, NumLoops, Callback,
- TransformCb);
+ TransformCb, RelaxNestForPeeledTransformation);
}
static bool
doForAllLoops(const Stmt *CurStmt, bool TryImperfectlyNestedLoops,
unsigned NumLoops,
- llvm::function_ref<bool(unsigned, const Stmt *)> Callback) {
+ llvm::function_ref<bool(unsigned, const Stmt *)> Callback,
+ bool RelaxNestForPeeledTransformation = false) {
auto &&NewCallback = [Callback](unsigned Cnt, const Stmt *CurStmt) {
return Callback(Cnt, CurStmt);
};
return doForAllLoops(const_cast<Stmt *>(CurStmt), TryImperfectlyNestedLoops,
- NumLoops, NewCallback);
+ NumLoops, NewCallback,
+ RelaxNestForPeeledTransformation);
}
/// Calls the specified callback function for all the loop bodies in \p
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 555e3454816ce..5ba11b2d8c42f 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -34,6 +34,13 @@ Stmt *OMPLoopBasedDirective::ignoreIntraTileHint(Stmt *S) {
return S;
}
+ArrayRef<Stmt *> OMPLoopBasedDirective::getStmtsBeforeNestedLoop(Stmt *Body) {
+ auto *CS = dyn_cast_or_null<CompoundStmt>(Body);
+ if (!CS || CS->size() < 2)
+ return {};
+ return ArrayRef(CS->body_begin(), CS->size() - 1);
+}
+
/// Like `Stmt::IgnoreContainers`, but stops at (and preserves) an intra-tile
/// hint wrapper so that the enclosing loop walker can deliver the hint to
/// loop-associated analysis (e.g. `collapse`). Non-hint attributed statements
@@ -133,10 +140,12 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
for (Stmt *S : CS->body()) {
if (!S)
continue;
- // Peek past an OMPCanonicalLoop wrapper and/or an intra-tile hint to
- // check whether this child is loop-like; keep the original (wrapped)
- // node in CurStmt so the hint still reaches the loop-analysis
+ // Peek past a single-child container so a nested loop or
+ // loop-transformation directive (e.g. `#pragma omp reverse` inside
+ // the body of an outer loop) is recognized here. Preserve any
+ // intra-tile hint wrapper so it still reaches the loop-analysis
// callback.
+ S = ignoreContainersKeepingIntraTileHint(S);
Stmt *Inner = S;
if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Inner))
Inner = CanonLoop->getLoopStmt();
@@ -152,7 +161,7 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
CurStmt = S;
continue;
}
- S = S->IgnoreContainers();
+ S = OMPLoopBasedDirective::ignoreIntraTileHint(S);
if (auto *InnerCS = dyn_cast_or_null<CompoundStmt>(S))
NextStatements.push_back(InnerCS);
}
@@ -174,9 +183,15 @@ bool OMPLoopBasedDirective::doForAllLoops(
Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops,
llvm::function_ref<bool(unsigned, Stmt *, Stmt *)> Callback,
llvm::function_ref<void(OMPLoopTransformationDirective *)>
- OnTransformationCallback) {
+ OnTransformationCallback,
+ bool RelaxNestForPeeledTransformation) {
CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
for (unsigned Cnt = 0; Cnt < NumLoops; ++Cnt) {
+ // A loop generated by a peeled transformation may hold statements before
+ // the next loop (e.g. `reverse` recomputes the user counter there). They
+ // are not intervening user code, so callers that re-emit them may look
+ // through them; the rest of the nest is still checked as usual.
+ bool PeeledTransformation = false;
while (true) {
auto *Dir = dyn_cast<OMPLoopTransformationDirective>(CurStmt);
if (!Dir)
@@ -202,6 +217,7 @@ bool OMPLoopBasedDirective::doForAllLoops(
}
CurStmt = TransformedStmt;
+ PeeledTransformation = true;
}
if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
CurStmt = CanonLoop->getLoopStmt();
@@ -223,6 +239,9 @@ bool OMPLoopBasedDirective::doForAllLoops(
"Expected canonical for or range-based for loops.");
CurStmt = cast<CXXForRangeStmt>(LoopStmt)->getBody();
}
+ if (RelaxNestForPeeledTransformation && PeeledTransformation &&
+ !getStmtsBeforeNestedLoop(CurStmt).empty())
+ CurStmt = cast<CompoundStmt>(CurStmt)->body_back();
CurStmt = OMPLoopBasedDirective::tryToFindNextInnerLoop(
CurStmt, TryImperfectlyNestedLoops);
}
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index efc570a8933f1..d0cf844acf6db 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15376,6 +15376,11 @@ bool SemaOpenMP::checkTransformableLoopNest(
SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
Stmt *&Body, SmallVectorImpl<SmallVector<Stmt *>> &OriginalInits) {
OriginalInits.emplace_back();
+ // Only tile and stripe re-emit the statements before the nested loop in a
+ // nested transformation's generated loop (see appendStmtsBeforeNestedLoop);
+ // other constructs would silently drop them.
+ bool RelaxNestForPeeledTransformation =
+ Kind == OMPD_tile || Kind == OMPD_stripe;
bool Result = OMPLoopBasedDirective::doForAllLoops(
AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false, NumLoops,
[this, &LoopHelpers, &Body, &OriginalInits,
@@ -15411,7 +15416,8 @@ bool SemaOpenMP::checkTransformableLoopNest(
},
[&OriginalInits](OMPLoopTransformationDirective *Transform) {
updatePreInits(Transform, OriginalInits.back());
- });
+ },
+ RelaxNestForPeeledTransformation);
assert(OriginalInits.back().empty() && "No preinit after innermost loop");
OriginalInits.pop_back();
return Result;
@@ -15713,7 +15719,8 @@ static void addLoopPreInits(ASTContext &Context,
/// Collect the loop statements (ForStmt or CXXRangeForStmt) of the affected
/// loop of a construct.
-static void collectLoopStmts(Stmt *AStmt, MutableArrayRef<Stmt *> LoopStmts) {
+static void collectLoopStmts(Stmt *AStmt, MutableArrayRef<Stmt *> LoopStmts,
+ bool RelaxNestForPeeledTransformation = false) {
size_t NumLoops = LoopStmts.size();
OMPLoopBasedDirective::doForAllLoops(
AStmt, /*TryImperfectlyNestedLoops=*/false, NumLoops,
@@ -15721,11 +15728,25 @@ static void collectLoopStmts(Stmt *AStmt, MutableArrayRef<Stmt *> LoopStmts) {
assert(!LoopStmts[Cnt] && "Loop statement must not yet be assigned");
LoopStmts[Cnt] = CurStmt;
return false;
- });
+ },
+ RelaxNestForPeeledTransformation);
assert(!is_contained(LoopStmts, nullptr) &&
"Expecting a loop statement for each affected loop");
}
+/// A non-innermost affected loop generated by a nested loop transformation
+/// (e.g. `omp reverse`) computes the user counter in statements ahead of the
+/// next loop. A construct that rebuilds the nest only keeps the innermost
+/// body, so it must re-emit these statements in the matching loop.
+static void appendStmtsBeforeNestedLoop(Stmt *LoopStmt,
+ SmallVectorImpl<Stmt *> &BodyParts) {
+ Stmt *Body = isa<ForStmt>(LoopStmt)
+ ? cast<ForStmt>(LoopStmt)->getBody()
+ : cast<CXXForRangeStmt>(LoopStmt)->getBody();
+ llvm::append_range(BodyParts,
+ OMPLoopBasedDirective::getStmtsBeforeNestedLoop(Body));
+}
+
/// Build and return a DeclRefExpr for the floor induction variable using the
/// SemaRef and the provided parameters.
static Expr *makeFloorIVRef(Sema &SemaRef, ArrayRef<VarDecl *> FloorIndVars,
@@ -15774,7 +15795,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
// Collect all affected loop statements.
SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
- collectLoopStmts(AStmt, LoopStmts);
+ collectLoopStmts(AStmt, LoopStmts, /*RelaxNestForPeeledTransformation=*/true);
SmallVector<Stmt *, 4> PreInits;
CaptureVars CopyTransformer(SemaRef);
@@ -16023,6 +16044,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
BodyParts.push_back(SourceCXXFor->getLoopVarStmt());
+ if (static_cast<unsigned>(I) + 1 < NumLoops)
+ appendStmtsBeforeNestedLoop(LoopStmt, BodyParts);
BodyParts.push_back(Inner);
Inner = CompoundStmt::Create(Context, BodyParts, FPOptionsOverride(),
Inner->getBeginLoc(), Inner->getEndLoc());
@@ -16131,7 +16154,7 @@ StmtResult SemaOpenMP::ActOnOpenMPStripeDirective(ArrayRef<OMPClause *> Clauses,
// Collect all affected loop statements.
SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
- collectLoopStmts(AStmt, LoopStmts);
+ collectLoopStmts(AStmt, LoopStmts, /*RelaxNestForPeeledTransformation=*/true);
SmallVector<Stmt *, 4> PreInits;
CaptureVars CopyTransformer(SemaRef);
@@ -16293,6 +16316,8 @@ StmtResult SemaOpenMP::ActOnOpenMPStripeDirective(ArrayRef<OMPClause *> Clauses,
BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
BodyParts.push_back(SourceCXXFor->getLoopVarStmt());
+ if (static_cast<unsigned>(I) + 1 < NumLoops)
+ appendStmtsBeforeNestedLoop(LoopStmt, BodyParts);
BodyParts.push_back(Inner);
Inner = CompoundStmt::Create(Context, BodyParts, FPOptionsOverride(),
Inner->getBeginLoc(), Inner->getEndLoc());
diff --git a/clang/test/OpenMP/stripe_messages.cpp b/clang/test/OpenMP/stripe_messages.cpp
index 3f171f40d71d9..2e4675c35cc6c 100644
--- a/clang/test/OpenMP/stripe_messages.cpp
+++ b/clang/test/OpenMP/stripe_messages.cpp
@@ -169,3 +169,16 @@ void f() {
;
}
} // namespace GH139433
+
+// The loop generated by 'reverse' may be a non-innermost affected loop, but
+// user code between it and the next affected loop is still intervening code.
+void stripe_of_reverse_intervening() {
+ // expected-error at +3 {{statement after '#pragma omp stripe' must be a for loop}}
+#pragma omp stripe sizes(2, 2)
+#pragma omp reverse
+ for (int i = 0; i < 7; ++i) {
+ int k = 3;
+ for (int j = 0; j < 7; ++j)
+ ;
+ }
+}
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index 021262ac9a54a..72ceef71566d6 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -210,3 +210,16 @@ void unroll_of_tile() {
;
}
} // namespace consumed_by_transformation
+
+// The loop generated by 'reverse' may be a non-innermost affected loop, but
+// user code between it and the next affected loop is still intervening code.
+void tile_of_reverse_intervening() {
+ // expected-error at +3 {{statement after '#pragma omp tile' must be a for loop}}
+#pragma omp tile sizes(2, 2)
+#pragma omp reverse
+ for (int i = 0; i < 7; ++i) {
+ int k = 3;
+ for (int j = 0; j < 7; ++j)
+ ;
+ }
+}
diff --git a/clang/test/OpenMP/tile_stripe_reverse_ast_print.cpp b/clang/test/OpenMP/tile_stripe_reverse_ast_print.cpp
new file mode 100644
index 0000000000000..5eee992f50483
--- /dev/null
+++ b/clang/test/OpenMP/tile_stripe_reverse_ast_print.cpp
@@ -0,0 +1,79 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 \
+// RUN: -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 \
+// RUN: -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 \
+// RUN: -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 \
+// RUN: -include-pch %t -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(...);
+
+// PRINT-LABEL: void tile_then_reverse(
+void tile_then_reverse() {
+ // PRINT: #pragma omp tile sizes(2, 2)
+ // PRINT: #pragma omp reverse
+#pragma omp tile sizes(2, 2)
+#pragma omp reverse
+ for (int i = 0; i < 20; ++i)
+ for (int j = 0; j < 20; ++j)
+ body(i, j);
+}
+
+// PRINT-LABEL: void stripe_then_reverse(
+void stripe_then_reverse() {
+ // PRINT: #pragma omp stripe sizes(2, 2)
+ // PRINT: #pragma omp reverse
+#pragma omp stripe sizes(2, 2)
+#pragma omp reverse
+ for (int i = 0; i < 20; ++i)
+ for (int j = 0; j < 20; ++j)
+ body(i, j);
+}
+
+// PRINT-LABEL: void tile_over_inner_reverse(
+void tile_over_inner_reverse() {
+ // PRINT: #pragma omp tile sizes(2, 2)
+#pragma omp tile sizes(2, 2)
+#pragma omp reverse
+ for (int j = 0; j < 20; ++j) {
+ // PRINT: #pragma omp reverse
+#pragma omp reverse
+ for (int i = 0; i < 20; ++i)
+ body(j, i);
+ }
+}
+
+// PRINT-LABEL: void stripe_over_inner_reverse(
+void stripe_over_inner_reverse() {
+ // PRINT: #pragma omp stripe sizes(2, 2)
+#pragma omp stripe sizes(2, 2)
+#pragma omp reverse
+ for (int j = 0; j < 20; ++j) {
+ // PRINT: #pragma omp reverse
+#pragma omp reverse
+ for (int i = 0; i < 20; ++i)
+ body(j, i);
+ }
+}
+
+// PRINT-LABEL: void for_collapse_then_reverse(
+void for_collapse_then_reverse() {
+ // PRINT: #pragma omp for collapse(2)
+ // PRINT: #pragma omp reverse
+#pragma omp for collapse(2)
+#pragma omp reverse
+ for (int j = 0; j < 20; j += 2)
+ for (int i = 0; i < 20; i += 3)
+ body(j, i);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/tile/tile-stripe-reverse.c b/openmp/runtime/test/transform/tile/tile-stripe-reverse.c
new file mode 100644
index 0000000000000..7f599737b1e8a
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/tile-stripe-reverse.c
@@ -0,0 +1,94 @@
+// RUN: %libomp-compile -fopenmp-version=60 && %libomp-run \
+// RUN: | FileCheck %s --match-full-lines
+
+// tile/stripe over a reverse that is not the innermost affected loop must
+// keep the reversal (regression test for #222401).
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ printf("tile\n");
+#pragma omp tile sizes(2, 2)
+#pragma omp reverse
+ for (int i = 0; i < 4; ++i)
+ for (int j = 0; j < 4; ++j)
+ printf("i=%d j=%d\n", i, j);
+
+ printf("stripe\n");
+#pragma omp stripe sizes(2, 2)
+#pragma omp reverse
+ for (int i = 0; i < 4; ++i)
+ for (int j = 0; j < 4; ++j)
+ printf("i=%d j=%d\n", i, j);
+
+ printf("nested\n");
+#pragma omp tile sizes(2, 2)
+#pragma omp reverse
+ for (int j = 0; j < 4; ++j) {
+#pragma omp reverse
+ for (int i = 0; i < 4; ++i)
+ printf("j=%d i=%d\n", j, i);
+ }
+
+ printf("done\n");
+ return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK: tile
+// CHECK-NEXT: i=3 j=0
+// CHECK-NEXT: i=3 j=1
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=3 j=2
+// CHECK-NEXT: i=3 j=3
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: i=2 j=3
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=1 j=3
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=0 j=3
+// CHECK-NEXT: stripe
+// CHECK-NEXT: i=3 j=0
+// CHECK-NEXT: i=3 j=1
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=3 j=2
+// CHECK-NEXT: i=3 j=3
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: i=2 j=3
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=1 j=3
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=0 j=3
+// CHECK-NEXT: nested
+// CHECK-NEXT: j=3 i=3
+// CHECK-NEXT: j=3 i=2
+// CHECK-NEXT: j=2 i=3
+// CHECK-NEXT: j=2 i=2
+// CHECK-NEXT: j=3 i=1
+// CHECK-NEXT: j=3 i=0
+// CHECK-NEXT: j=2 i=1
+// CHECK-NEXT: j=2 i=0
+// CHECK-NEXT: j=1 i=3
+// CHECK-NEXT: j=1 i=2
+// CHECK-NEXT: j=0 i=3
+// CHECK-NEXT: j=0 i=2
+// CHECK-NEXT: j=1 i=1
+// CHECK-NEXT: j=1 i=0
+// CHECK-NEXT: j=0 i=1
+// CHECK-NEXT: j=0 i=0
+// CHECK-NEXT: done
>From 921638a44b85a2401a560c8625d227da3753b1ec Mon Sep 17 00:00:00 2001
From: Henrich Lauko <xlauko at mail.muni.cz>
Date: Wed, 7 Oct 2026 14:03:47 +0200
Subject: [PATCH 51/52] [CIR][CodeGen][NFC] Share the Itanium
__vmi_class_type_info flags computation (#227256)
Deduplicates the `__vmi_class_type_info` and `__base_class_type_info` flags and
`computeVMIClassTypeInfoFlags` between CIR and classic CodeGen into
`ItaniumCXXABIUtils.h`.
Assisted-by: Claude Code (Claude Fable 5.1).
---
.../clang/CodeGenUtils/ItaniumCXXABIUtils.h | 21 +++++
clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 80 +-----------------
clang/lib/CodeGen/ItaniumCXXABI.cpp | 82 +------------------
clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp | 54 ++++++++++++
4 files changed, 81 insertions(+), 156 deletions(-)
diff --git a/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h b/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h
index fb125e796aa19..59b94c9bb08c3 100644
--- a/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h
+++ b/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h
@@ -44,10 +44,31 @@ enum PointerTypeInfoFlags {
PTI_Noexcept = 0x40,
};
+/// Flags for abi::__vmi_class_type_info.
+enum VMIClassTypeInfoFlags {
+ /// VMI_NonDiamondRepeat - Class has non-diamond repeated inheritance.
+ VMI_NonDiamondRepeat = 0x1,
+
+ /// VMI_DiamondShaped - Class is diamond shaped.
+ VMI_DiamondShaped = 0x2
+};
+
+/// Flags for abi::__base_class_type_info.
+enum BaseClassTypeInfoFlags {
+ /// BCTI_Virtual - Base class is virtual.
+ BCTI_Virtual = 0x1,
+
+ /// BCTI_Public - Base class is public.
+ BCTI_Public = 0x2
+};
+
/// Compute the src2dst_offset hint as described in the Itanium C++ ABI [2.9.7].
CharUnits computeOffsetHint(ASTContext &Ctx, const CXXRecordDecl *Src,
const CXXRecordDecl *Dst);
+/// Compute the value of the flags member in abi::__vmi_class_type_info.
+unsigned computeVMIClassTypeInfoFlags(const CXXRecordDecl *RD);
+
/// Returns whether the given type contains an incomplete class type. This is
/// true if
///
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index b8d329ab06bb0..763ed026f8714 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -638,27 +638,8 @@ class CIRGenItaniumRTTIBuilder {
};
} // namespace
-// TODO(cir): Will be removed after sharing them with the classical codegen
namespace {
-// VMI type info flags.
-enum {
- /// VMI_NonDiamondRepeat - Class has non-diamond repeated inheritance.
- VMI_NonDiamondRepeat = 0x1,
-
- /// VMI_DiamondShaped - Class is diamond shaped.
- VMI_DiamondShaped = 0x2
-};
-
-// Base class type info flags.
-enum {
- /// BCTI_Virtual - Base class is virtual.
- BCTI_Virtual = 0x1,
-
- /// BCTI_Public - Base class is public.
- BCTI_Public = 0x2
-};
-
/// Given a builtin type, returns whether the type
/// info for that type is defined in the standard library.
/// TODO(cir): this can unified with LLVM codegen
@@ -854,61 +835,6 @@ static bool shouldUseExternalRttiDescriptor(CIRGenModule &cgm, QualType ty) {
return false;
}
-/// Contains virtual and non-virtual bases seen when traversing a class
-/// hierarchy.
-struct SeenBases {
- llvm::SmallPtrSet<const CXXRecordDecl *, 16> nonVirtualBases;
- llvm::SmallPtrSet<const CXXRecordDecl *, 16> virtualBases;
-};
-
-/// Compute the value of the flags member in abi::__vmi_class_type_info.
-///
-static unsigned computeVmiClassTypeInfoFlags(const CXXBaseSpecifier *base,
- SeenBases &bases) {
-
- unsigned flags = 0;
- auto *baseDecl = base->getType()->castAsCXXRecordDecl();
-
- if (base->isVirtual()) {
- // Mark the virtual base as seen.
- if (!bases.virtualBases.insert(baseDecl).second) {
- // If this virtual base has been seen before, then the class is diamond
- // shaped.
- flags |= VMI_DiamondShaped;
- } else {
- if (bases.nonVirtualBases.count(baseDecl))
- flags |= VMI_NonDiamondRepeat;
- }
- } else {
- // Mark the non-virtual base as seen.
- if (!bases.nonVirtualBases.insert(baseDecl).second) {
- // If this non-virtual base has been seen before, then the class has non-
- // diamond shaped repeated inheritance.
- flags |= VMI_NonDiamondRepeat;
- } else {
- if (bases.virtualBases.count(baseDecl))
- flags |= VMI_NonDiamondRepeat;
- }
- }
-
- // Walk all bases.
- for (const auto &bs : baseDecl->bases())
- flags |= computeVmiClassTypeInfoFlags(&bs, bases);
-
- return flags;
-}
-
-static unsigned computeVmiClassTypeInfoFlags(const CXXRecordDecl *rd) {
- unsigned flags = 0;
- SeenBases bases;
-
- // Walk all bases.
- for (const auto &bs : rd->bases())
- flags |= computeVmiClassTypeInfoFlags(&bs, bases);
-
- return flags;
-}
-
// Return whether the given record decl has a "single,
// public, non-virtual base at offset zero (i.e. the derived class is dynamic
// iff the base is)", according to Itanium C++ ABI, 2.95p6b.
@@ -1217,7 +1143,7 @@ void CIRGenItaniumRTTIBuilder::buildVMIClassTypeInfo(mlir::Location loc,
// __flags is a word with flags describing details about the class
// structure, which may be referenced by using the __flags_masks
// enumeration. These flags refer to both direct and indirect bases.
- unsigned flags = computeVmiClassTypeInfoFlags(rd);
+ unsigned flags = CodeGenUtils::computeVMIClassTypeInfoFlags(rd);
fields.push_back(cir::IntAttr::get(unsignedIntLTy, flags));
// Itanium C++ ABI 2.9.5p6c:
@@ -1284,9 +1210,9 @@ void CIRGenItaniumRTTIBuilder::buildVMIClassTypeInfo(mlir::Location loc,
// The low-order byte of __offset_flags contains flags, as given by the
// masks from the enumeration __offset_flags_masks.
if (base.isVirtual())
- offsetFlags |= BCTI_Virtual;
+ offsetFlags |= CodeGenUtils::BCTI_Virtual;
if (base.getAccessSpecifier() == AS_public)
- offsetFlags |= BCTI_Public;
+ offsetFlags |= CodeGenUtils::BCTI_Public;
fields.push_back(cir::IntAttr::get(offsetFlagsLTy, offsetFlags));
}
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 5fe80e16f68c8..8e9aaa1313528 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3545,24 +3545,6 @@ class ItaniumRTTIBuilder {
ItaniumRTTIBuilder(const ItaniumCXXABI &ABI)
: CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {}
- // VMI type info flags.
- enum {
- /// VMI_NonDiamondRepeat - Class has non-diamond repeated inheritance.
- VMI_NonDiamondRepeat = 0x1,
-
- /// VMI_DiamondShaped - Class is diamond shaped.
- VMI_DiamondShaped = 0x2
- };
-
- // Base class type info flags.
- enum {
- /// BCTI_Virtual - Base class is virtual.
- BCTI_Virtual = 0x1,
-
- /// BCTI_Public - Base class is public.
- BCTI_Public = 0x2
- };
-
/// BuildTypeInfo - Build the RTTI type info struct for the given type, or
/// link to an existing RTTI descriptor if one already exists.
llvm::Constant *BuildTypeInfo(QualType Ty);
@@ -4381,64 +4363,6 @@ void ItaniumRTTIBuilder::BuildSIClassTypeInfo(const CXXRecordDecl *RD) {
Fields.push_back(BaseTypeInfo);
}
-namespace {
- /// SeenBases - Contains virtual and non-virtual bases seen when traversing
- /// a class hierarchy.
- struct SeenBases {
- llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
- llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
- };
-}
-
-/// ComputeVMIClassTypeInfoFlags - Compute the value of the flags member in
-/// abi::__vmi_class_type_info.
-///
-static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
- SeenBases &Bases) {
-
- unsigned Flags = 0;
-
- auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
- if (Base->isVirtual()) {
- // Mark the virtual base as seen.
- if (!Bases.VirtualBases.insert(BaseDecl).second) {
- // If this virtual base has been seen before, then the class is diamond
- // shaped.
- Flags |= ItaniumRTTIBuilder::VMI_DiamondShaped;
- } else {
- if (Bases.NonVirtualBases.count(BaseDecl))
- Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
- }
- } else {
- // Mark the non-virtual base as seen.
- if (!Bases.NonVirtualBases.insert(BaseDecl).second) {
- // If this non-virtual base has been seen before, then the class has non-
- // diamond shaped repeated inheritance.
- Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
- } else {
- if (Bases.VirtualBases.count(BaseDecl))
- Flags |= ItaniumRTTIBuilder::VMI_NonDiamondRepeat;
- }
- }
-
- // Walk all bases.
- for (const auto &I : BaseDecl->bases())
- Flags |= ComputeVMIClassTypeInfoFlags(&I, Bases);
-
- return Flags;
-}
-
-static unsigned ComputeVMIClassTypeInfoFlags(const CXXRecordDecl *RD) {
- unsigned Flags = 0;
- SeenBases Bases;
-
- // Walk all bases.
- for (const auto &I : RD->bases())
- Flags |= ComputeVMIClassTypeInfoFlags(&I, Bases);
-
- return Flags;
-}
-
/// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for
/// classes with bases that do not satisfy the abi::__si_class_type_info
/// constraints, according ti the Itanium C++ ABI, 2.9.5p5c.
@@ -4450,7 +4374,7 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
// __flags is a word with flags describing details about the class
// structure, which may be referenced by using the __flags_masks
// enumeration. These flags refer to both direct and indirect bases.
- unsigned Flags = ComputeVMIClassTypeInfoFlags(RD);
+ unsigned Flags = CodeGenUtils::computeVMIClassTypeInfoFlags(RD);
Fields.push_back(llvm::ConstantInt::get(UnsignedIntLTy, Flags));
// Itanium C++ ABI 2.9.5p6c:
@@ -4517,9 +4441,9 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
// The low-order byte of __offset_flags contains flags, as given by the
// masks from the enumeration __offset_flags_masks.
if (Base.isVirtual())
- OffsetFlags |= BCTI_Virtual;
+ OffsetFlags |= CodeGenUtils::BCTI_Virtual;
if (Base.getAccessSpecifier() == AS_public)
- OffsetFlags |= BCTI_Public;
+ OffsetFlags |= CodeGenUtils::BCTI_Public;
Fields.push_back(llvm::ConstantInt::getSigned(OffsetFlagsLTy, OffsetFlags));
}
diff --git a/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp b/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp
index 1073be5a76acf..a1ab26cfc17b0 100644
--- a/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp
+++ b/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp
@@ -61,6 +61,60 @@ CharUnits computeOffsetHint(ASTContext &Ctx, const CXXRecordDecl *Src,
return Offset;
}
+namespace {
+/// Contains virtual and non-virtual bases seen when traversing a class
+/// hierarchy.
+struct SeenBases {
+ llvm::SmallPtrSet<const CXXRecordDecl *, 16> NonVirtualBases;
+ llvm::SmallPtrSet<const CXXRecordDecl *, 16> VirtualBases;
+};
+} // namespace
+
+static unsigned computeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
+ SeenBases &Bases) {
+ unsigned Flags = 0;
+
+ auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
+ if (Base->isVirtual()) {
+ // Mark the virtual base as seen.
+ if (!Bases.VirtualBases.insert(BaseDecl).second) {
+ // If this virtual base has been seen before, then the class is diamond
+ // shaped.
+ Flags |= VMI_DiamondShaped;
+ } else {
+ if (Bases.NonVirtualBases.count(BaseDecl))
+ Flags |= VMI_NonDiamondRepeat;
+ }
+ } else {
+ // Mark the non-virtual base as seen.
+ if (!Bases.NonVirtualBases.insert(BaseDecl).second) {
+ // If this non-virtual base has been seen before, then the class has non-
+ // diamond shaped repeated inheritance.
+ Flags |= VMI_NonDiamondRepeat;
+ } else {
+ if (Bases.VirtualBases.count(BaseDecl))
+ Flags |= VMI_NonDiamondRepeat;
+ }
+ }
+
+ // Walk all bases.
+ for (const auto &I : BaseDecl->bases())
+ Flags |= computeVMIClassTypeInfoFlags(&I, Bases);
+
+ return Flags;
+}
+
+unsigned computeVMIClassTypeInfoFlags(const CXXRecordDecl *RD) {
+ unsigned Flags = 0;
+ SeenBases Bases;
+
+ // Walk all bases.
+ for (const auto &I : RD->bases())
+ Flags |= computeVMIClassTypeInfoFlags(&I, Bases);
+
+ return Flags;
+}
+
/// Returns whether the given record type is incomplete.
static bool isIncompleteClassType(const RecordType *RecordTy) {
return !RecordTy->getDecl()->getDefinitionOrSelf()->isCompleteDefinition();
>From e7421d4c4a1aee48dc5648ecfe66192552a2847d Mon Sep 17 00:00:00 2001
From: Henrich Lauko <hlauko at nvidia.com>
Date: Tue, 29 Sep 2026 09:55:33 +0000
Subject: [PATCH 52/52] [CIR][CodeGen][NFC] Share canUseSingleInheritance
Deduplicates `canUseSingleInheritance` between CIR and classic CodeGen into
`ItaniumCXXABIUtils.h`.
Assisted-by: Claude Code (Claude Fable 5.1).
---
.../clang/CodeGenUtils/ItaniumCXXABIUtils.h | 5 +++
clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 30 ++---------------
clang/lib/CodeGen/ItaniumCXXABI.cpp | 32 ++-----------------
clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp | 22 +++++++++++++
4 files changed, 31 insertions(+), 58 deletions(-)
diff --git a/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h b/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h
index 59b94c9bb08c3..9d6a20ef82ab3 100644
--- a/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h
+++ b/clang/include/clang/CodeGenUtils/ItaniumCXXABIUtils.h
@@ -62,6 +62,11 @@ enum BaseClassTypeInfoFlags {
BCTI_Public = 0x2
};
+/// Return whether the given record decl has a "single, public, non-virtual
+/// base at offset zero (i.e. the derived class is dynamic iff the base is)",
+/// according to Itanium C++ ABI, 2.95p6b.
+bool canUseSingleInheritance(const CXXRecordDecl *RD);
+
/// Compute the src2dst_offset hint as described in the Itanium C++ ABI [2.9.7].
CharUnits computeOffsetHint(ASTContext &Ctx, const CXXRecordDecl *Src,
const CXXRecordDecl *Dst);
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index 763ed026f8714..25526f0668f00 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -835,32 +835,6 @@ static bool shouldUseExternalRttiDescriptor(CIRGenModule &cgm, QualType ty) {
return false;
}
-// Return whether the given record decl has a "single,
-// public, non-virtual base at offset zero (i.e. the derived class is dynamic
-// iff the base is)", according to Itanium C++ ABI, 2.95p6b.
-// TODO(cir): this can unified with LLVM codegen
-static bool canUseSingleInheritance(const CXXRecordDecl *rd) {
- // Check the number of bases.
- if (rd->getNumBases() != 1)
- return false;
-
- // Get the base.
- CXXRecordDecl::base_class_const_iterator base = rd->bases_begin();
-
- // Check that the base is not virtual.
- if (base->isVirtual())
- return false;
-
- // Check that the base is public.
- if (base->getAccessSpecifier() != AS_public)
- return false;
-
- // Check that the class is dynamic iff the base is.
- auto *baseDecl = base->getType()->castAsCXXRecordDecl();
- return baseDecl->isEmpty() ||
- baseDecl->isDynamicClass() == rd->isDynamicClass();
-}
-
const char *vTableClassNameForType(const CIRGenModule &cgm, const Type *ty) {
// abi::__class_type_info.
static const char *const classTypeInfo =
@@ -929,7 +903,7 @@ const char *vTableClassNameForType(const CIRGenModule &cgm, const Type *ty) {
return classTypeInfo;
}
- if (canUseSingleInheritance(rd)) {
+ if (CodeGenUtils::canUseSingleInheritance(rd)) {
return siClassTypeInfo;
}
@@ -1425,7 +1399,7 @@ mlir::Attribute CIRGenItaniumRTTIBuilder::buildTypeInfo(
break;
}
- if (canUseSingleInheritance(rd)) {
+ if (CodeGenUtils::canUseSingleInheritance(rd)) {
buildSIClassTypeInfo(loc, rd);
} else {
buildVMIClassTypeInfo(loc, rd);
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 8e9aaa1313528..c851385822291 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3822,34 +3822,6 @@ static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
return false;
}
-// CanUseSingleInheritance - Return whether the given record decl has a "single,
-// public, non-virtual base at offset zero (i.e. the derived class is dynamic
-// iff the base is)", according to Itanium C++ ABI, 2.95p6b.
-static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
- // Check the number of bases.
- if (RD->getNumBases() != 1)
- return false;
-
- // Get the base.
- CXXRecordDecl::base_class_const_iterator Base = RD->bases_begin();
-
- // Check that the base is not virtual.
- if (Base->isVirtual())
- return false;
-
- // Check that the base is public.
- if (Base->getAccessSpecifier() != AS_public)
- return false;
-
- // Check that the class is dynamic iff the base is.
- auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
- if (!BaseDecl->isEmpty() &&
- BaseDecl->isDynamicClass() != RD->isDynamicClass())
- return false;
-
- return true;
-}
-
void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
llvm::Constant *StorageAddress) {
// abi::__class_type_info.
@@ -3926,7 +3898,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
if (!RD->hasDefinition() || !RD->getNumBases()) {
VTableName = ClassTypeInfo;
- } else if (CanUseSingleInheritance(RD)) {
+ } else if (CodeGenUtils::canUseSingleInheritance(RD)) {
VTableName = SIClassTypeInfo;
} else {
VTableName = VMIClassTypeInfo;
@@ -4244,7 +4216,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
break;
}
- if (CanUseSingleInheritance(RD))
+ if (CodeGenUtils::canUseSingleInheritance(RD))
BuildSIClassTypeInfo(RD);
else
BuildVMIClassTypeInfo(RD);
diff --git a/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp b/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp
index a1ab26cfc17b0..2f22ab69d4c5a 100644
--- a/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp
+++ b/clang/lib/CodeGenUtils/ItaniumCXXABIUtils.cpp
@@ -61,6 +61,28 @@ CharUnits computeOffsetHint(ASTContext &Ctx, const CXXRecordDecl *Src,
return Offset;
}
+bool canUseSingleInheritance(const CXXRecordDecl *RD) {
+ // Check the number of bases.
+ if (RD->getNumBases() != 1)
+ return false;
+
+ // Get the base.
+ CXXRecordDecl::base_class_const_iterator Base = RD->bases_begin();
+
+ // Check that the base is not virtual.
+ if (Base->isVirtual())
+ return false;
+
+ // Check that the base is public.
+ if (Base->getAccessSpecifier() != AS_public)
+ return false;
+
+ // Check that the class is dynamic iff the base is.
+ auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();
+ return BaseDecl->isEmpty() ||
+ BaseDecl->isDynamicClass() == RD->isDynamicClass();
+}
+
namespace {
/// Contains virtual and non-virtual bases seen when traversing a class
/// hierarchy.
More information about the llvm-branch-commits
mailing list