[Lldb-commits] [clang] [flang] [libc] [libcxx] [lld] [lldb] [llvm] [mlir] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)
Florian Hahn via lldb-commits
lldb-commits at lists.llvm.org
Fri Jul 18 08:48:14 PDT 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/148239
>From 42d11944daf17b42cc9eae6a1b14e25740f37de4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 18 Jun 2025 14:07:04 +0100
Subject: [PATCH 1/6] [LV] Vectorize maxnum/minnum w/o fast-math flags.
Update LV to vectorize maxnum/minnum reductions without fast-math flags,
by adding an extra check in the loop if any inputs to maxnum/minnum are
NaN. If any input is NaN,
*exit the vector loop,
*compute the reduction result up to the vector iteration that contained NaN
inputs and
* resume in the scalar loop
New recurrence kinds are added for reductions using maxnum/minnum
without fast-math flags. The new recurrence kinds are not supported in
the code to generate IR to perform the reductions to prevent accidential
mis-use. Users need to add the required checks ensuring no NaN inputs,
and convert to regular FMin/FMax recurrence kinds.
---
llvm/include/llvm/Analysis/IVDescriptors.h | 2 +
llvm/lib/Analysis/IVDescriptors.cpp | 23 ++-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 2 +
.../Transforms/Vectorize/LoopVectorize.cpp | 14 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 6 +
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 1 +
.../Vectorize/VPlanConstruction.cpp | 137 ++++++++++++++++++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +-
.../Transforms/Vectorize/VPlanTransforms.h | 5 +
.../AArch64/fmax-without-fast-math-flags.ll | 54 ++++++-
.../AArch64/fmin-without-fast-math-flags.ll | 54 ++++++-
...fmax-without-fast-math-flags-interleave.ll | 54 ++++++-
.../fmax-without-fast-math-flags.ll | 90 ++++++++++--
.../fmin-without-fast-math-flags.ll | 90 ++++++++++--
.../LoopVectorize/minmax_reduction.ll | 6 +-
16 files changed, 492 insertions(+), 62 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index b985292ccee40..e28901c072e29 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,6 +47,8 @@ enum class RecurKind {
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
+ FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
+ FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 39f74beca082f..ef095b73d5419 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -941,10 +941,27 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
};
- if (isIntMinMaxRecurrenceKind(Kind) ||
- (HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
+ if (isIntMinMaxRecurrenceKind(Kind))
return isMinMaxPattern(I, Kind, Prev);
- else if (isFMulAddIntrinsic(I))
+ if (isFPMinMaxRecurrenceKind(Kind)) {
+ if (HasRequiredFMF())
+ return isMinMaxPattern(I, Kind, Prev);
+ // We may be able to vectorize FMax/FMin reductions using maxnum/minnum
+ // intrinsics with extra checks ensuring the inputs are not NaN.
+ auto *StartV = dyn_cast<ConstantFP>(
+ OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
+ if (StartV && !StartV->getValue().isNaN() &&
+ isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
+ if (((Kind == RecurKind::FMax &&
+ match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
+ Kind == RecurKind::FMaxNumNoFMFs))
+ return InstDesc(I, RecurKind::FMaxNumNoFMFs);
+ if (((Kind == RecurKind::FMin &&
+ match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
+ Kind == RecurKind::FMinNumNoFMFs))
+ return InstDesc(I, RecurKind::FMinNumNoFMFs);
+ }
+ } else if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 200d1fb854155..0cf7bc7af878c 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,8 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
case RecurKind::UMin:
return Intrinsic::vector_reduce_umin;
case RecurKind::FMax:
+ case RecurKind::FMaxNumNoFMFs:
return Intrinsic::vector_reduce_fmax;
case RecurKind::FMin:
+ case RecurKind::FMinNumNoFMFs:
return Intrinsic::vector_reduce_fmin;
case RecurKind::FMaximum:
return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ceeabd65cced3..b070b04cc3c35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4347,8 +4347,15 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
ElementCount VF) const {
// Cross iteration phis such as reductions need special handling and are
// currently unsupported.
- if (any_of(OrigLoop->getHeader()->phis(),
- [&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
+ if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
+ if (Legal->isReductionVariable(&Phi)) {
+ RecurKind RK =
+ Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+ return RK == RecurKind::FMinNumNoFMFs ||
+ RK == RecurKind::FMaxNumNoFMFs;
+ }
+ return Legal->isFixedOrderRecurrence(&Phi);
+ }))
return false;
// Phis with uses outside of the loop require special handling and are
@@ -8769,6 +8776,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Adjust the recipes for any inloop reductions.
adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+ if (!VPlanTransforms::runPass(
+ VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
+ return nullptr;
// Transform recipes to abstract recipes if it is legal and beneficial and
// clamp the range for better cost estimation.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index da6af353c709f..b3040d0317c95 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,6 +23193,8 @@ class HorizontalReduction {
case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
+ case RecurKind::FMaxNumNoFMFs:
+ case RecurKind::FMinNumNoFMFs:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
@@ -23330,6 +23332,8 @@ class HorizontalReduction {
case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
+ case RecurKind::FMaxNumNoFMFs:
+ case RecurKind::FMinNumNoFMFs:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
@@ -23432,6 +23436,8 @@ class HorizontalReduction {
case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
+ case RecurKind::FMaxNumNoFMFs:
+ case RecurKind::FMinNumNoFMFs:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 204268e586b43..70c34ec3c25cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
unsigned Opcode;
public:
+ VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+ : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+ Opcode(I.getOpcode()) {}
+
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &Metadata,
DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode) {}
- VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
- : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
- Opcode(I.getOpcode()) {}
-
~VPWidenRecipe() override = default;
VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index b27a7ffeed208..66657b98b094b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -84,6 +84,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
return ResTy;
}
case Instruction::ICmp:
+ case Instruction::FCmp:
case VPInstruction::ActiveLaneMask:
assert(inferScalarType(R->getOperand(0)) ==
inferScalarType(R->getOperand(1)) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7fb5e82f9d32b..69efb9f89b479 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -652,3 +652,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
}
}
+
+static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
+ VPReductionPHIRecipe *RedPhi) {
+ auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+ if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+ !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+ return nullptr;
+
+ if (MinMaxOp->getOperand(0) == RedPhi)
+ return MinMaxOp->getOperand(1);
+ return MinMaxOp->getOperand(0);
+}
+
+/// Returns true if there VPlan is read-only and execution can be resumed at the
+/// beginning of the last vector iteration in the scalar loop
+static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
+ for (VPBlockBase *VPB : vp_depth_first_shallow(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+ auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+ if (!VPBB)
+ return false;
+ for (auto &R : *VPBB) {
+ if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+ continue;
+ if (R.mayWriteToMemory())
+ return false;
+ }
+ }
+ return true;
+}
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPValue *AnyNaN = nullptr;
+ VPReductionPHIRecipe *RedPhiR = nullptr;
+ VPRecipeWithIRFlags *MinMaxOp = nullptr;
+ bool HasUnsupportedPhi = false;
+ for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+ HasUnsupportedPhi |=
+ !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+ VPReductionPHIRecipe>(&R);
+ auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+ if (!Cur)
+ continue;
+ if (RedPhiR)
+ return false;
+ if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
+ Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+ continue;
+
+ RedPhiR = Cur;
+ MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+ RedPhiR->getBackedgeValue()->getDefiningRecipe());
+ if (!MinMaxOp)
+ return false;
+ VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
+ if (!In)
+ return false;
+
+ auto *IsNaN =
+ new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
+ IsNaN->insertBefore(MinMaxOp);
+ AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
+ AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
+ }
+
+ if (!AnyNaN)
+ return true;
+
+ if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+ return false;
+
+ auto *MiddleVPBB = Plan.getMiddleBlock();
+ auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
+ if (!RdxResult ||
+ RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
+ RdxResult->getOperand(0) != RedPhiR)
+ return false;
+
+ auto *ScalarPH = Plan.getScalarPreheader();
+ // Update the resume phis in the scalar preheader. They all must either resume
+ // from the reduction result or the canonical induction. Bail out if there are
+ // other resume phis.
+ for (auto &R : ScalarPH->phis()) {
+ auto *ResumeR = cast<VPPhi>(&R);
+ VPValue *VecV = ResumeR->getOperand(0);
+ VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
+ if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
+ return false;
+ ResumeR->setOperand(
+ 1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
+ ResumeR->addOperand(BypassV);
+ }
+
+ // Create a new reduction phi recipe with either FMin/FMax, replacing
+ // FMinNumNoFMFs/FMaxNumNoFMFs.
+ RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+ ? RecurKind::FMin
+ : RecurKind::FMax;
+ auto *NewRedPhiR = new VPReductionPHIRecipe(
+ cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
+ *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
+ NewRedPhiR->addOperand(RedPhiR->getOperand(1));
+ NewRedPhiR->insertBefore(RedPhiR);
+ RedPhiR->replaceAllUsesWith(NewRedPhiR);
+ RedPhiR->eraseFromParent();
+
+ // Update the loop exit condition to exit if either any of the inputs is NaN
+ // or the vector trip count is reached.
+ VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+ VPBuilder Builder(LatchVPBB->getTerminator());
+ auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+ assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+ "Unexpected terminator");
+ auto *IsLatchExitTaken =
+ Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+ LatchExitingBranch->getOperand(1));
+ auto *AnyExitTaken =
+ Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
+ Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
+ LatchExitingBranch->eraseFromParent();
+
+ // Split the middle block and introduce a new block, branching to the scalar
+ // preheader to resume iteration in the scalar loop if any NaNs have been
+ // encountered.
+ MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+ Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
+ auto *NewSel =
+ Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+ RdxResult->setOperand(1, NewSel);
+ Builder.setInsertPoint(MiddleVPBB);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
+ VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+ MiddleVPBB->swapSuccessors();
+ std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+ return true;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1664bcc3881aa..57b713d3dfcb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -587,6 +587,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
return Builder.CreateFreeze(Op, Name);
}
+ case Instruction::FCmp:
case Instruction::ICmp: {
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
@@ -860,7 +861,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *Res = State.get(getOperand(0));
for (VPValue *Op : drop_begin(operands()))
Res = Builder.CreateOr(Res, State.get(Op));
- return Builder.CreateOrReduce(Res);
+ return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
}
case VPInstruction::FirstActiveLane: {
if (getNumOperands() == 1) {
@@ -1033,6 +1034,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
switch (getOpcode()) {
case Instruction::ExtractElement:
case Instruction::Freeze:
+ case Instruction::FCmp:
case Instruction::ICmp:
case Instruction::Select:
case VPInstruction::AnyOf:
@@ -1068,6 +1070,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
return Op == getOperand(1);
case Instruction::PHI:
return true;
+ case Instruction::FCmp:
case Instruction::ICmp:
case Instruction::Select:
case Instruction::Or:
@@ -1100,6 +1103,7 @@ bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const {
switch (getOpcode()) {
default:
return false;
+ case Instruction::FCmp:
case Instruction::ICmp:
case Instruction::Select:
return vputils::onlyFirstPartUsed(this);
@@ -1786,7 +1790,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
return Opcode == Instruction::ZExt;
break;
case OperationType::Cmp:
- return Opcode == Instruction::ICmp;
+ return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
case OperationType::Other:
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 84a12470f45e4..52e33ad0d4f99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,6 +103,11 @@ struct VPlanTransforms {
/// not valid.
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
+ /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
+ /// If they do, try to update the vector loop to exit early if any input is
+ /// NaN and resume executing in the scalar loop to handle the NaNs there.
+ static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
+
/// Clear NSW/NUW flags from reduction instructions if necessary.
static void clearReductionWrapFlags(VPlan &Plan);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 451574a258c2b..1a9752a1ef9fd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fmaxnum(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index e93ee5563b057..3deaa5f0e1f60 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -42,18 +42,58 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fminnum(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index b2e080fef2e57..8ef5b2720e4cd 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -42,18 +42,58 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fmaxnum(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5661406b88a5a..332f3a60c5b3e 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fmaxnum_1(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -227,18 +258,49 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fmaxnum_2(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 148beb64a3609..0e3566b18432a 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -192,18 +192,49 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fminnum_1(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[L]], float [[MAX]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -227,18 +258,49 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-LABEL: define float @fminnum_2(
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
+; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC1:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC1]], align 4
; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.minnum.f32(float [[MAX]], float [[L]])
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 85a90f2e04c5e..1429dc26c5207 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1001,8 +1001,9 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmin_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.minnum.v2f32
+; CHECK: <2 x float> @llvm.minnum.v2f32
define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
@@ -1021,8 +1022,9 @@ for.body: ; preds = %entry, %for.body
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}
+; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmax_intrinsic_nofast(
-; CHECK-NOT: <2 x float> @llvm.maxnum.v2f32
+; CHECK: <2 x float> @llvm.maxnum.v2f32
define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
>From de2d6f2c89a3ead527a03c23eaaa4b7ce2ade11b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 09:53:59 +0100
Subject: [PATCH 2/6] !fixup address latest comments, thanks
---
llvm/include/llvm/Analysis/IVDescriptors.h | 4 +-
llvm/lib/Analysis/IVDescriptors.cpp | 31 ++--
llvm/lib/Transforms/Utils/LoopUtils.cpp | 4 +-
.../Vectorize/LoopVectorizationPlanner.h | 12 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 10 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 12 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +-
.../Vectorize/VPlanConstruction.cpp | 144 +++++++++---------
.../Transforms/Vectorize/VPlanTransforms.h | 1 +
.../AArch64/fmax-without-fast-math-flags.ll | 26 ++--
.../AArch64/fmin-without-fast-math-flags.ll | 26 ++--
...fmax-without-fast-math-flags-interleave.ll | 26 ++--
.../fmax-without-fast-math-flags.ll | 44 +++---
.../fmin-without-fast-math-flags.ll | 44 +++---
.../LoopVectorize/minmax_reduction.ll | 2 +
15 files changed, 212 insertions(+), 182 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index e28901c072e29..84a7672cd47f3 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
- FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
- FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
+ FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
+ FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index ef095b73d5419..accda5a9713fc 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -944,23 +944,24 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
if (isIntMinMaxRecurrenceKind(Kind))
return isMinMaxPattern(I, Kind, Prev);
if (isFPMinMaxRecurrenceKind(Kind)) {
+ InstDesc Res = isMinMaxPattern(I, Kind, Prev);
if (HasRequiredFMF())
- return isMinMaxPattern(I, Kind, Prev);
+ return Res;
+
+ if (!Res.isRecurrence())
+ return InstDesc(false, I);
+
// We may be able to vectorize FMax/FMin reductions using maxnum/minnum
- // intrinsics with extra checks ensuring the inputs are not NaN.
- auto *StartV = dyn_cast<ConstantFP>(
- OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
- if (StartV && !StartV->getValue().isNaN() &&
- isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
- if (((Kind == RecurKind::FMax &&
- match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
- Kind == RecurKind::FMaxNumNoFMFs))
- return InstDesc(I, RecurKind::FMaxNumNoFMFs);
- if (((Kind == RecurKind::FMin &&
- match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
- Kind == RecurKind::FMinNumNoFMFs))
- return InstDesc(I, RecurKind::FMinNumNoFMFs);
- }
+ // intrinsics with extra checks ensuring the vector loop handles only
+ // non-NaN inputs.
+ if (Kind == RecurKind::FMax &&
+ match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+ return InstDesc(I, RecurKind::FMaxNum);
+ if (Kind == RecurKind::FMin &&
+ match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+ return InstDesc(I, RecurKind::FMinNum);
+
+ return InstDesc(false, I);
} else if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 0cf7bc7af878c..7bb7a7b116384 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -938,10 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
case RecurKind::UMin:
return Intrinsic::vector_reduce_umin;
case RecurKind::FMax:
- case RecurKind::FMaxNumNoFMFs:
+ case RecurKind::FMaxNum:
return Intrinsic::vector_reduce_fmax;
case RecurKind::FMin:
- case RecurKind::FMinNumNoFMFs:
+ case RecurKind::FMinNum:
return Intrinsic::vector_reduce_fmin;
case RecurKind::FMaximum:
return Intrinsic::vector_reduce_fmaximum;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 11853859484e3..f57ce0c3ccb4d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -230,7 +230,6 @@ class VPBuilder {
/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
/// and \p B.
- /// TODO: add createFCmp when needed.
VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
@@ -240,6 +239,17 @@ class VPBuilder {
new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
}
+ /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A
+ /// and \p B.
+ VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
+ DebugLoc DL = DebugLoc::getUnknown(),
+ const Twine &Name = "") {
+ assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE &&
+ Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate");
+ return tryInsertInstruction(
+ new VPInstruction(Instruction::FCmp, {A, B}, Pred, DL, Name));
+ }
+
VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b070b04cc3c35..712c5f8eb4c56 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,14 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
ElementCount VF) const {
- // Cross iteration phis such as reductions need special handling and are
- // currently unsupported.
+ // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+ // reductions need special handling and are currently unsupported.
if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
if (Legal->isReductionVariable(&Phi)) {
RecurKind RK =
Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
- return RK == RecurKind::FMinNumNoFMFs ||
- RK == RecurKind::FMaxNumNoFMFs;
+ return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
}
return Legal->isFixedOrderRecurrence(&Phi);
}))
@@ -8776,6 +8775,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// Adjust the recipes for any inloop reductions.
adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
+
+ // Apply mandatory transformation to handle FP maxnum/minnum reduction with
+ // NaNs if possible, bail out otherwise.
if (!VPlanTransforms::runPass(
VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b3040d0317c95..c1e4ff00a65e9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23193,8 +23193,8 @@ class HorizontalReduction {
case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
- case RecurKind::FMaxNumNoFMFs:
- case RecurKind::FMinNumNoFMFs:
+ case RecurKind::FMaxNum:
+ case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
@@ -23332,8 +23332,8 @@ class HorizontalReduction {
case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
- case RecurKind::FMaxNumNoFMFs:
- case RecurKind::FMinNumNoFMFs:
+ case RecurKind::FMaxNum:
+ case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
@@ -23436,8 +23436,8 @@ class HorizontalReduction {
case RecurKind::FindFirstIVUMin:
case RecurKind::FindLastIVSMax:
case RecurKind::FindLastIVUMax:
- case RecurKind::FMaxNumNoFMFs:
- case RecurKind::FMinNumNoFMFs:
+ case RecurKind::FMaxNum:
+ case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 70c34ec3c25cd..204268e586b43 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1356,16 +1356,16 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
unsigned Opcode;
public:
- VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
- : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
- Opcode(I.getOpcode()) {}
-
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags, const VPIRMetadata &Metadata,
DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode) {}
+ VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
+ : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+ Opcode(I.getOpcode()) {}
+
~VPWidenRecipe() override = default;
VPWidenRecipe *clone() override {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 69efb9f89b479..8d1ec5b68457e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -653,77 +653,75 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
}
}
-static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
- VPReductionPHIRecipe *RedPhi) {
- auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
- if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
- !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
- return nullptr;
-
- if (MinMaxOp->getOperand(0) == RedPhi)
- return MinMaxOp->getOperand(1);
- return MinMaxOp->getOperand(0);
-}
-
-/// Returns true if there VPlan is read-only and execution can be resumed at the
-/// beginning of the last vector iteration in the scalar loop
-static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
- for (VPBlockBase *VPB : vp_depth_first_shallow(
- Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
- auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
- if (!VPBB)
- return false;
- for (auto &R : *VPBB) {
- if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
- continue;
- if (R.mayWriteToMemory())
- return false;
- }
- }
- return true;
-}
-
bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPValue *AnyNaN = nullptr;
VPReductionPHIRecipe *RedPhiR = nullptr;
- VPRecipeWithIRFlags *MinMaxOp = nullptr;
+ VPValue *MinMaxOp = nullptr;
bool HasUnsupportedPhi = false;
+
+ auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
+ VPReductionPHIRecipe *RedPhi) -> VPValue * {
+ auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+ if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+ !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+ return nullptr;
+
+ if (MinMaxOp->getOperand(0) == RedPhi)
+ return MinMaxOp->getOperand(1);
+ assert(MinMaxOp->getOperand(1) == RedPhi &&
+ "Reduction phi operand expected");
+ return MinMaxOp->getOperand(0);
+ };
+
for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+ // TODO: Also support first-order recurrence phis.
HasUnsupportedPhi |=
!isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
VPReductionPHIRecipe>(&R);
auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
if (!Cur)
continue;
+ // For now, only a single reduction is supported.
+ // TODO: Support multiple MaxNum/MinNum reductions and other reductions.
if (RedPhiR)
return false;
- if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
- Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+ if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
+ Cur->getRecurrenceKind() != RecurKind::FMinNum)
continue;
RedPhiR = Cur;
- MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+ auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
RedPhiR->getBackedgeValue()->getDefiningRecipe());
- if (!MinMaxOp)
+ if (!MinMaxR)
return false;
- VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
- if (!In)
+ MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
+ if (!MinMaxOp)
return false;
-
- auto *IsNaN =
- new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
- IsNaN->insertBefore(MinMaxOp);
- AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
- AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
}
- if (!AnyNaN)
+ if (!RedPhiR)
return true;
- if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+ if (HasUnsupportedPhi || !Plan.hasScalarTail())
return false;
+ /// Check if the vector loop of \p Plan can early exit and restart
+ /// execution of last vector iteration in the scalar loop. This requires all
+ /// recipes up to early exit point be side-effect free as they are
+ /// re-executed. Currently we check that the loop is free of any recipe that
+ /// may write to memory. Expected to operate on an early VPlan w/o nested
+ /// regions.
+ for (VPBlockBase *VPB : vp_depth_first_shallow(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+ auto *VPBB = cast<VPBasicBlock>(VPB);
+ for (auto &R : *VPBB) {
+ if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+ continue;
+ if (R.mayWriteToMemory())
+ return false;
+ }
+ }
+
auto *MiddleVPBB = Plan.getMiddleBlock();
auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
if (!RdxResult ||
@@ -731,24 +729,9 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
RdxResult->getOperand(0) != RedPhiR)
return false;
- auto *ScalarPH = Plan.getScalarPreheader();
- // Update the resume phis in the scalar preheader. They all must either resume
- // from the reduction result or the canonical induction. Bail out if there are
- // other resume phis.
- for (auto &R : ScalarPH->phis()) {
- auto *ResumeR = cast<VPPhi>(&R);
- VPValue *VecV = ResumeR->getOperand(0);
- VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
- if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
- return false;
- ResumeR->setOperand(
- 1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
- ResumeR->addOperand(BypassV);
- }
-
// Create a new reduction phi recipe with either FMin/FMax, replacing
- // FMinNumNoFMFs/FMaxNumNoFMFs.
- RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
+ // FMinNum/FMaxNum.
+ RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
? RecurKind::FMin
: RecurKind::FMax;
auto *NewRedPhiR = new VPReductionPHIRecipe(
@@ -769,23 +752,40 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
auto *IsLatchExitTaken =
Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
LatchExitingBranch->getOperand(1));
+
+ VPValue *IsNaN = Builder.createFCmp(CmpInst::FCMP_UNO, MinMaxOp, MinMaxOp);
+ VPValue *AnyNaN = Builder.createNaryOp(VPInstruction::AnyOf, {IsNaN});
auto *AnyExitTaken =
Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
LatchExitingBranch->eraseFromParent();
- // Split the middle block and introduce a new block, branching to the scalar
- // preheader to resume iteration in the scalar loop if any NaNs have been
- // encountered.
- MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
+ // If we exit early due to NaNs, compute the final reduction result based on
+ // the reduction phi at the beginning of the last vector iteration.
Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
auto *NewSel =
Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
RdxResult->setOperand(1, NewSel);
- Builder.setInsertPoint(MiddleVPBB);
- Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
- VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
- MiddleVPBB->swapSuccessors();
- std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
+
+ auto *ScalarPH = Plan.getScalarPreheader();
+ // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+ // true, the resume from the start of the last vector iteration via the
+ // canonical IV, otherwise from the original value.
+ for (auto &R : ScalarPH->phis()) {
+ auto *ResumeR = cast<VPPhi>(&R);
+ VPValue *VecV = ResumeR->getOperand(0);
+ if (VecV == RdxResult)
+ continue;
+ if (VecV != &Plan.getVectorTripCount())
+ return false;
+ auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
+ ResumeR->setOperand(0, NewSel);
+ }
+
+ auto *MiddleTerm = MiddleVPBB->getTerminator();
+ Builder.setInsertPoint(MiddleTerm);
+ VPValue *MiddleCond = MiddleTerm->getOperand(0);
+ VPValue *NewCond = Builder.createAnd(MiddleCond, Builder.createNot(AnyNaN));
+ MiddleTerm->setOperand(0, NewCond);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 52e33ad0d4f99..b1692d3e6b337 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -106,6 +106,7 @@ struct VPlanTransforms {
/// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
/// If they do, try to update the vector loop to exit early if any input is
/// NaN and resume executing in the scalar loop to handle the NaNs there.
+ /// Return false if this attempt was unsuccessful.
static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
/// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 1a9752a1ef9fd..6810c90f0b829 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index 3deaa5f0e1f60..a18714f94343b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -57,31 +57,33 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 8ef5b2720e4cd..5e60b32e3ae6b 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -57,31 +57,33 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP8]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP6]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
-; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
-; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
+; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -93,7 +95,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 332f3a60c5b3e..5199bc2bad1da 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -270,25 +272,27 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 0e3566b18432a..f220b900d4d0f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -204,25 +204,27 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -234,7 +236,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
@@ -270,25 +272,27 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP4]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]]
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
+; CHECK-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[MIDDLE_BLOCK_SPLIT:.*]]
-; CHECK: [[MIDDLE_BLOCK_SPLIT]]:
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[IV]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ -1.000000e+07, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -300,7 +304,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK_SPLIT]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
index 1429dc26c5207..e7ab02cd98a5e 100644
--- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll
@@ -1004,6 +1004,7 @@ for.body: ; preds = %entry, %for.body
; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmin_intrinsic_nofast(
; CHECK: <2 x float> @llvm.minnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
define float @fmin_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
@@ -1025,6 +1026,7 @@ for.body: ; preds = %entry, %for.body
; This can be vectorized with additional runtime checks for NaNs.
; CHECK-LABEL: @fmax_intrinsic_nofast(
; CHECK: <2 x float> @llvm.maxnum.v2f32
+; CHECK: fcmp uno <2 x float> [[OP:.+]], [[OP]]
define float @fmax_intrinsic_nofast(ptr nocapture readonly %x) {
entry:
br label %for.body
>From 8ebc7ff41e3f68c42a99492fce9a5f55e5f23bde Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 17 Jul 2025 15:36:12 +0100
Subject: [PATCH 3/6] !fixup add assertion for intrinsic ID
---
.../lib/Transforms/Vectorize/VPlanConstruction.cpp | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8d1ec5b68457e..21e3ec3e3f0fe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -697,6 +697,20 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
if (!MinMaxOp)
return false;
+
+#ifndef NDEBUG
+ Intrinsic::ID RdxIntrinsicId =
+ Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+ : Intrinsic::minnum;
+ assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+ cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+ RdxIntrinsicId) ||
+ (isa<VPReplicateRecipe>(MinMaxR) &&
+ cast<IntrinsicInst>(
+ cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
+ ->getIntrinsicID() == RdxIntrinsicId) &&
+ "Intrinsic did not match recurrence kind");
+#endif
}
if (!RedPhiR)
>From 914a0b4b32617cc63b1c902402fd670e4055267d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 10:27:24 +0100
Subject: [PATCH 4/6] !fixup address latest comments, thanks
---
llvm/include/llvm/Analysis/IVDescriptors.h | 5 +-
llvm/lib/Analysis/IVDescriptors.cpp | 18 +--
llvm/lib/Transforms/Utils/LoopUtils.cpp | 7 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 12 +-
.../Vectorize/VPlanConstruction.cpp | 108 ++++++++----------
.../AArch64/fmax-without-fast-math-flags.ll | 7 +-
.../AArch64/fmin-without-fast-math-flags.ll | 7 +-
...fmax-without-fast-math-flags-interleave.ll | 7 +-
.../fmax-without-fast-math-flags.ll | 4 +-
.../fmin-without-fast-math-flags.ll | 4 +-
10 files changed, 88 insertions(+), 91 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 84a7672cd47f3..1dc73205a0ebb 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -47,8 +47,8 @@ enum class RecurKind {
FMul, ///< Product of floats.
FMin, ///< FP min implemented in terms of select(cmp()).
FMax, ///< FP max implemented in terms of select(cmp()).
- FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
- FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
+ FMinNum, ///< FP min with llvm.minnum semantics including NaNs.
+ FMaxNum, ///< FP max with llvm.maxnum semantics including NaNs.
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
@@ -252,6 +252,7 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is a floating-point min/max kind.
static bool isFPMinMaxRecurrenceKind(RecurKind Kind) {
return Kind == RecurKind::FMin || Kind == RecurKind::FMax ||
+ Kind == RecurKind::FMinNum || Kind == RecurKind::FMaxNum ||
Kind == RecurKind::FMinimum || Kind == RecurKind::FMaximum ||
Kind == RecurKind::FMinimumNum || Kind == RecurKind::FMaximumNum;
}
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index accda5a9713fc..a88bbd7b65d53 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -945,21 +945,25 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
return isMinMaxPattern(I, Kind, Prev);
if (isFPMinMaxRecurrenceKind(Kind)) {
InstDesc Res = isMinMaxPattern(I, Kind, Prev);
- if (HasRequiredFMF())
- return Res;
-
if (!Res.isRecurrence())
return InstDesc(false, I);
+ if (HasRequiredFMF())
+ return Res;
+
// We may be able to vectorize FMax/FMin reductions using maxnum/minnum
// intrinsics with extra checks ensuring the vector loop handles only
// non-NaN inputs.
- if (Kind == RecurKind::FMax &&
- match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+ if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) {
+ assert(Kind == RecurKind::FMax &&
+ "unexpected recurrence kind for maxnum");
return InstDesc(I, RecurKind::FMaxNum);
- if (Kind == RecurKind::FMin &&
- match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+ }
+ if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) {
+ assert(Kind == RecurKind::FMin &&
+ "unexpected recurrence kind for minnum");
return InstDesc(I, RecurKind::FMinNum);
+ }
return InstDesc(false, I);
} else if (isFMulAddIntrinsic(I))
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 7bb7a7b116384..154f50acf3d1d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1039,8 +1039,10 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
case RecurKind::SMax:
return Intrinsic::smax;
case RecurKind::FMin:
+ case RecurKind::FMinNum:
return Intrinsic::minnum;
case RecurKind::FMax:
+ case RecurKind::FMaxNum:
return Intrinsic::maxnum;
case RecurKind::FMinimum:
return Intrinsic::minimum;
@@ -1098,7 +1100,8 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
Value *Right) {
Type *Ty = Left->getType();
if (Ty->isIntOrIntVectorTy() ||
- (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
+ (RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
+ RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
// TODO: Add float minnum/maxnum support when FMF nnan is set.
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
@@ -1310,6 +1313,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
case RecurKind::UMin:
case RecurKind::FMax:
case RecurKind::FMin:
+ case RecurKind::FMinNum:
+ case RecurKind::FMaxNum:
case RecurKind::FMinimum:
case RecurKind::FMaximum:
case RecurKind::FMinimumNum:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 712c5f8eb4c56..db0902c589d49 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4345,15 +4345,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
ElementCount VF) const {
- // Cross iteration phis such as first-order recurrences and FMaxNum/FMinNum
+ // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
// reductions need special handling and are currently unsupported.
if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
- if (Legal->isReductionVariable(&Phi)) {
- RecurKind RK =
- Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
- return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
- }
- return Legal->isFixedOrderRecurrence(&Phi);
+ if (!Legal->isReductionVariable(&Phi))
+ return Legal->isFixedOrderRecurrence(&Phi);
+ RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
+ return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
}))
return false;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 21e3ec3e3f0fe..a426d06b4befe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -656,25 +656,42 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPReductionPHIRecipe *RedPhiR = nullptr;
- VPValue *MinMaxOp = nullptr;
bool HasUnsupportedPhi = false;
- auto GetMinMaxCompareValue = [](VPSingleDefRecipe *MinMaxOp,
- VPReductionPHIRecipe *RedPhi) -> VPValue * {
- auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
- if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+ auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
+ auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
+ RedPhiR->getBackedgeValue()->getDefiningRecipe());
+ if (!MinMaxR)
+ return nullptr;
+
+ auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
+ if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
!(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
return nullptr;
- if (MinMaxOp->getOperand(0) == RedPhi)
- return MinMaxOp->getOperand(1);
- assert(MinMaxOp->getOperand(1) == RedPhi &&
+#ifndef NDEBUG
+ Intrinsic::ID RdxIntrinsicId =
+ RedPhiR->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
+ : Intrinsic::minnum;
+ assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
+ cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
+ RdxIntrinsicId) ||
+ (RepR &&
+ cast<IntrinsicInst>(RepR->getUnderlyingInstr())->getIntrinsicID() ==
+ RdxIntrinsicId) &&
+ "Intrinsic did not match recurrence kind");
+#endif
+
+ if (MinMaxR->getOperand(0) == RedPhiR)
+ return MinMaxR->getOperand(1);
+
+ assert(MinMaxR->getOperand(1) == RedPhiR &&
"Reduction phi operand expected");
- return MinMaxOp->getOperand(0);
+ return MinMaxR->getOperand(0);
};
for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
- // TODO: Also support first-order recurrence phis.
+ // TODO: Also support fixed-order recurrence phis.
HasUnsupportedPhi |=
!isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
VPReductionPHIRecipe>(&R);
@@ -688,34 +705,23 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
Cur->getRecurrenceKind() != RecurKind::FMinNum)
continue;
-
RedPhiR = Cur;
- auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
- RedPhiR->getBackedgeValue()->getDefiningRecipe());
- if (!MinMaxR)
- return false;
- MinMaxOp = GetMinMaxCompareValue(MinMaxR, RedPhiR);
- if (!MinMaxOp)
- return false;
-
-#ifndef NDEBUG
- Intrinsic::ID RdxIntrinsicId =
- Cur->getRecurrenceKind() == RecurKind::FMaxNum ? Intrinsic::maxnum
- : Intrinsic::minnum;
- assert((isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
- cast<VPWidenIntrinsicRecipe>(MinMaxR)->getVectorIntrinsicID() ==
- RdxIntrinsicId) ||
- (isa<VPReplicateRecipe>(MinMaxR) &&
- cast<IntrinsicInst>(
- cast<VPReplicateRecipe>(MinMaxR)->getUnderlyingInstr())
- ->getIntrinsicID() == RdxIntrinsicId) &&
- "Intrinsic did not match recurrence kind");
-#endif
}
if (!RedPhiR)
return true;
+ RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+ assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+ "unsupported reduction");
+
+ VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+ if (!MinMaxOp)
+ return false;
+
+ // We won't be able to resume execution in the scalar tail, if there are
+ // unsupported header phis or there is no scalar tail at all, due to
+ // tail-folding.
if (HasUnsupportedPhi || !Plan.hasScalarTail())
return false;
@@ -729,35 +735,12 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
auto *VPBB = cast<VPBasicBlock>(VPB);
for (auto &R : *VPBB) {
- if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
- continue;
- if (R.mayWriteToMemory())
+ if (R.mayWriteToMemory() &&
+ !match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
return false;
}
}
- auto *MiddleVPBB = Plan.getMiddleBlock();
- auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
- if (!RdxResult ||
- RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
- RdxResult->getOperand(0) != RedPhiR)
- return false;
-
- // Create a new reduction phi recipe with either FMin/FMax, replacing
- // FMinNum/FMaxNum.
- RecurKind NewRK = RedPhiR->getRecurrenceKind() == RecurKind::FMinNum
- ? RecurKind::FMin
- : RecurKind::FMax;
- auto *NewRedPhiR = new VPReductionPHIRecipe(
- cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
- *RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
- NewRedPhiR->addOperand(RedPhiR->getOperand(1));
- NewRedPhiR->insertBefore(RedPhiR);
- RedPhiR->replaceAllUsesWith(NewRedPhiR);
- RedPhiR->eraseFromParent();
-
- // Update the loop exit condition to exit if either any of the inputs is NaN
- // or the vector trip count is reached.
VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
VPBuilder Builder(LatchVPBB->getTerminator());
auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
@@ -776,9 +759,18 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
// If we exit early due to NaNs, compute the final reduction result based on
// the reduction phi at the beginning of the last vector iteration.
+ auto *RdxResult = find_singleton<VPSingleDefRecipe>(
+ RedPhiR->users(), [](VPUser *U, bool) -> VPSingleDefRecipe * {
+ auto *VPI = dyn_cast<VPInstruction>(U);
+ if (VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult)
+ return VPI;
+ return nullptr;
+ });
+
+ auto *MiddleVPBB = Plan.getMiddleBlock();
Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
auto *NewSel =
- Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
+ Builder.createSelect(AnyNaN, RedPhiR, RdxResult->getOperand(1));
RdxResult->setOperand(1, NewSel);
auto *ScalarPH = Plan.getScalarPreheader();
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 6810c90f0b829..427a05cc1c843 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index a18714f94343b..1a8e5940d88e7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -50,8 +50,8 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0x7FF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fminnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp olt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 5e60b32e3ae6b..a2eddad179216 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -50,8 +50,8 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float 0xFFF8000000000000), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 4
@@ -73,8 +73,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP8]]
; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP6]], i64 [[IV]], i64 [[N_VEC]]
-; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp ogt <4 x float> [[TMP11]], [[TMP12]]
-; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP11]], <4 x float> [[TMP12]]
+; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
; CHECK-NEXT: [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-NEXT: [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 5199bc2bad1da..6204d540df40c 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0xFFF8000000000000, float 0xFFF8000000000000, float 0xFFF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index f220b900d4d0f..68bc8d0640a3f 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -200,7 +200,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
@@ -268,7 +268,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ <float -1.000000e+07, float 0x7FF8000000000000, float 0x7FF8000000000000, float 0x7FF8000000000000>, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ splat (float -1.000000e+07), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[GEP_SRC]], i32 0
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
>From 18312dc501beba7c0c77b50c7f6e344f44174ee3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:19:36 +0100
Subject: [PATCH 5/6] Merge remote-tracking branch 'origin/main' into
lv-vectorize-fmaxnum-without-fmfs
---
.../workflows/build-ci-container-windows.yml | 2 -
.github/workflows/build-ci-container.yml | 2 -
clang/docs/ReleaseNotes.rst | 3 +
clang/include/clang/Basic/BuiltinsAMDGPU.def | 1 +
clang/include/clang/Sema/Overload.h | 2 -
.../Core/PathSensitive/ExprEngine.h | 7 +-
clang/lib/AST/ByteCode/Descriptor.cpp | 2 +
clang/lib/AST/ByteCode/Descriptor.h | 4 +
clang/lib/AST/ByteCode/Disasm.cpp | 1 +
clang/lib/AST/ByteCode/Interp.cpp | 5 +-
clang/lib/AST/ByteCode/InterpBlock.h | 5 -
clang/lib/AST/ByteCode/InterpBuiltin.cpp | 3 +-
clang/lib/AST/ByteCode/InterpFrame.cpp | 5 +
clang/lib/AST/ByteCode/Pointer.h | 5 +
clang/lib/CodeGen/CGCall.cpp | 15 +-
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 1 +
clang/lib/Driver/ToolChain.cpp | 40 +-
clang/lib/Sema/SemaConcept.cpp | 5 +
clang/lib/Sema/SemaOverload.cpp | 49 +-
clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 46 +-
clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp | 48 --
clang/test/AST/ByteCode/mutable.cpp | 56 ++-
clang/test/AST/ByteCode/unions.cpp | 14 +
clang/test/Analysis/div-zero-cxx20.cpp | 61 +++
clang/test/Analysis/div-zero.cpp | 60 +++
clang/test/CodeGen/64bit-swiftcall.c | 2 +-
clang/test/CodeGen/AArch64/byval-temp.c | 16 +-
.../AArch64/pure-scalable-args-empty-union.c | 2 +-
.../test/CodeGen/AArch64/pure-scalable-args.c | 42 +-
.../AArch64/struct-coerce-using-ptr.cpp | 4 +-
...-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c | 6 +-
...cle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp | 2 +-
clang/test/CodeGen/LoongArch/bitint.c | 6 +-
clang/test/CodeGen/PowerPC/ppc64-vector.c | 2 +-
clang/test/CodeGen/RISCV/riscv-abi.cpp | 8 +-
.../RISCV/riscv-vector-callingconv-llvm-ir.c | 10 +-
.../riscv-vector-callingconv-llvm-ir.cpp | 10 +-
clang/test/CodeGen/RISCV/riscv32-abi.c | 74 +--
clang/test/CodeGen/RISCV/riscv32-vararg.c | 2 +-
clang/test/CodeGen/RISCV/riscv64-abi.c | 18 +-
clang/test/CodeGen/RISCV/riscv64-vararg.c | 2 +-
.../test/CodeGen/SystemZ/systemz-abi-vector.c | 52 +-
clang/test/CodeGen/SystemZ/systemz-abi.c | 38 +-
.../test/CodeGen/SystemZ/systemz-inline-asm.c | 2 +-
clang/test/CodeGen/X86/cx-complex-range.c | 2 +-
.../test/CodeGen/X86/x86_32-arguments-win32.c | 14 +-
.../test/CodeGen/X86/x86_64-arguments-win32.c | 2 +-
clang/test/CodeGen/aapcs64-align.cpp | 4 +-
clang/test/CodeGen/arm-aapcs-vfp.c | 2 +-
clang/test/CodeGen/arm-abi-vector.c | 6 +-
clang/test/CodeGen/arm-swiftcall.c | 2 +-
clang/test/CodeGen/arm64-abi-vector.c | 14 +-
clang/test/CodeGen/arm64-arguments.c | 26 +-
.../CodeGen/arm64-microsoft-arguments.cpp | 2 +-
clang/test/CodeGen/armv7k-abi.c | 2 +-
clang/test/CodeGen/atomic-arm64.c | 2 +-
clang/test/CodeGen/attr-noundef.cpp | 7 +-
clang/test/CodeGen/cx-complex-range.c | 36 +-
clang/test/CodeGen/ext-int-cc.c | 36 +-
clang/test/CodeGen/isfpclass.c | 2 +-
.../math-libcalls-tbaa-indirect-args.c | 86 ++--
clang/test/CodeGen/mingw-long-double.c | 6 +-
clang/test/CodeGen/ms_abi.c | 4 +-
clang/test/CodeGen/pass-by-value-noalias.c | 4 +-
clang/test/CodeGen/ptrauth-in-c-struct.c | 4 +-
clang/test/CodeGen/regcall.c | 10 +-
clang/test/CodeGen/regcall2.c | 2 +-
clang/test/CodeGen/regcall4.c | 10 +-
clang/test/CodeGen/sparcv9-abi.c | 4 +-
clang/test/CodeGen/vectorcall.c | 46 +-
clang/test/CodeGen/win-fp128.c | 2 +-
clang/test/CodeGen/win64-i128.c | 4 +-
clang/test/CodeGen/windows-swiftcall.c | 2 +-
.../CodeGenCXX/aarch64-mangle-sve-vectors.cpp | 4 +-
clang/test/CodeGenCXX/arm-cc.cpp | 2 +-
.../CodeGenCXX/attr-target-mv-inalloca.cpp | 8 +-
clang/test/CodeGenCXX/copy-initialization.cpp | 2 +-
clang/test/CodeGenCXX/debug-info.cpp | 2 +-
.../empty-nontrivially-copyable.cpp | 4 +-
clang/test/CodeGenCXX/fastcall.cpp | 2 +-
.../CodeGenCXX/homogeneous-aggregates.cpp | 14 +-
clang/test/CodeGenCXX/inalloca-lambda.cpp | 6 +-
.../test/CodeGenCXX/inalloca-overaligned.cpp | 8 +-
clang/test/CodeGenCXX/inalloca-vector.cpp | 4 +-
.../CodeGenCXX/inheriting-constructor.cpp | 8 +-
.../member-function-pointer-calls.cpp | 4 +-
.../CodeGenCXX/microsoft-abi-arg-order.cpp | 4 +-
.../CodeGenCXX/microsoft-abi-byval-thunks.cpp | 12 +-
.../microsoft-abi-member-pointers.cpp | 6 +-
.../microsoft-abi-sret-and-byval.cpp | 26 +-
.../CodeGenCXX/microsoft-abi-unknown-arch.cpp | 2 +-
clang/test/CodeGenCXX/ms-property.cpp | 4 +-
clang/test/CodeGenCXX/nrvo.cpp | 4 +-
.../test/CodeGenCXX/pass-by-value-noalias.cpp | 12 +-
.../CodeGenCXX/ptrauth-qualifier-struct.cpp | 2 +-
clang/test/CodeGenCXX/regparm.cpp | 2 +-
clang/test/CodeGenCXX/trivial_abi.cpp | 8 +-
clang/test/CodeGenCXX/uncopyable-args.cpp | 32 +-
clang/test/CodeGenCXX/wasm-args-returns.cpp | 12 +-
.../test/CodeGenCXX/windows-x86-swiftcall.cpp | 4 +-
.../nontrivial-c-struct-exception.m | 2 +-
.../test/CodeGenObjC/pass-by-value-noalias.m | 4 +-
clang/test/CodeGenObjC/weak-in-c-struct.m | 6 +-
.../test/CodeGenObjCXX/objc-struct-cxx-abi.mm | 12 +-
clang/test/CodeGenObjCXX/property-objects.mm | 2 +-
.../CodeGenObjCXX/ptrauth-struct-cxx-abi.mm | 2 +-
.../CodeGenOpenCL/builtins-amdgcn-gfx1250.cl | 38 ++
.../test/Driver/print-multi-selection-flags.c | 19 +
clang/test/Headers/stdarg.cpp | 4 +-
.../test/OpenMP/for_firstprivate_codegen.cpp | 30 +-
.../OpenMP/parallel_firstprivate_codegen.cpp | 216 ++++-----
.../OpenMP/sections_firstprivate_codegen.cpp | 14 +-
.../OpenMP/single_firstprivate_codegen.cpp | 14 +-
..._teams_distribute_firstprivate_codegen.cpp | 82 ++--
...bute_parallel_for_firstprivate_codegen.cpp | 224 ++++-----
.../teams_distribute_firstprivate_codegen.cpp | 90 ++--
...bute_parallel_for_firstprivate_codegen.cpp | 124 ++---
.../OpenMP/teams_firstprivate_codegen.cpp | 80 ++--
clang/test/Sema/dllexport.c | 4 +
...overload-resolution-deferred-templates.cpp | 28 ++
.../test/SemaTemplate/concepts-using-decl.cpp | 21 +
flang/lib/Lower/Bridge.cpp | 15 +-
flang/lib/Lower/ConvertConstant.cpp | 2 +-
flang/lib/Lower/ConvertExpr.cpp | 26 +-
flang/lib/Lower/ConvertVariable.cpp | 26 +-
flang/lib/Lower/Runtime.cpp | 4 +-
flang/lib/Lower/VectorSubscripts.cpp | 2 +-
flang/lib/Optimizer/Builder/FIRBuilder.cpp | 2 +-
.../Optimizer/Builder/Runtime/Intrinsics.cpp | 6 +-
flang/lib/Optimizer/Builder/Runtime/Stop.cpp | 2 +-
.../LowerHLFIROrderedAssignments.cpp | 4 +-
.../lib/Optimizer/Transforms/MemoryUtils.cpp | 2 +-
.../Transforms/PolymorphicOpConversion.cpp | 2 +-
.../Transforms/SimplifyFIROperations.cpp | 2 +-
flang/test/Lower/CUDA/cuda-set-allocator.cuf | 32 ++
.../Optimizer/FortranVariableTest.cpp | 6 +-
libc/cmake/modules/LLVMLibCTestRules.cmake | 17 +-
libcxx/include/__config_site.in | 1 -
libcxx/test/libcxx/fuzzing/random.pass.cpp | 7 +-
.../std/depr/depr.c.headers/math_h.pass.cpp | 7 +-
.../test/std/numerics/c.math/cmath.pass.cpp | 7 +-
lld/ELF/BPSectionOrderer.cpp | 6 +-
lld/test/ELF/bp-section-orderer.s | 21 +-
lldb/source/Expression/IRExecutionUnit.cpp | 16 +-
.../TestNSDictionarySynthetic.py | 3 +-
.../PDB/Inputs/UdtLayoutTest.script | 1 +
llvm/Maintainers.md | 14 +
llvm/docs/CodingStandards.rst | 46 +-
.../llvm/CodeGen/MachineBlockFrequencyInfo.h | 4 +-
llvm/include/llvm/IR/DebugInfo.h | 5 +-
llvm/include/llvm/IR/DebugInfoMetadata.h | 3 -
.../llvm/Transforms/Utils/SSAUpdater.h | 4 -
llvm/lib/CodeGen/CodeGenPrepare.cpp | 59 +--
.../lib/CodeGen/MachineBlockFrequencyInfo.cpp | 4 +-
llvm/lib/CodeGen/MachineDebugify.cpp | 18 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 16 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 4 +-
llvm/lib/IR/DebugInfo.cpp | 25 +-
llvm/lib/IR/DebugInfoMetadata.cpp | 9 -
llvm/lib/Object/IRSymtab.cpp | 45 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 4 +-
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp | 108 +++++
.../Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h | 23 +
.../AMDGPU/AMDGPUResourceUsageAnalysis.cpp | 234 +--------
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 13 +
llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h | 2 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 146 ++++--
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 1 -
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 8 +-
llvm/lib/Target/AMDGPU/SIRegisterInfo.h | 6 +-
llvm/lib/Target/AMDGPU/VOP1Instructions.td | 1 +
.../LoongArch/LoongArchFrameLowering.cpp | 7 +-
llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 122 +++--
llvm/lib/Target/X86/X86CallingConv.cpp | 2 +-
llvm/lib/TargetParser/X86TargetParser.cpp | 6 +-
llvm/lib/Transforms/Coroutines/CoroFrame.cpp | 52 +-
llvm/lib/Transforms/Coroutines/CoroInternal.h | 5 +-
llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 23 +-
llvm/lib/Transforms/Coroutines/SpillUtils.cpp | 6 +-
llvm/lib/Transforms/IPO/MergeFunctions.cpp | 19 +-
.../InstCombine/InstCombineInternal.h | 3 -
.../InstCombine/InstructionCombining.cpp | 78 +--
.../Scalar/ConstraintElimination.cpp | 1 +
.../Scalar/DeadStoreElimination.cpp | 1 +
llvm/lib/Transforms/Scalar/JumpThreading.cpp | 39 +-
.../lib/Transforms/Scalar/LoopInterchange.cpp | 24 +-
.../Transforms/Scalar/LoopStrengthReduce.cpp | 184 ++++---
llvm/lib/Transforms/Utils/CodeExtractor.cpp | 8 +-
llvm/lib/Transforms/Utils/Debugify.cpp | 7 -
llvm/lib/Transforms/Utils/InlineFunction.cpp | 3 +-
llvm/lib/Transforms/Utils/Local.cpp | 103 +---
.../Transforms/Utils/LoopRotationUtils.cpp | 23 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 1 -
.../Utils/PromoteMemoryToRegister.cpp | 49 +-
llvm/lib/Transforms/Utils/SSAUpdater.cpp | 22 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 39 +-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 9 +-
llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 6 +-
.../AArch64/sve-intrinsics-ldst-ext.ll | 24 +
llvm/test/CodeGen/AMDGPU/agpr-remat.ll | 18 +-
.../AMDGPU/amdgpu-prepare-agpr-alloc.mir | 95 ++++
.../bad-agpr-vgpr-regalloc-priority.mir | 74 +++
.../branch-folding-implicit-def-subreg.ll | 46 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 4 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 4 +
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll | 4 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll | 4 +-
.../CodeGen/AMDGPU/no-fold-accvgpr-mov.ll | 10 +-
.../CodeGen/AMDGPU/no-fold-accvgpr-mov.mir | 28 +-
.../CodeGen/AMDGPU/no-fold-accvgpr-read.mir | 26 +-
...al-regcopy-and-spill-missed-at-regalloc.ll | 20 +-
.../AMDGPU/ptradd-sdag-optimizations.ll | 196 ++++----
.../CodeGen/AMDGPU/spill-vector-superclass.ll | 6 +-
.../CodeGen/LoongArch/calling-conv-common.ll | 48 +-
.../CodeGen/LoongArch/calling-conv-half.ll | 16 +-
.../LoongArch/can-not-realign-stack.ll | 44 +-
.../CodeGen/LoongArch/emergency-spill-slot.ll | 4 +-
llvm/test/CodeGen/LoongArch/frame.ll | 107 ++++-
.../CodeGen/LoongArch/intrinsic-memcpy.ll | 8 +-
llvm/test/CodeGen/LoongArch/lasx/fpowi.ll | 88 ++--
.../lasx/ir-instruction/extractelement.ll | 120 ++---
.../lasx/ir-instruction/insertelement.ll | 132 +++---
llvm/test/CodeGen/LoongArch/llvm.sincos.ll | 150 +++---
llvm/test/CodeGen/LoongArch/lsx/pr146455.ll | 287 +++++++++++
...realignment-with-variable-sized-objects.ll | 24 +-
.../CodeGen/LoongArch/stack-realignment.ll | 80 ++--
.../LoongArch/unaligned-memcpy-inline.ll | 14 +-
llvm/test/CodeGen/LoongArch/vararg.ll | 70 +--
llvm/test/CodeGen/NVPTX/i8x4-instructions.ll | 47 ++
llvm/test/CodeGen/NVPTX/trunc-tofp.ll | 81 ++++
.../CodeGen/RISCV/short-forward-branch-opt.ll | 32 +-
llvm/test/CodeGen/RISCV/zdinx-spill.ll | 26 -
.../AddressSanitizer/lifetime.ll | 447 +++++++++++++++---
llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s | 45 ++
llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s | 45 ++
.../MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s | 52 ++
llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s | 52 ++
.../MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s | 12 +
llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s | 12 +
.../gfx1250_asm_vop3_from_vop1-fake16.s | 36 ++
.../MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s | 36 ++
.../gfx1250_asm_vop3_from_vop1_dpp16-fake16.s | 44 ++
.../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s | 44 ++
.../gfx1250_asm_vop3_from_vop1_dpp8-fake16.s | 4 +
.../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s | 4 +
.../Disassembler/AMDGPU/gfx1250_dasm_vop1.txt | 45 ++
.../AMDGPU/gfx1250_dasm_vop1_dpp16.txt | 39 ++
.../AMDGPU/gfx1250_dasm_vop1_dpp8.txt | 9 +
.../AMDGPU/gfx1250_dasm_vop3_from_vop1.txt | 36 ++
.../gfx1250_dasm_vop3_from_vop1_dpp16.txt | 33 ++
.../gfx1250_dasm_vop3_from_vop1_dpp8.txt | 7 +-
.../DeadStoreElimination/zeroed-missing.ll | 2 +-
.../LoopInterchange/force-interchange.ll | 43 ++
.../LoopStrengthReduce/X86/lifetime-use.ll | 59 +++
.../AArch64/maxbandwidth-regpressure.ll | 38 ++
.../X86/split-node-reorder-node-with-ops.ll | 2 -
llvm/tools/llvm-dis/llvm-dis.cpp | 14 -
llvm/unittests/CodeGen/LexicalScopesTest.cpp | 2 +-
llvm/unittests/IR/DebugInfoTest.cpp | 2 +
.../Transforms/Utils/DebugifyTest.cpp | 9 +-
llvm/unittests/Transforms/Utils/LocalTest.cpp | 56 ---
.../gn/secondary/libcxx/include/BUILD.gn | 1 -
.../secondary/llvm/lib/Target/AMDGPU/BUILD.gn | 1 +
llvm/utils/gn/secondary/llvm/test/BUILD.gn | 1 +
.../secondary/llvm/tools/llvm-ir2vec/BUILD.gn | 9 +
mlir/docs/Tutorials/transform/Ch2.md | 30 +-
.../Conversion/VectorToXeGPU/VectorToXeGPU.h | 3 -
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 63 ++-
.../common/include/PluginInterface.h | 28 +-
offload/plugins-nextgen/cuda/src/rtl.cpp | 10 +-
.../unittests/OffloadAPI/common/Fixtures.hpp | 3 -
.../OffloadAPI/event/olWaitEvent.cpp | 3 -
275 files changed, 4637 insertions(+), 2846 deletions(-)
create mode 100644 clang/test/Analysis/div-zero-cxx20.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
create mode 100644 llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
create mode 100644 llvm/test/CodeGen/NVPTX/trunc-tofp.ll
create mode 100644 llvm/test/Transforms/LoopInterchange/force-interchange.ll
create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
create mode 100644 llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index 59079f057d021..f76c69f29fb30 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -11,8 +11,6 @@ on:
- .github/workflows/build-ci-container-windows.yml
- '.github/workflows/containers/github-action-ci-windows/**'
pull_request:
- branches:
- - main
paths:
- .github/workflows/build-ci-container-windows.yml
- '.github/workflows/containers/github-action-ci-windows/**'
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 3159aae32ca51..7f01264af8534 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -11,8 +11,6 @@ on:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
pull_request:
- branches:
- - main
paths:
- .github/workflows/build-ci-container.yml
- '.github/workflows/containers/github-action-ci/**'
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index fcd3887ec7a09..ea16029268dba 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -808,6 +808,7 @@ Bug Fixes in This Version
nested scopes. (#GH147495)
- Fixed a failed assertion with an operator call expression which comes from a
macro expansion when performing analysis for nullability attributes. (#GH138371)
+- Fixed a concept equivalent checking crash due to untransformed constraint expressions. (#GH146614)
Bug Fixes to Compiler Builtins
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1206,6 +1207,8 @@ Static Analyzer
---------------
- Fixed a crash when C++20 parenthesized initializer lists are used. This issue
was causing a crash in clang-tidy. (#GH136041)
+- The Clang Static Analyzer now handles parenthesized initialization.
+ (#GH148875)
New features
^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index ed51f1d5de447..a916af7e0c2df 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -673,6 +673,7 @@ TARGET_BUILTIN(__builtin_amdgcn_tanhf, "ff", "nc", "tanh-insts")
TARGET_BUILTIN(__builtin_amdgcn_tanhh, "hh", "nc", "tanh-insts")
TARGET_BUILTIN(__builtin_amdgcn_tanh_bf16, "yy", "nc", "bf16-trans-insts")
TARGET_BUILTIN(__builtin_amdgcn_rcp_bf16, "yy", "nc", "bf16-trans-insts")
+TARGET_BUILTIN(__builtin_amdgcn_sqrt_bf16, "yy", "nc", "bf16-trans-insts")
TARGET_BUILTIN(__builtin_amdgcn_rsq_bf16, "yy", "nc", "bf16-trans-insts")
TARGET_BUILTIN(__builtin_amdgcn_log_bf16, "yy", "nc", "bf16-trans-insts")
TARGET_BUILTIN(__builtin_amdgcn_exp2_bf16, "yy", "nc", "bf16-trans-insts")
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index a70335bef9dd4..d34a4146ddbd6 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -1491,8 +1491,6 @@ class Sema;
OverloadingResult
BestViableFunctionImpl(Sema &S, SourceLocation Loc,
OverloadCandidateSet::iterator &Best);
- void PerfectViableFunction(Sema &S, SourceLocation Loc,
- OverloadCandidateSet::iterator &Best);
};
bool isBetterOverloadCandidate(Sema &S, const OverloadCandidate &Cand1,
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index 6370586e218ef..fbb34340a5c67 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -499,9 +499,6 @@ class ExprEngine {
void VisitGuardedExpr(const Expr *Ex, const Expr *L, const Expr *R,
ExplodedNode *Pred, ExplodedNodeSet &Dst);
- void VisitInitListExpr(const InitListExpr *E, ExplodedNode *Pred,
- ExplodedNodeSet &Dst);
-
/// VisitAttributedStmt - Transfer function logic for AttributedStmt.
void VisitAttributedStmt(const AttributedStmt *A, ExplodedNode *Pred,
ExplodedNodeSet &Dst);
@@ -591,6 +588,10 @@ class ExprEngine {
ExplodedNode *Pred,
ExplodedNodeSet &Dst);
+ void ConstructInitList(const Expr *Source, ArrayRef<Expr *> Args,
+ bool IsTransparent, ExplodedNode *Pred,
+ ExplodedNodeSet &Dst);
+
/// evalEagerlyAssumeBifurcation - Given the nodes in 'Src', eagerly assume
/// concrete boolean values for 'Ex', storing the resulting nodes in 'Dst'.
void evalEagerlyAssumeBifurcation(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index c89eca9bef440..5b9f44518fcc2 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -162,6 +162,8 @@ static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
Desc->IsConst = IsConst || D->IsConst;
Desc->IsFieldMutable = IsMutable || D->IsMutable;
Desc->IsVolatile = IsVolatile || D->IsVolatile;
+ // True if this field is const AND the parent is mutable.
+ Desc->IsConstInMutable = Desc->IsConst && IsMutable;
if (auto Fn = D->CtorFn)
Fn(B, Ptr + FieldOffset, Desc->IsConst, Desc->IsFieldMutable,
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 4591eabb69bb4..0227e4c0c7e38 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -101,6 +101,10 @@ struct InlineDescriptor {
/// Flag indicating if the field is mutable (if in a record).
LLVM_PREFERRED_TYPE(bool)
unsigned IsFieldMutable : 1;
+ /// Flag indicating if this field is a const field nested in
+ /// a mutable parent field.
+ LLVM_PREFERRED_TYPE(bool)
+ unsigned IsConstInMutable : 1;
/// Flag indicating if the field is an element of a composite array.
LLVM_PREFERRED_TYPE(bool)
unsigned IsArrayElement : 1;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index f64501f4a31e8..74399d177b5a2 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -445,6 +445,7 @@ LLVM_DUMP_METHOD void InlineDescriptor::dump(llvm::raw_ostream &OS) const {
OS << "InUnion: " << InUnion << "\n";
OS << "IsFieldMutable: " << IsFieldMutable << "\n";
OS << "IsArrayElement: " << IsArrayElement << "\n";
+ OS << "IsConstInMutable: " << IsConstInMutable << '\n';
OS << "Desc: ";
if (Desc)
Desc->dump(OS);
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index e8b519478c026..df5e3be83d741 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -566,7 +566,10 @@ bool CheckDowncast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
bool CheckConst(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
assert(Ptr.isLive() && "Pointer is not live");
- if (!Ptr.isConst() || Ptr.isMutable())
+ if (!Ptr.isConst())
+ return true;
+
+ if (Ptr.isMutable() && !Ptr.isConstInMutable())
return true;
if (!Ptr.isBlockPointer())
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 7798b6f886a85..51622238e275c 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -14,11 +14,6 @@
#define LLVM_CLANG_AST_INTERP_BLOCK_H
#include "Descriptor.h"
-#include "clang/AST/ComparisonCategories.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/Expr.h"
-#include "llvm/ADT/PointerUnion.h"
#include "llvm/Support/raw_ostream.h"
namespace clang {
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index de0b97fd93c76..462b9a11e0a5c 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -7,7 +7,6 @@
//===----------------------------------------------------------------------===//
#include "../ExprConstShared.h"
#include "Boolean.h"
-#include "Compiler.h"
#include "EvalEmitter.h"
#include "Interp.h"
#include "InterpBuiltinBitCast.h"
@@ -2906,6 +2905,8 @@ static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
if (!copyField(F, /*Activate=*/true))
return false;
} else {
+ if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))
+ return false;
Pointer DestField = Dest.atField(F.Offset);
zeroAll(DestField);
}
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index a5a4bd25fe712..d62a4f6275b50 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -128,6 +128,11 @@ static bool shouldSkipInBacktrace(const Function *F) {
if (FD->getDeclName().getCXXOverloadedOperator() == OO_New ||
FD->getDeclName().getCXXOverloadedOperator() == OO_Array_New)
return true;
+
+ if (const auto *MD = dyn_cast<CXXMethodDecl>(FD);
+ MD && MD->getParent()->isAnonymousStructOrUnion())
+ return true;
+
return false;
}
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index e6a64e6658f06..da74013cf83a6 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -576,6 +576,11 @@ class Pointer {
return true;
return isRoot() ? getDeclDesc()->IsConst : getInlineDesc()->IsConst;
}
+ bool isConstInMutable() const {
+ if (!isBlockPointer())
+ return false;
+ return isRoot() ? false : getInlineDesc()->IsConstInMutable;
+ }
/// Checks if an object or a subfield is volatile.
bool isVolatile() const {
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index c8c3d6b20c496..19d8ba26d44d8 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2852,8 +2852,21 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
if (AI.getInReg())
Attrs.addAttribute(llvm::Attribute::InReg);
- if (AI.getIndirectByVal())
+ // Depending on the ABI, this may be either a byval or a dead_on_return
+ // argument.
+ if (AI.getIndirectByVal()) {
Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
+ } else {
+ // Add dead_on_return when the object's lifetime ends in the callee.
+ // This includes trivially-destructible objects, as well as objects
+ // whose destruction / clean-up is carried out within the callee (e.g.,
+ // Obj-C ARC-managed structs, MSVC callee-destroyed objects).
+ if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
+ ParamType->castAs<RecordType>()
+ ->getDecl()
+ ->isParamDestroyedInCallee())
+ Attrs.addAttribute(llvm::Attribute::DeadOnReturn);
+ }
auto *Decl = ParamType->getAsRecordDecl();
if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index a7d796ecccc61..ee736a2816218 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -416,6 +416,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case AMDGPU::BI__builtin_amdgcn_sqrt:
case AMDGPU::BI__builtin_amdgcn_sqrtf:
case AMDGPU::BI__builtin_amdgcn_sqrth:
+ case AMDGPU::BI__builtin_amdgcn_sqrt_bf16:
return emitBuiltinWithOneOverloadedType<1>(*this, E,
Intrinsic::amdgcn_sqrt);
case AMDGPU::BI__builtin_amdgcn_rsq:
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 3f9b808b2722e..481f575518b93 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -258,10 +258,10 @@ static void getAArch64MultilibFlags(const Driver &D,
processMultilibCustomFlags(Result, Args);
}
-static void getARMMultilibFlags(const Driver &D,
- const llvm::Triple &Triple,
- const llvm::opt::ArgList &Args,
- Multilib::flags_list &Result) {
+static void getARMMultilibFlags(const Driver &D, const llvm::Triple &Triple,
+ llvm::Reloc::Model RelocationModel,
+ const llvm::opt::ArgList &Args,
+ Multilib::flags_list &Result) {
std::vector<StringRef> Features;
llvm::ARM::FPUKind FPUKind = tools::arm::getARMTargetFeatures(
D, Triple, Args, Features, false /*ForAs*/, true /*ForMultilib*/);
@@ -304,6 +304,18 @@ static void getARMMultilibFlags(const Driver &D,
llvm_unreachable("Invalid float ABI");
}
+ if (RelocationModel == llvm::Reloc::ROPI ||
+ RelocationModel == llvm::Reloc::ROPI_RWPI)
+ Result.push_back("-fropi");
+ else
+ Result.push_back("-fno-ropi");
+
+ if (RelocationModel == llvm::Reloc::RWPI ||
+ RelocationModel == llvm::Reloc::ROPI_RWPI)
+ Result.push_back("-frwpi");
+ else
+ Result.push_back("-fno-rwpi");
+
const Arg *BranchProtectionArg =
Args.getLastArgNoClaim(options::OPT_mbranch_protection_EQ);
if (BranchProtectionArg) {
@@ -344,6 +356,18 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
const llvm::Triple Triple(ComputeEffectiveClangTriple(Args));
Result.push_back("--target=" + Triple.str());
+ // A difference of relocation model (absolutely addressed data, PIC, Arm
+ // ROPI/RWPI) is likely to change whether a particular multilib variant is
+ // compatible with a given link. Determine the relocation model of the
+ // current link, so as to add appropriate multilib flags.
+ llvm::Reloc::Model RelocationModel;
+ unsigned PICLevel;
+ bool IsPIE;
+ {
+ RegisterEffectiveTriple TripleRAII(*this, Triple);
+ std::tie(RelocationModel, PICLevel, IsPIE) = ParsePICArgs(*this, Args);
+ }
+
switch (Triple.getArch()) {
case llvm::Triple::aarch64:
case llvm::Triple::aarch64_32:
@@ -354,7 +378,7 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
case llvm::Triple::armeb:
case llvm::Triple::thumb:
case llvm::Triple::thumbeb:
- getARMMultilibFlags(D, Triple, Args, Result);
+ getARMMultilibFlags(D, Triple, RelocationModel, Args, Result);
break;
case llvm::Triple::riscv32:
case llvm::Triple::riscv64:
@@ -376,6 +400,12 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
else
Result.push_back("-fexceptions");
+ if (RelocationModel == llvm::Reloc::PIC_)
+ Result.push_back(IsPIE ? (PICLevel > 1 ? "-fPIE" : "-fpie")
+ : (PICLevel > 1 ? "-fPIC" : "-fpic"));
+ else
+ Result.push_back("-fno-pic");
+
// Sort and remove duplicates.
std::sort(Result.begin(), Result.end());
Result.erase(llvm::unique(Result), Result.end());
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 834417f8e15ac..5205ca0bca6fa 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -925,7 +925,12 @@ static const Expr *SubstituteConstraintExpressionWithoutSatisfaction(
ND && ND->isFunctionOrFunctionTemplate()) {
ScopeForParameters.emplace(S, /*CombineWithOuterScope=*/true);
const FunctionDecl *FD = ND->getAsFunction();
+ if (FunctionTemplateDecl *Template = FD->getDescribedFunctionTemplate();
+ Template && Template->getInstantiatedFromMemberTemplate())
+ FD = Template->getInstantiatedFromMemberTemplate()->getTemplatedDecl();
for (auto *PVD : FD->parameters()) {
+ if (ScopeForParameters->getInstantiationOfIfExists(PVD))
+ continue;
if (!PVD->isParameterPack()) {
ScopeForParameters->InstantiatedLocal(PVD, PVD);
continue;
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 1b54628c5e564..5dd5b495480d9 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -11354,55 +11354,18 @@ OverloadingResult OverloadCandidateSet::BestViableFunction(Sema &S,
DeferredCandidatesCount != 0 && !ResolutionByPerfectCandidateIsDisabled;
if (TwoPhaseResolution) {
-
- PerfectViableFunction(S, Loc, Best);
- if (Best != end())
- return ResultForBestCandidate(Best);
+ OverloadingResult Res = BestViableFunctionImpl(S, Loc, Best);
+ if (Best != end() && Best->isPerfectMatch(S.Context)) {
+ if (!(HasDeferredTemplateConstructors &&
+ isa_and_nonnull<CXXConversionDecl>(Best->Function)))
+ return Res;
+ }
}
InjectNonDeducedTemplateCandidates(S);
return BestViableFunctionImpl(S, Loc, Best);
}
-void OverloadCandidateSet::PerfectViableFunction(
- Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
-
- Best = end();
- for (auto It = Candidates.begin(); It != Candidates.end(); ++It) {
-
- if (!It->isPerfectMatch(S.getASTContext()))
- continue;
-
- // We found a suitable conversion function
- // but if there is a template constructor in the target class
- // we might prefer that instead.
- if (HasDeferredTemplateConstructors &&
- isa_and_nonnull<CXXConversionDecl>(It->Function)) {
- Best = end();
- break;
- }
-
- if (Best == end()) {
- Best = It;
- continue;
- }
- if (Best->Function && It->Function) {
- FunctionDecl *D =
- S.getMoreConstrainedFunction(Best->Function, It->Function);
- if (D == nullptr) {
- Best = end();
- break;
- }
- if (D == It->Function)
- Best = It;
- continue;
- }
- // ambiguous
- Best = end();
- break;
- }
-}
-
OverloadingResult OverloadCandidateSet::BestViableFunctionImpl(
Sema &S, SourceLocation Loc, OverloadCandidateSet::iterator &Best) {
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index c77ef26da568d..d87484470f8b5 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1941,7 +1941,6 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
case Stmt::ConceptSpecializationExprClass:
case Stmt::CXXRewrittenBinaryOperatorClass:
case Stmt::RequiresExprClass:
- case Expr::CXXParenListInitExprClass:
case Stmt::EmbedExprClass:
// Fall through.
@@ -2315,11 +2314,22 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
break;
}
- case Stmt::InitListExprClass:
+ case Stmt::InitListExprClass: {
+ const InitListExpr *E = cast<InitListExpr>(S);
Bldr.takeNodes(Pred);
- VisitInitListExpr(cast<InitListExpr>(S), Pred, Dst);
+ ConstructInitList(E, E->inits(), E->isTransparent(), Pred, Dst);
Bldr.addNodes(Dst);
break;
+ }
+
+ case Expr::CXXParenListInitExprClass: {
+ const CXXParenListInitExpr *E = cast<CXXParenListInitExpr>(S);
+ Bldr.takeNodes(Pred);
+ ConstructInitList(E, E->getInitExprs(), /*IsTransparent*/ false, Pred,
+ Dst);
+ Bldr.addNodes(Dst);
+ break;
+ }
case Stmt::MemberExprClass:
Bldr.takeNodes(Pred);
@@ -4114,3 +4124,33 @@ void *ProgramStateTrait<ReplayWithoutInlining>::GDMIndex() {
}
void ExprEngine::anchor() { }
+
+void ExprEngine::ConstructInitList(const Expr *E, ArrayRef<Expr *> Args,
+ bool IsTransparent, ExplodedNode *Pred,
+ ExplodedNodeSet &Dst) {
+ assert((isa<InitListExpr, CXXParenListInitExpr>(E)));
+
+ const LocationContext *LC = Pred->getLocationContext();
+
+ StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
+ ProgramStateRef S = Pred->getState();
+ QualType T = E->getType().getCanonicalType();
+
+ bool IsCompound = T->isArrayType() || T->isRecordType() ||
+ T->isAnyComplexType() || T->isVectorType();
+
+ if (Args.size() > 1 || (E->isPRValue() && IsCompound && !IsTransparent)) {
+ llvm::ImmutableList<SVal> ArgList = getBasicVals().getEmptySValList();
+ for (Expr *E : llvm::reverse(Args))
+ ArgList = getBasicVals().prependSVal(S->getSVal(E, LC), ArgList);
+
+ B.generateNode(E, Pred,
+ S->BindExpr(E, LC, svalBuilder.makeCompoundVal(T, ArgList)));
+ } else {
+ B.generateNode(E, Pred,
+ S->BindExpr(E, LC,
+ Args.size() == 0
+ ? getSValBuilder().makeZeroVal(T)
+ : S->getSVal(Args.front(), LC)));
+ }
+}
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index fa8e669b6bb2f..f1a25a750dd0d 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -771,54 +771,6 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
Bldr.generateNode(B, Pred, state->BindExpr(B, Pred->getLocationContext(), X));
}
-void ExprEngine::VisitInitListExpr(const InitListExpr *IE,
- ExplodedNode *Pred,
- ExplodedNodeSet &Dst) {
- StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
-
- ProgramStateRef state = Pred->getState();
- const LocationContext *LCtx = Pred->getLocationContext();
- QualType T = getContext().getCanonicalType(IE->getType());
- unsigned NumInitElements = IE->getNumInits();
-
- if (!IE->isGLValue() && !IE->isTransparent() &&
- (T->isArrayType() || T->isRecordType() || T->isVectorType() ||
- T->isAnyComplexType())) {
- llvm::ImmutableList<SVal> vals = getBasicVals().getEmptySValList();
-
- // Handle base case where the initializer has no elements.
- // e.g: static int* myArray[] = {};
- if (NumInitElements == 0) {
- SVal V = svalBuilder.makeCompoundVal(T, vals);
- B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
- return;
- }
-
- for (const Stmt *S : llvm::reverse(*IE)) {
- SVal V = state->getSVal(cast<Expr>(S), LCtx);
- vals = getBasicVals().prependSVal(V, vals);
- }
-
- B.generateNode(IE, Pred,
- state->BindExpr(IE, LCtx,
- svalBuilder.makeCompoundVal(T, vals)));
- return;
- }
-
- // Handle scalars: int{5} and int{} and GLvalues.
- // Note, if the InitListExpr is a GLvalue, it means that there is an address
- // representing it, so it must have a single init element.
- assert(NumInitElements <= 1);
-
- SVal V;
- if (NumInitElements == 0)
- V = getSValBuilder().makeZeroVal(T);
- else
- V = state->getSVal(IE->getInit(0), LCtx);
-
- B.generateNode(IE, Pred, state->BindExpr(IE, LCtx, V));
-}
-
void ExprEngine::VisitGuardedExpr(const Expr *Ex,
const Expr *L,
const Expr *R,
diff --git a/clang/test/AST/ByteCode/mutable.cpp b/clang/test/AST/ByteCode/mutable.cpp
index aebbea920578c..35c5a0389921e 100644
--- a/clang/test/AST/ByteCode/mutable.cpp
+++ b/clang/test/AST/ByteCode/mutable.cpp
@@ -1,11 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++11 -verify=expected,expected11,both,both11 %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++14 -verify=expected,expected14,both %s
-// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11 %s
-// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both %s
-
-
-
-
+// RUN: %clang_cc1 -std=c++11 -verify=expected,expected11,both,both11 %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++14 -verify=expected,expected14,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++11 -verify=ref,ref11,both,both11 %s
+// RUN: %clang_cc1 -std=c++14 -verify=ref,ref14,both %s
namespace Simple {
struct S {
@@ -26,3 +22,47 @@ namespace Simple {
static_assert(s2.a2 == 12, ""); // both11-error {{not an integral constant expression}} \
// both11-note {{initializer of 's2' is not a constant expression}}
}
+#if __cplusplus >= 201402L
+namespace ConstInMutable {
+ class B {
+ public:
+
+ const int f;
+ constexpr B() : f(12) {}
+ };
+ class A {
+ public:
+ mutable B b;
+ constexpr A() = default;
+ };
+ constexpr int constInMutable() {
+ A a;
+
+ int *m = (int*)&a.b.f;
+ *m = 12; // both-note {{modification of object of const-qualified type 'const int' is not allowed in a constant expression}}
+ return 1;
+ }
+ static_assert(constInMutable() == 1, ""); // both-error {{not an integral constant expression}} \
+ // both-note {{in call to}}
+}
+
+namespace MutableInConst {
+ class C {
+ public:
+ mutable int c;
+ constexpr C() : c(50) {}
+ };
+ class D {
+ public:
+ C c;
+ constexpr D() {}
+ };
+ constexpr int mutableInConst() {
+ const D d{};
+ int *m = (int*)&d.c.c;
+ *m = 12;
+ return 1;
+ }
+ static_assert(mutableInConst() == 1, "");
+}
+#endif
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 0fa44a259a4ff..7cfd0d677a7b3 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -847,6 +847,20 @@ namespace Activation2 {
}
static_assert(change_member_indirectly() == 4);
}
+
+namespace CopyCtorMutable {
+ struct E {
+ union { // expected-note {{read of mutable member 'b'}}
+ int a;
+ mutable int b; // both-note {{here}}
+ };
+ };
+ constexpr E e1 = {{1}};
+ constexpr E e2 = e1; // both-error {{constant}} \
+ // ref-note {{read of mutable member 'b'}} \
+ // both-note {{in call}}
+}
+
#endif
namespace AddressComparison {
diff --git a/clang/test/Analysis/div-zero-cxx20.cpp b/clang/test/Analysis/div-zero-cxx20.cpp
new file mode 100644
index 0000000000000..00ea96e796777
--- /dev/null
+++ b/clang/test/Analysis/div-zero-cxx20.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core.DivideZero -std=c++20 -verify %s
+
+namespace GH148875 {
+struct A {
+ int x;
+ A(int v) : x(v) {}
+};
+
+struct B {
+ int x;
+ B() : x(0) {}
+};
+
+struct C {
+ int x, y;
+ C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+ int x;
+};
+
+struct E {
+ D d;
+ E(int a) : d(a) {}
+};
+
+struct F {
+ int x;
+};
+
+int t1() {
+ A a{42};
+ return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+ B b{};
+ return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+ C c1{1, -1};
+ return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+ C c2{0, 0};
+ return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+ E e{32};
+ return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+ F f(32);
+ return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+} // namespace GH148875
diff --git a/clang/test/Analysis/div-zero.cpp b/clang/test/Analysis/div-zero.cpp
index 063450d8883b0..51ea25e828a18 100644
--- a/clang/test/Analysis/div-zero.cpp
+++ b/clang/test/Analysis/div-zero.cpp
@@ -11,3 +11,63 @@ int fooPR10616 (int qX ) {
return (a % (qX-1)); // expected-warning {{Division by zero}}
}
+
+namespace GH148875 {
+struct A {
+ int x;
+ A(int v) : x(v) {}
+};
+
+struct B {
+ int x;
+ B() : x(0) {}
+};
+
+struct C {
+ int x, y;
+ C(int a, int b) : x(a), y(b) {}
+};
+
+struct D {
+ int x;
+};
+
+struct E {
+ D d;
+ E(int a) : d{a} {}
+};
+
+struct F {
+ int x;
+};
+
+int t1() {
+ A a{42};
+ return 1 / (a.x - 42); // expected-warning {{Division by zero}}
+}
+
+int t2() {
+ B b{};
+ return 1 / b.x; // expected-warning {{Division by zero}}
+}
+
+int t3() {
+ C c1{1, -1};
+ return 1 / (c1.x + c1.y); // expected-warning {{Division by zero}}
+}
+
+int t4() {
+ C c2{0, 0};
+ return 1 / (c2.x + c2.y); // expected-warning {{Division by zero}}
+}
+
+int t5() {
+ E e{32};
+ return 1 / (e.d.x - 32); // expected-warning {{Division by zero}}
+}
+
+int t6() {
+ F f{32};
+ return 1 / (f.x - 32); // expected-warning {{Division by zero}}
+}
+}
diff --git a/clang/test/CodeGen/64bit-swiftcall.c b/clang/test/CodeGen/64bit-swiftcall.c
index 7f8aa02d97ce1..448bca7acbca3 100644
--- a/clang/test/CodeGen/64bit-swiftcall.c
+++ b/clang/test/CodeGen/64bit-swiftcall.c
@@ -239,7 +239,7 @@ TEST(struct_big_1)
// CHECK-LABEL: define {{.*}} void @return_struct_big_1(ptr dead_on_unwind noalias writable sret
// Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
/*****************************************************************************/
/********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/AArch64/byval-temp.c b/clang/test/CodeGen/AArch64/byval-temp.c
index 0ee0312b2362d..5033b6cf5ac03 100644
--- a/clang/test/CodeGen/AArch64/byval-temp.c
+++ b/clang/test/CodeGen/AArch64/byval-temp.c
@@ -30,10 +30,10 @@ void example(void) {
// Then, memcpy `l` to the temporary stack space.
// CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
// Finally, call using a pointer to the temporary stack space.
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
// Now, do the same for the second call, using the second temporary alloca.
// CHECK-O0-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O0-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O0-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
// CHECK-O0-NEXT: ret void
//
// At O3, we should have lifetime markers to help the optimizer re-use the temporary allocas.
@@ -58,7 +58,7 @@ void example(void) {
// Then, memcpy `l` to the temporary stack space.
// CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
// Finally, call using a pointer to the temporary stack space.
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
//
// The lifetime of the temporary used to pass a pointer to the struct ends here.
// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp]])
@@ -66,7 +66,7 @@ void example(void) {
// Now, do the same for the second call, using the second temporary alloca.
// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[byvaltemp1]])
// CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
-// CHECK-O3-NEXT: call void @pass_large(ptr noundef %[[byvaltemp1]])
+// CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp1]])
//
// Mark the end of the lifetime of `l`.
@@ -88,12 +88,12 @@ void example_BitInt(void) {
// CHECK-O0-NEXT: [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
// CHECK-O0-NEXT: [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
// CHECK-O0-NEXT: store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16
-// CHECK-O0-NEXT: call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O0-NEXT: call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
// CHECK-O0-NEXT: [[TMP1:%.*]] = load i256, ptr [[L]], align 16
// CHECK-O0-NEXT: [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
// CHECK-O0-NEXT: [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
// CHECK-O0-NEXT: store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16
-// CHECK-O0-NEXT: call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O0-NEXT: call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
// CHECK-O0-NEXT: ret void
//
// CHECK-O3-LABEL: define dso_local void @example_BitInt(
@@ -108,13 +108,13 @@ void example_BitInt(void) {
// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]])
// CHECK-O3-NEXT: [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
// CHECK-O3-NEXT: store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT: call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP]])
+// CHECK-O3-NEXT: call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]])
// CHECK-O3-NEXT: [[TMP1:%.*]] = load i256, ptr [[L]], align 16, !tbaa [[TBAA6]]
// CHECK-O3-NEXT: [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]])
// CHECK-O3-NEXT: [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
// CHECK-O3-NEXT: store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16, !tbaa [[TBAA6]]
-// CHECK-O3-NEXT: call void @pass_large_BitInt(ptr noundef [[INDIRECT_ARG_TEMP1]])
+// CHECK-O3-NEXT: call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]])
// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr [[L]])
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
index 546910068c78a..804e14a2ea34b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args-empty-union.c
@@ -19,7 +19,7 @@ void f0(S0 *p) {
use0(*p);
}
// CHECK-C: declare void @use0(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
-// CHECK-CXX: declare void @use0(ptr noundef)
+// CHECK-CXX: declare void @use0(ptr dead_on_return noundef)
#ifdef __cplusplus
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index fecd370d09be3..48988f7a1722b 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -92,7 +92,7 @@ void test_argpass_simple(PST *p) {
// CHECK-AAPCS-NEXT: ret void
// CHECK-AAPCS: declare void @argpass_simple_callee(<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_simple_callee(ptr noundef)
+// CHECK-DARWIN: declare void @argpass_simple_callee(ptr dead_on_return noundef)
// Boundary case of using the last available Z-reg, PST expanded.
// 0.0 -> d0-d3
@@ -107,7 +107,7 @@ void test_argpass_last_z(PST *p) {
argpass_last_z_callee(.0, .0, .0, .0, *p);
}
// CHECK-AAPCS: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_callee(double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef)
// Like the above, but using a tuple type to occupy some registers.
@@ -123,7 +123,7 @@ void test_argpass_last_z_tuple(PST *p, svfloat64x4_t x) {
argpass_last_z_tuple_callee(x, *p);
}
// CHECK-AAPCS: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_z_tuple_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, ptr dead_on_return noundef)
// Boundary case of using the last available P-reg, PST expanded.
@@ -139,7 +139,7 @@ void test_argpass_last_p(PST *p) {
argpass_last_p_callee(svpfalse(), svpfalse_c(), *p);
}
// CHECK-AAPCS: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), <vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr noundef)
+// CHECK-DARWIN: declare void @argpass_last_p_callee(<vscale x 16 x i1>, target("aarch64.svcount"), ptr dead_on_return noundef)
// Not enough Z-regs, push PST to memory and pass a pointer, Z-regs and
@@ -157,7 +157,7 @@ void test_argpass_no_z(PST *p, double dummy, svmfloat8_t u, int8x16_t v, mfloat8
void argpass_no_z_callee(svmfloat8_t, int8x16_t, mfloat8x16_t, double, double, int, PST, int, double, svbool_t);
argpass_no_z_callee(u, v, w, .0, .0, 1, *p, 2, 3.0, svptrue_b64());
}
-// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_callee(<vscale x 16 x i8>, <16 x i8> noundef, <16 x i8>, double noundef, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
// Like the above, using a tuple to occupy some registers.
@@ -173,7 +173,7 @@ void test_argpass_no_z_tuple_f64(PST *p, float dummy, svfloat64x4_t x) {
double, svbool_t);
argpass_no_z_tuple_f64_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
}
-// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_f64_callee(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
// Likewise, using a different tuple.
@@ -189,7 +189,7 @@ void test_argpass_no_z_tuple_mfp8(PST *p, float dummy, svmfloat8x4_t x) {
double, svbool_t);
argpass_no_z_tuple_mfp8_callee(x, .0, 1, *p, 2, 3.0, svptrue_b64());
}
-// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_z_tuple_mfp8_callee(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, double noundef, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
// Not enough Z-regs (consumed by a HFA), PST passed indirectly
@@ -204,8 +204,8 @@ void test_argpass_no_z_hfa(HFA *h, PST *p) {
void argpass_no_z_hfa_callee(double, HFA, int, PST, int, svbool_t);
argpass_no_z_hfa_callee(.0, *h, 1, *p, 2, svptrue_b64());
}
-// CHECK-AAPCS: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float] alignstack(8), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hfa_callee(double noundef, [4 x float], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
// Not enough Z-regs (consumed by a HVA), PST passed indirectly
// 0.0 -> d0
@@ -219,8 +219,8 @@ void test_argpass_no_z_hva(HVA *h, PST *p) {
void argpass_no_z_hva_callee(double, HVA, int, PST, int, svbool_t);
argpass_no_z_hva_callee(.0, *h, 1, *p, 2, svptrue_b64());
}
-// CHECK-AAPCS: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-AAPCS: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>] alignstack(16), i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @argpass_no_z_hva_callee(double noundef, [4 x <16 x i8>], i32 noundef, ptr dead_on_return noundef, i32 noundef, <vscale x 16 x i1>)
// Not enough P-regs, PST passed indirectly, Z-regs and P-regs still available.
// true -> p0-p2
@@ -233,7 +233,7 @@ void test_argpass_no_p(PST *p) {
void argpass_no_p_callee(svbool_t, svbool_t, svbool_t, int, PST, int, double, svbool_t);
argpass_no_p_callee(svptrue_b8(), svptrue_b16(), svptrue_b32(), 1, *p, 2, 3.0, svptrue_b64());
}
-// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
// Like above, using a tuple to occupy some registers.
@@ -250,7 +250,7 @@ void test_argpass_no_p_tuple(PST *p, svbool_t u, svboolx2_t v) {
svbool_t);
argpass_no_p_tuple_callee(v, u, 1, *p, 2, 3.0, svptrue_b64());
}
-// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
+// CHECK: declare void @argpass_no_p_tuple_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, i32 noundef, ptr dead_on_return noundef, i32 noundef, double noundef, <vscale x 16 x i1>)
// HFAs go back-to-back to memory, afterwards Z-regs not available, PST passed indirectly.
@@ -263,8 +263,8 @@ void test_after_hfa(HFA *h, PST *p) {
void after_hfa_callee(double, double, double, double, double, HFA, PST, HFA, svbool_t);
after_hfa_callee(.0, .0, .0, .0, .0, *h, *p, *h, svpfalse());
}
-// CHECK-AAPCS: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
-// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr noundef, [4 x float], <vscale x 16 x i1>)
+// CHECK-AAPCS: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float] alignstack(8), ptr dead_on_return noundef, [4 x float] alignstack(8), <vscale x 16 x i1>)
+// CHECK-DARWIN: declare void @after_hfa_callee(double noundef, double noundef, double noundef, double noundef, double noundef, [4 x float], ptr dead_on_return noundef, [4 x float], <vscale x 16 x i1>)
// Small PST, not enough registers, passed indirectly, unlike other small
// aggregates.
@@ -277,7 +277,7 @@ void test_small_pst(SmallPST *p, SmallAgg *s) {
void small_pst_callee(SmallAgg, double, double, double, double, double, double, double, double, double, SmallPST, double);
small_pst_callee(*s, .0, .0, .0, .0, .0, .0, .0, .0, 1.0, *p, 2.0);
}
-// CHECK-AAPCS: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr noundef, double noundef)
+// CHECK-AAPCS: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, ptr dead_on_return noundef, double noundef)
// CHECK-DARWIN: declare void @small_pst_callee([2 x i64], double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, double noundef, i128, double noundef)
@@ -326,12 +326,12 @@ void test_pass_variadic(PST *p, PST *q) {
pass_variadic_callee(*p, *q);
}
// CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr dead_on_return noundef nonnull %byval-temp)
// CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %p, i64 96, i1 false)
// CHECK-DARWIN: call void @llvm.lifetime.start.p0(i64 96, ptr nonnull %byval-temp1)
// CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp1, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
-// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr noundef nonnull %byval-temp, ptr noundef nonnull %byval-temp1)
+// CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr dead_on_return noundef nonnull %byval-temp, ptr dead_on_return noundef nonnull %byval-temp1)
// Test passing a small PST, still passed indirectly, despite being <= 128 bits
@@ -340,7 +340,7 @@ void test_small_pst_variadic(SmallPST *p) {
small_pst_variadic_callee(0, *p);
}
// CHECK-AAPCS: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) %byval-temp, ptr noundef nonnull align 16 dereferenceable(16) %p, i64 16, i1 false)
-// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr noundef nonnull %byval-temp)
+// CHECK-AAPCS: call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, ptr dead_on_return noundef nonnull %byval-temp)
// CHECK-DARWIN: %0 = load i128, ptr %p, align 16
// CHECK-DARWIN: tail call void (i32, ...) @small_pst_variadic_callee(i32 noundef 0, i128 %0)
@@ -467,7 +467,7 @@ void test_tuple_reg_count(svfloat32_t x, svfloat32x2_t y) {
svfloat32_t, svfloat32_t, svfloat32_t, svfloat32x2_t);
test_tuple_reg_count_callee(x, x, x, x, x, x, x, y);
}
-// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr noundef)
+// CHECK-AAPCS: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, ptr dead_on_return noundef)
// CHECK-DARWIN: declare void @test_tuple_reg_count_callee(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
// Regression test for incorrect passing of SVE vector tuples
@@ -476,5 +476,5 @@ void test_tuple_reg_count_bool(svboolx4_t x, svboolx4_t y) {
void test_tuple_reg_count_bool_callee(svboolx4_t, svboolx4_t);
test_tuple_reg_count_bool_callee(x, y);
}
-// CHECK-AAPCS: declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr noundef)
+// CHECK-AAPCS: declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, ptr dead_on_return noundef)
// CHECK-DARWIN: declare void @test_tuple_reg_count_bool_callee(<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>)
diff --git a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
index b1232921df363..f0c9ef28201a5 100644
--- a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
+++ b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp
@@ -82,7 +82,7 @@ struct Sppp {
int *x, *y, *z;
};
// CHECK-A64-LABEL: define dso_local void @_Z4Tppp4Sppp(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
// CHECK-A64-NEXT: [[ENTRY:.*:]]
// CHECK-A64-NEXT: [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
// CHECK-A64-NEXT: store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
@@ -490,7 +490,7 @@ struct Spa3 {
int* xs[3];
};
// CHECK-A64-LABEL: define dso_local void @_Z4Tpa34Spa3(
-// CHECK-A64-SAME: ptr noundef [[S:%.*]]) #[[ATTR0]] {
+// CHECK-A64-SAME: ptr dead_on_return noundef [[S:%.*]]) #[[ATTR0]] {
// CHECK-A64-NEXT: [[ENTRY:.*:]]
// CHECK-A64-NEXT: [[S_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
// CHECK-A64-NEXT: store ptr [[S]], ptr [[S_INDIRECT_ADDR]], align 8
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index c3d0541229fac..d244a8ba88572 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -58,7 +58,7 @@ typedef int8_t vec_int8 __attribute__((vector_size(N / 8)));
// CHECK128-NEXT: ret <16 x i8> [[CASTFIXEDSVE]]
// CHECK-LABEL: define{{.*}} void @f2(
-// CHECK-SAME: ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr noundef readonly captures(none) %0)
+// CHECK-SAME: ptr dead_on_unwind noalias writable writeonly sret(<[[#div(VBITS,8)]] x i8>) align 16 captures(none) initializes((0, [[#div(VBITS,8)]])) %agg.result, ptr dead_on_return noundef readonly captures(none) %0)
// CHECK-NEXT: entry:
// CHECK-NEXT: [[X:%.*]] = load <[[#div(VBITS,8)]] x i8>, ptr [[TMP0:%.*]], align 16, [[TBAA6:!tbaa !.*]]
// CHECK-NEXT: [[CASTSCALABLESVE:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v[[#div(VBITS,8)]]i8(<vscale x 16 x i8> poison, <[[#div(VBITS,8)]] x i8> [[X]], i64 0)
@@ -88,13 +88,13 @@ typedef svint8_t vec2 __attribute__((arm_sve_vector_bits(N)));
// CHECK-NEXT: [[X:%.*]] = tail call <[[#div(VBITS,8)]] x i8> @llvm.vector.extract.v[[#div(VBITS,8)]]i8.nxv16i8(<vscale x 16 x i8> [[X_COERCE:%.*]], i64 0)
// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
// CHECK-NEXT: store <[[#div(VBITS,8)]] x i8> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6]]
-// CHECK-NEXT: call void @f3(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECK-NEXT: call void @f3(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
// CHECK-NEXT: ret void
// CHECK128-LABEL: declare void @f3(<16 x i8> noundef)
// CHECK-LABEL: declare void @f3(
-// CHECK-SAME: ptr noundef)
+// CHECK-SAME: ptr dead_on_return noundef)
void g(vec2 x) { f3(x); } // OK
#endif
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
index e82069aab2486..d42ecb663050f 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
@@ -75,7 +75,7 @@ typedef svint16_t vec2 __attribute__((arm_sve_vector_bits(N)));
// CHECKWIDE-NEXT: [[X:%.*]] = tail call <[[#div(VBITS, 16)]] x i16> @llvm.vector.extract.v[[#div(VBITS, 16)]]i16.nxv8i16(<vscale x 8 x i16> [[X_COERCE:%.*]], i64 0)
// CHECKWIDE-NEXT: call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
// CHECKWIDE-NEXT: store <[[#div(VBITS, 16)]] x i16> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6:!tbaa !.*]]
-// CHECKWIDE-NEXT: call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
+// CHECKWIDE-NEXT: call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
// CHECKWIDE-NEXT: call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
// CHECKWIDE-NEXT: ret void
void g(vec2 x) { f(x); } // OK
diff --git a/clang/test/CodeGen/LoongArch/bitint.c b/clang/test/CodeGen/LoongArch/bitint.c
index f346f569d0eb0..950f5308e32cc 100644
--- a/clang/test/CodeGen/LoongArch/bitint.c
+++ b/clang/test/CodeGen/LoongArch/bitint.c
@@ -26,12 +26,12 @@ void pass_BitInt129(_BitInt(129));
// LA32-NEXT: [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
// LA32-NEXT: [[STOREDV:%.*]] = sext i65 [[LOADEDV1]] to i128
// LA32-NEXT: store i128 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA32-NEXT: call void @pass_BitInt65(ptr noundef [[BYVAL_TEMP]])
+// LA32-NEXT: call void @pass_BitInt65(ptr dead_on_return noundef [[BYVAL_TEMP]])
// LA32-NEXT: [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
// LA32-NEXT: [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
// LA32-NEXT: [[STOREDV4:%.*]] = sext i129 [[LOADEDV2]] to i256
// LA32-NEXT: store i256 [[STOREDV4]], ptr [[BYVAL_TEMP3]], align 16
-// LA32-NEXT: call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP3]])
+// LA32-NEXT: call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP3]])
// LA32-NEXT: ret void
//
// LA64-LABEL: define dso_local void @example_BitInt(
@@ -54,7 +54,7 @@ void pass_BitInt129(_BitInt(129));
// LA64-NEXT: [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
// LA64-NEXT: [[STOREDV:%.*]] = sext i129 [[LOADEDV2]] to i256
// LA64-NEXT: store i256 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
-// LA64-NEXT: call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP]])
+// LA64-NEXT: call void @pass_BitInt129(ptr dead_on_return noundef [[BYVAL_TEMP]])
// LA64-NEXT: ret void
//
void example_BitInt(void) {
diff --git a/clang/test/CodeGen/PowerPC/ppc64-vector.c b/clang/test/CodeGen/PowerPC/ppc64-vector.c
index 5d3dd86a009d5..2e99781f84910 100644
--- a/clang/test/CodeGen/PowerPC/ppc64-vector.c
+++ b/clang/test/CodeGen/PowerPC/ppc64-vector.c
@@ -39,7 +39,7 @@ v8i16 test_v8i16(v8i16 x)
return x;
}
-// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr noundef %0)
+// CHECK: define{{.*}} void @test_v16i16(ptr dead_on_unwind noalias writable sret(<16 x i16>) align 32 %agg.result, ptr dead_on_return noundef %0)
v16i16 test_v16i16(v16i16 x)
{
return x;
diff --git a/clang/test/CodeGen/RISCV/riscv-abi.cpp b/clang/test/CodeGen/RISCV/riscv-abi.cpp
index fe1a2b6d8595c..d2e080829e72f 100644
--- a/clang/test/CodeGen/RISCV/riscv-abi.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-abi.cpp
@@ -75,7 +75,7 @@ struct child3_int64_s : parent3_float_s {
};
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z30float_int64_struct_inheritance14child3_int64_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD3_INT64_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
// LP64-LABEL: define dso_local [2 x i64] @_Z30float_int64_struct_inheritance14child3_int64_s
@@ -99,7 +99,7 @@ struct child4_double_s : parent4_double_s {
};
// ILP32-ILP32F-LABEL: define dso_local void @_Z32double_double_struct_inheritance15child4_double_s
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD4_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local { double, double } @_Z32double_double_struct_inheritance15child4_double_s
@@ -130,11 +130,11 @@ struct child5_virtual_s : virtual parent5_virtual_s {
};
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 4 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
// LP64-LP64F-LP64D-LABEL: define dso_local void @_Z38int32_float_virtual_struct_inheritance16child5_virtual_s
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_CHILD5_VIRTUAL_S:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
struct child5_virtual_s int32_float_virtual_struct_inheritance(struct child5_virtual_s a) {
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
index 82e43fff0c3aa..bc89cb532bdcc 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
@@ -138,7 +138,7 @@ struct st_i32x4x9 {
typedef int __attribute__((vector_size(256))) int32x64_t;
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_too_large(ptr dead_on_return noundef %0)
void __attribute__((riscv_vls_cc)) test_too_large(int32x64_t arg) {}
// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_too_large_256(<vscale x 16 x i32> noundef %arg.coerce)
void __attribute__((riscv_vls_cc(256))) test_too_large_256(int32x64_t arg) {}
@@ -173,9 +173,9 @@ void __attribute__((riscv_vls_cc)) test_st_i32x8x2(struct st_i32x8x2 arg) {}
// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x8x2_256(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
void __attribute__((riscv_vls_cc(256))) test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr dead_on_return noundef %arg)
void __attribute__((riscv_vls_cc)) test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr dead_on_return noundef %arg)
void __attribute__((riscv_vls_cc(256))) test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -188,7 +188,7 @@ void __attribute__((riscv_vls_cc)) test_st_i32x4x8(struct st_i32x4x8 arg) {}
// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
void __attribute__((riscv_vls_cc(256))) test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr dead_on_return noundef %arg)
void __attribute__((riscv_vls_cc)) test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr dead_on_return noundef %arg)
void __attribute__((riscv_vls_cc(256))) test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
index 5f6539796c20d..128610e578c26 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
@@ -118,7 +118,7 @@ struct st_i32x4x9 {
typedef int __attribute__((vector_size(256))) int32x64_t;
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr noundef %0)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z14test_too_largeDv64_i(ptr dead_on_return noundef %0)
[[riscv::vls_cc]] void test_too_large(int32x64_t arg) {}
// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z18test_too_large_256Dv64_i(<vscale x 16 x i32> noundef %arg.coerce)
[[riscv::vls_cc(256)]] void test_too_large_256(int32x64_t arg) {}
@@ -153,9 +153,9 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x8x2_25610st_i32x8x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
[[riscv::vls_cc(256)]] void test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr dead_on_return noundef %arg)
[[riscv::vls_cc]] void test_st_i32x64x2(struct st_i32x64x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr dead_on_return noundef %arg)
[[riscv::vls_cc(256)]] void test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x310st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
@@ -168,7 +168,7 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x8_25610st_i32x4x8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
[[riscv::vls_cc(256)]] void test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr dead_on_return noundef %arg)
[[riscv::vls_cc]] void test_st_i32x4x9(struct st_i32x4x9 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr noundef %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x9_25610st_i32x4x9(ptr dead_on_return noundef %arg)
[[riscv::vls_cc(256)]] void test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-abi.c b/clang/test/CodeGen/RISCV/riscv32-abi.c
index b53f9a9169146..a9e56d40817ae 100644
--- a/clang/test/CodeGen/RISCV/riscv32-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv32-abi.c
@@ -246,7 +246,7 @@ struct large {
};
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_large
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_agg_large(struct large x) {
@@ -266,7 +266,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
typedef unsigned char v16i8 __attribute__((vector_size(16)));
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_vec_large_v16i8
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_vec_large_v16i8(v16i8 x) {
@@ -285,7 +285,7 @@ v16i8 f_vec_large_v16i8_ret(void) {
// if they were passed in registers.
// ILP32-ILP32F-ILP32D-LABEL: define dso_local i32 @f_scalar_stack_1
-// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 [[A_COERCE:%.*]], [2 x i32] [[B_COERCE:%.*]], i64 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -343,7 +343,7 @@ struct large f_scalar_stack_6(float a, int64_t b, double c, long double d,
// they would be if passed via registers.
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_agg_stack
-// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (double noundef [[A:%.*]], i64 noundef [[B:%.*]], double noundef [[C:%.*]], i64 noundef [[D:%.*]], i32 [[E_COERCE:%.*]], [2 x i32] [[F_COERCE:%.*]], i64 [[G_COERCE:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_agg_stack(double a, int64_t b, double c, int64_t d, struct tiny e,
@@ -366,7 +366,7 @@ struct double_int8_s { double d; int64_t i; };
struct int_double_s { int a; double b; };
// ILP32-ILP32F-LABEL: define dso_local void @f_int_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_int_double_s_arg
@@ -482,7 +482,7 @@ struct zbf_double_zbf_s f_ret_zbf_double_zbf_s(void) {
struct double_float_s { double f; float g; };
// ILP32-ILP32F-LABEL: define dso_local void @f_double_double_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_double_s_arg
@@ -504,7 +504,7 @@ struct double_double_s f_ret_double_double_s(void) {
}
// ILP32-ILP32F-LABEL: define dso_local void @f_double_float_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_float_s_arg
@@ -526,7 +526,7 @@ struct double_float_s f_ret_double_float_s(void) {
}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_double_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_double_double_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -543,7 +543,7 @@ struct double_int64bf_s { double d; int64_t i : 32; };
struct double_int8_zbf_s { double d; int8_t i; int : 0; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_double_int8_s_arg(struct double_int8_s a) {}
@@ -557,7 +557,7 @@ struct double_int8_s f_ret_double_int8_s(void) {
}
// ILP32-ILP32F-LABEL: define dso_local void @f_double_uint8_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_uint8_s_arg
@@ -579,7 +579,7 @@ struct double_uint8_s f_ret_double_uint8_s(void) {
}
// ILP32-ILP32F-LABEL: define dso_local void @f_double_int32_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_int32_s_arg
@@ -601,7 +601,7 @@ struct double_int32_s f_ret_double_int32_s(void) {
}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_double_int64_s_arg(struct double_int64_s a) {}
@@ -615,7 +615,7 @@ struct double_int64_s f_ret_double_int64_s(void) {
}
// ILP32-ILP32F-LABEL: define dso_local void @f_double_int64bf_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_int64bf_s_arg
@@ -640,7 +640,7 @@ struct double_int64bf_s f_ret_double_int64bf_s(void) {
// floating point calling convention.
// ILP32-ILP32F-LABEL: define dso_local void @f_double_int8_zbf_s
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_int8_zbf_s
@@ -662,14 +662,14 @@ struct double_int8_zbf_s f_ret_double_int8_zbf_s(void) {
}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_int8_s_arg_insufficient_gprs
-// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], i32 noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_double_int8_s_arg_insufficient_gprs(int a, int b, int c, int d, int e,
int f, int g, int h, struct double_int8_s i) {}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_struct_double_int8_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr noundef [[I:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], double noundef [[H:%.*]], ptr dead_on_return noundef [[I:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double d,
@@ -679,7 +679,7 @@ void f_struct_double_int8_insufficient_fprs(float a, double b, double c, double
// floating-point value should be passed as if it were an fp+fp struct.
// ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublecomplex
@@ -703,7 +703,7 @@ double __complex__ f_ret_doublecomplex(void) {
struct doublecomplex_s { double __complex__ c; };
// ILP32-ILP32F-LABEL: define dso_local void @f_doublecomplex_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublecomplex_s_arg
@@ -754,7 +754,7 @@ struct doublearr1_s f_ret_doublearr1_s(void) {
struct doublearr2_s { double a[2]; };
// ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublearr2_s_arg
@@ -778,7 +778,7 @@ struct doublearr2_s f_ret_doublearr2_s(void) {
struct doublearr2_tricky1_s { struct { double f[1]; } g[2]; };
// ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublearr2_tricky1_s_arg
@@ -802,7 +802,7 @@ struct doublearr2_tricky1_s f_ret_doublearr2_tricky1_s(void) {
struct doublearr2_tricky2_s { struct {}; struct { double f[1]; } g[2]; };
// ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublearr2_tricky2_s_arg
@@ -826,7 +826,7 @@ struct doublearr2_tricky2_s f_ret_doublearr2_tricky2_s(void) {
struct doublearr2_tricky3_s { union {}; struct { double f[1]; } g[2]; };
// ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublearr2_tricky3_s_arg
@@ -850,7 +850,7 @@ struct doublearr2_tricky3_s f_ret_doublearr2_tricky3_s(void) {
struct doublearr2_tricky4_s { union {}; struct { struct {}; double f[1]; } g[2]; };
// ILP32-ILP32F-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_doublearr2_tricky4_s_arg
@@ -877,7 +877,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
struct int_double_int_s { int a; double b; int c; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_double_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_int_double_int_s_arg(struct int_double_int_s a) {}
@@ -893,7 +893,7 @@ struct int_double_int_s f_ret_int_double_int_s(void) {
struct int64_double_s { int64_t a; double b; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_int64_double_s_arg(struct int64_double_s a) {}
@@ -909,7 +909,7 @@ struct int64_double_s f_ret_int64_double_s(void) {
struct char_char_double_s { char a; char b; double c; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_char_char_double_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_char_char_double_s_arg(struct char_char_double_s a) {}
@@ -948,7 +948,7 @@ union double_u f_ret_double_u(void) {
// double+double structs by the ABI.
// ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_INT32_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local { double, i32 } @f_ret_double_int32_s_double_int32_s_just_sufficient_gprs
@@ -961,7 +961,7 @@ struct double_int32_s f_ret_double_int32_s_double_int32_s_just_sufficient_gprs(
}
// ILP32-ILP32F-LABEL: define dso_local void @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_DOUBLE_DOUBLE_S:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local { double, double } @f_ret_double_double_s_double_int32_s_just_sufficient_gprs
@@ -974,7 +974,7 @@ struct double_double_s f_ret_double_double_s_double_int32_s_just_sufficient_gprs
}
// ILP32-ILP32F-LABEL: define dso_local void @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
-// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], i32 noundef [[A:%.*]], i32 noundef [[B:%.*]], i32 noundef [[C:%.*]], i32 noundef [[D:%.*]], i32 noundef [[E:%.*]], i32 noundef [[F:%.*]], i32 noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local { double, double } @f_ret_doublecomplex_double_int32_s_just_sufficient_gprs
@@ -1189,7 +1189,7 @@ struct float_int32_s f_ret_float_int32_s(void) {
}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_float_int64_s_arg(struct float_int64_s a) {}
@@ -1465,7 +1465,7 @@ struct floatarr2_tricky4_s f_ret_floatarr2_tricky4_s(void) {
struct int_float_int_s { int a; float b; int c; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_int_float_int_s_arg(struct int_float_int_s a) {}
@@ -1481,7 +1481,7 @@ struct int_float_int_s f_ret_int_float_int_s(void) {
struct int64_float_s { int64_t a; float b; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_int64_float_s_arg(struct int64_float_s a) {}
@@ -1619,7 +1619,7 @@ struct zbf_float16_zbf_s f_ret_zbf_float16_zbf_s(void) {
struct double_float16_s { double f; _Float16 g; };
// ILP32-ILP32F-LABEL: define dso_local void @f_double_float16_s_arg
-// ILP32-ILP32F-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F: entry:
//
// ILP32D-LABEL: define dso_local void @f_double_float16_s_arg
@@ -1641,7 +1641,7 @@ struct double_float16_s f_ret_double_float16_s(void) {
}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_double_float16_s_arg_insufficient_fprs
-// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr noundef [[H:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (float noundef [[A:%.*]], double noundef [[B:%.*]], double noundef [[C:%.*]], double noundef [[D:%.*]], double noundef [[E:%.*]], double noundef [[F:%.*]], double noundef [[G:%.*]], ptr dead_on_return noundef [[H:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_double_float16_s_arg_insufficient_fprs(float a, double b, double c, double d,
@@ -1725,7 +1725,7 @@ struct float16_int32_s f_ret_float16_int32_s(void) {
}
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_float16_int64_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_float16_int64_s_arg(struct float16_int64_s a) {}
@@ -2001,7 +2001,7 @@ struct float16arr2_tricky4_s f_ret_float16arr2_tricky4_s(void) {
struct int_float16_int_s { int a; _Float16 b; int c; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int_float16_int_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_int_float16_int_s_arg(struct int_float16_int_s a) {}
@@ -2017,7 +2017,7 @@ struct int_float16_int_s f_ret_int_float16_int_s(void) {
struct int64_float16_s { int64_t a; _Float16 b; };
// ILP32-ILP32F-ILP32D-LABEL: define dso_local void @f_int64_float16_s_arg
-// ILP32-ILP32F-ILP32D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// ILP32-ILP32F-ILP32D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// ILP32-ILP32F-ILP32D: entry:
//
void f_int64_float16_s_arg(struct int64_float16_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv32-vararg.c b/clang/test/CodeGen/RISCV/riscv32-vararg.c
index 2b332410f8637..ed301f9269bb8 100644
--- a/clang/test/CodeGen/RISCV/riscv32-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv32-vararg.c
@@ -64,7 +64,7 @@ int f_va_callee(int, ...);
// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
// CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[COERCE_DIVE]], align 8
// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[BYVAL_TEMP]], ptr align 4 [[DOTCOMPOUNDLITERAL6]], i32 16, i1 false)
-// CHECK-NEXT: [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT: [[CALL:%.*]] = call i32 (i32, ...) @f_va_callee(i32 noundef 1, i32 noundef 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i32 [[TMP0]], [2 x i32] [[TMP1]], i64 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
// CHECK-NEXT: ret void
//
void f_va_caller(void) {
diff --git a/clang/test/CodeGen/RISCV/riscv64-abi.c b/clang/test/CodeGen/RISCV/riscv64-abi.c
index 021565238904e..dc01750e56970 100644
--- a/clang/test/CodeGen/RISCV/riscv64-abi.c
+++ b/clang/test/CodeGen/RISCV/riscv64-abi.c
@@ -242,7 +242,7 @@ struct large {
};
// LP64-LP64F-LP64D-LABEL: define dso_local void @f_agg_large
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[X:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[X:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
void f_agg_large(struct large x) {
@@ -262,7 +262,7 @@ struct large f_agg_large_ret(int32_t i, int8_t j) {
typedef unsigned char v32i8 __attribute__((vector_size(32)));
// LP64-LP64F-LP64D-LABEL: define dso_local void @f_vec_large_v32i8
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[TMP0:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[TMP0:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
void f_vec_large_v32i8(v32i8 x) {
@@ -281,7 +281,7 @@ v32i8 f_vec_large_v32i8_ret(void) {
// if they were passed in registers.
// LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_1
-// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i64 [[A_COERCE:%.*]], [2 x i64] [[B_COERCE:%.*]], i128 [[C_COERCE:%.*]], ptr dead_on_return noundef [[D:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]], i8 noundef signext [[H:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
@@ -290,7 +290,7 @@ int f_scalar_stack_1(struct tiny a, struct small b, struct small_aligned c,
}
// LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_2
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], i64 noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
@@ -299,7 +299,7 @@ int f_scalar_stack_2(int32_t a, __int128_t b, int64_t c, long double d, v32i8 e,
}
// LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_3
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], double noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
@@ -312,7 +312,7 @@ int f_scalar_stack_3(int32_t a, __int128_t b, double c, long double d, v32i8 e,
// to pass a pointer.
// LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_4
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
@@ -321,7 +321,7 @@ struct large f_scalar_stack_4(uint32_t a, __int128_t b, long double c, v32i8 d,
}
// LP64-LP64F-LP64D-LABEL: define dso_local void @f_scalar_stack_5
-// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_unwind noalias writable sret([[STRUCT_LARGE:%.*]]) align 8 [[AGG_RESULT:%.*]], double noundef [[A:%.*]], i128 noundef [[B:%.*]], fp128 noundef [[C:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[E:%.*]], i8 noundef signext [[F:%.*]], i8 noundef zeroext [[G:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
@@ -330,7 +330,7 @@ struct large f_scalar_stack_5(double a, __int128_t b, long double c, v32i8 d,
}
// LP64-LP64F-LP64D-LABEL: define dso_local signext i32 @f_scalar_stack_6
-// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (i32 noundef signext [[A:%.*]], i128 noundef [[B:%.*]], float noundef [[C:%.*]], fp128 noundef [[D:%.*]], ptr dead_on_return noundef [[TMP0:%.*]], i8 noundef zeroext [[F:%.*]], i8 noundef signext [[G:%.*]], i8 noundef zeroext [[H:%.*]], half noundef [[I:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
int f_scalar_stack_6(int32_t a, __int128_t b, float c, long double d, v32i8 e,
@@ -1440,7 +1440,7 @@ struct doublearr2_tricky4_s f_ret_doublearr2_tricky4_s(void) {
struct int_double_int_s { int a; double b; int c; };
// LP64-LP64F-LP64D-LABEL: define dso_local void @f_int_double_int_s_arg
-// LP64-LP64F-LP64D-SAME: (ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// LP64-LP64F-LP64D-SAME: (ptr dead_on_return noundef [[A:%.*]]) #[[ATTR0]] {
// LP64-LP64F-LP64D: entry:
//
void f_int_double_int_s_arg(struct int_double_int_s a) {}
diff --git a/clang/test/CodeGen/RISCV/riscv64-vararg.c b/clang/test/CodeGen/RISCV/riscv64-vararg.c
index a278f74ca4a86..17802553c795a 100644
--- a/clang/test/CodeGen/RISCV/riscv64-vararg.c
+++ b/clang/test/CodeGen/RISCV/riscv64-vararg.c
@@ -74,7 +74,7 @@ int f_va_callee(int, ...);
// CHECK-NEXT: [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL4]], i32 0, i32 0
// CHECK-NEXT: [[TMP2:%.*]] = load i128, ptr [[COERCE_DIVE]], align 16
// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[BYVAL_TEMP]], ptr align 8 [[DOTCOMPOUNDLITERAL6]], i64 32, i1 false)
-// CHECK-NEXT: [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT: [[CALL:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i64 noundef 3, double noundef 4.000000e+00, double noundef 5.000000e+00, i64 [[TMP0]], [2 x i64] [[TMP1]], i128 [[TMP2]], ptr dead_on_return noundef [[BYVAL_TEMP]])
// CHECK-NEXT: [[CALL11:%.*]] = call signext i32 (i32, ...) @f_va_callee(i32 noundef signext 1, i32 noundef signext 2, i32 noundef signext 3, i32 noundef signext 4, fp128 noundef 0xL00000000000000004001400000000000, i32 noundef signext 6, i32 noundef signext 7, i32 noundef signext 8, i32 noundef signext 9)
// CHECK-NEXT: [[A13:%.*]] = getelementptr inbounds nuw [[STRUCT_SMALL_ALIGNED]], ptr [[DOTCOMPOUNDLITERAL12]], i32 0, i32 0
// CHECK-NEXT: store i128 5, ptr [[A13]], align 16
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
index e5704709a3a33..fab6050a0d876 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi-vector.c
@@ -58,91 +58,91 @@ unsigned int align = __alignof__ (v16i8);
// CHECK-VECTOR: @align ={{.*}} global i32 8
v1i8 pass_v1i8(v1i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i8(ptr dead_on_unwind noalias writable sret(<1 x i8>) align 1 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x i8> @pass_v1i8(<1 x i8> %{{.*}})
v2i8 pass_v2i8(v2i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i8(ptr dead_on_unwind noalias writable sret(<2 x i8>) align 2 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <2 x i8> @pass_v2i8(<2 x i8> %{{.*}})
v4i8 pass_v4i8(v4i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i8(ptr dead_on_unwind noalias writable sret(<4 x i8>) align 4 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <4 x i8> @pass_v4i8(<4 x i8> %{{.*}})
v8i8 pass_v8i8(v8i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i8(ptr dead_on_unwind noalias writable sret(<8 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <8 x i8> @pass_v8i8(<8 x i8> %{{.*}})
v16i8 pass_v16i8(v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v16i8(ptr dead_on_unwind noalias writable sret(<16 x i8>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <16 x i8> @pass_v16i8(<16 x i8> %{{.*}})
v32i8 pass_v32i8(v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr %0)
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 32 %{{.*}}, ptr dead_on_return %0)
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_v32i8(ptr dead_on_unwind noalias writable sret(<32 x i8>) align 8 %{{.*}}, ptr dead_on_return %0)
v1i16 pass_v1i16(v1i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i16(ptr dead_on_unwind noalias writable sret(<1 x i16>) align 2 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x i16> @pass_v1i16(<1 x i16> %{{.*}})
v2i16 pass_v2i16(v2i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i16(ptr dead_on_unwind noalias writable sret(<2 x i16>) align 4 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <2 x i16> @pass_v2i16(<2 x i16> %{{.*}})
v4i16 pass_v4i16(v4i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i16(ptr dead_on_unwind noalias writable sret(<4 x i16>) align 8 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <4 x i16> @pass_v4i16(<4 x i16> %{{.*}})
v8i16 pass_v8i16(v8i16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v8i16(ptr dead_on_unwind noalias writable sret(<8 x i16>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <8 x i16> @pass_v8i16(<8 x i16> %{{.*}})
v1i32 pass_v1i32(v1i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i32(ptr dead_on_unwind noalias writable sret(<1 x i32>) align 4 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x i32> @pass_v1i32(<1 x i32> %{{.*}})
v2i32 pass_v2i32(v2i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i32(ptr dead_on_unwind noalias writable sret(<2 x i32>) align 8 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <2 x i32> @pass_v2i32(<2 x i32> %{{.*}})
v4i32 pass_v4i32(v4i32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4i32(ptr dead_on_unwind noalias writable sret(<4 x i32>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <4 x i32> @pass_v4i32(<4 x i32> %{{.*}})
v1i64 pass_v1i64(v1i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i64(ptr dead_on_unwind noalias writable sret(<1 x i64>) align 8 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x i64> @pass_v1i64(<1 x i64> %{{.*}})
v2i64 pass_v2i64(v2i64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2i64(ptr dead_on_unwind noalias writable sret(<2 x i64>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <2 x i64> @pass_v2i64(<2 x i64> %{{.*}})
v1i128 pass_v1i128(v1i128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1i128(ptr dead_on_unwind noalias writable sret(<1 x i128>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x i128> @pass_v1i128(<1 x i128> %{{.*}})
v1f32 pass_v1f32(v1f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f32(ptr dead_on_unwind noalias writable sret(<1 x float>) align 4 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x float> @pass_v1f32(<1 x float> %{{.*}})
v2f32 pass_v2f32(v2f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f32(ptr dead_on_unwind noalias writable sret(<2 x float>) align 8 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <2 x float> @pass_v2f32(<2 x float> %{{.*}})
v4f32 pass_v4f32(v4f32 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v4f32(ptr dead_on_unwind noalias writable sret(<4 x float>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <4 x float> @pass_v4f32(<4 x float> %{{.*}})
v1f64 pass_v1f64(v1f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f64(ptr dead_on_unwind noalias writable sret(<1 x double>) align 8 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x double> @pass_v1f64(<1 x double> %{{.*}})
v2f64 pass_v2f64(v2f64 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v2f64(ptr dead_on_unwind noalias writable sret(<2 x double>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <2 x double> @pass_v2f64(<2 x double> %{{.*}})
v1f128 pass_v1f128(v1f128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_v1f128(ptr dead_on_unwind noalias writable sret(<1 x fp128>) align 16 %{{.*}}, ptr dead_on_return %0)
// CHECK-VECTOR-LABEL: define{{.*}} <1 x fp128> @pass_v1f128(<1 x fp128> %{{.*}})
@@ -170,13 +170,13 @@ struct agg_v8i8 pass_agg_v8i8(struct agg_v8i8 arg) { return arg; }
struct agg_v16i8 { v16i8 a; };
struct agg_v16i8 pass_agg_v16i8(struct agg_v16i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v16i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v16i8) align 8 %{{.*}}, <16 x i8> %{{.*}})
struct agg_v32i8 { v32i8 a; };
struct agg_v32i8 pass_agg_v32i8(struct agg_v32i8 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr %{{.*}})
-// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 32 %{{.*}}, ptr dead_on_return %{{.*}})
+// CHECK-VECTOR-LABEL: define{{.*}} void @pass_agg_v32i8(ptr dead_on_unwind noalias writable sret(%struct.agg_v32i8) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
// Verify that the following are *not* vector-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-abi.c b/clang/test/CodeGen/SystemZ/systemz-abi.c
index f26084ab44eae..83137ae6d5f82 100644
--- a/clang/test/CodeGen/SystemZ/systemz-abi.c
+++ b/clang/test/CodeGen/SystemZ/systemz-abi.c
@@ -50,7 +50,7 @@ long long pass_longlong(long long arg) { return arg; }
// CHECK-LABEL: define{{.*}} i64 @pass_longlong(i64 %{{.*}})
__int128 pass_int128(__int128 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_int128(ptr dead_on_unwind noalias writable sret(i128) align 8 %{{.*}}, ptr dead_on_return %0)
_Float16 pass__Float16(_Float16 arg) { return arg; }
// CHECK-LABEL: define{{.*}} half @pass__Float16(half %{{.*}})
@@ -62,37 +62,37 @@ double pass_double(double arg) { return arg; }
// CHECK-LABEL: define{{.*}} double @pass_double(double %{{.*}})
long double pass_longdouble(long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr %0)
+// CHECK-LABEL: define{{.*}} void @pass_longdouble(ptr dead_on_unwind noalias writable sret(fp128) align 8 %{{.*}}, ptr dead_on_return %0)
// Complex types
_Complex char pass_complex_char(_Complex char arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_char(ptr dead_on_unwind noalias writable sret({ i8, i8 }) align 1 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex short pass_complex_short(_Complex short arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_short(ptr dead_on_unwind noalias writable sret({ i16, i16 }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex int pass_complex_int(_Complex int arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_int(ptr dead_on_unwind noalias writable sret({ i32, i32 }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex long pass_complex_long(_Complex long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_long(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex long long pass_complex_longlong(_Complex long long arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longlong(ptr dead_on_unwind noalias writable sret({ i64, i64 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex _Float16 pass_complex__Float16(_Complex _Float16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex__Float16(ptr dead_on_unwind noalias writable sret({ half, half }) align 2 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex float pass_complex_float(_Complex float arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_float(ptr dead_on_unwind noalias writable sret({ float, float }) align 4 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex double pass_complex_double(_Complex double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_double(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
_Complex long double pass_complex_longdouble(_Complex long double arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr %{{.*}}arg)
+// CHECK-LABEL: define{{.*}} void @pass_complex_longdouble(ptr dead_on_unwind noalias writable sret({ fp128, fp128 }) align 8 %{{.*}}, ptr dead_on_return %{{.*}}arg)
// Aggregate types
@@ -107,7 +107,7 @@ struct agg_2byte pass_agg_2byte(struct agg_2byte arg) { return arg; }
struct agg_3byte { char a[3]; };
struct agg_3byte pass_agg_3byte(struct agg_3byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_3byte(ptr dead_on_unwind noalias writable sret(%struct.agg_3byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
struct agg_4byte { char a[4]; };
struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
@@ -115,15 +115,15 @@ struct agg_4byte pass_agg_4byte(struct agg_4byte arg) { return arg; }
struct agg_5byte { char a[5]; };
struct agg_5byte pass_agg_5byte(struct agg_5byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_5byte(ptr dead_on_unwind noalias writable sret(%struct.agg_5byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
struct agg_6byte { char a[6]; };
struct agg_6byte pass_agg_6byte(struct agg_6byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_6byte(ptr dead_on_unwind noalias writable sret(%struct.agg_6byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
struct agg_7byte { char a[7]; };
struct agg_7byte pass_agg_7byte(struct agg_7byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_7byte(ptr dead_on_unwind noalias writable sret(%struct.agg_7byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
struct agg_8byte { char a[8]; };
struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
@@ -131,7 +131,7 @@ struct agg_8byte pass_agg_8byte(struct agg_8byte arg) { return arg; }
struct agg_16byte { char a[16]; };
struct agg_16byte pass_agg_16byte(struct agg_16byte arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_16byte(ptr dead_on_unwind noalias writable sret(%struct.agg_16byte) align 1 %{{.*}}, ptr dead_on_return %{{.*}})
// Float-like aggregate types
@@ -153,7 +153,7 @@ struct agg_double pass_agg_double(struct agg_double arg) { return arg; }
struct agg_longdouble { long double a; };
struct agg_longdouble pass_agg_longdouble(struct agg_longdouble arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_longdouble(ptr dead_on_unwind noalias writable sret(%struct.agg_longdouble) align 8 %{{.*}}, ptr dead_on_return %{{.*}})
struct agg__Float16_a4 { _Float16 a __attribute__((aligned (4))); };
struct agg__Float16_a4 pass_agg__Float16_a4(struct agg__Float16_a4 arg) { return arg; }
@@ -167,7 +167,7 @@ struct agg__Float16_a8 pass_agg__Float16_a8(struct agg__Float16_a8 arg) { return
struct agg__Float16_a16 { _Float16 a __attribute__((aligned (16))); };
struct agg__Float16_a16 pass_agg__Float16_a16(struct agg__Float16_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg__Float16_a16(ptr dead_on_unwind noalias writable sret(%struct.agg__Float16_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
struct agg_float_a8 { float a __attribute__((aligned (8))); };
struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
@@ -176,7 +176,7 @@ struct agg_float_a8 pass_agg_float_a8(struct agg_float_a8 arg) { return arg; }
struct agg_float_a16 { float a __attribute__((aligned (16))); };
struct agg_float_a16 pass_agg_float_a16(struct agg_float_a16 arg) { return arg; }
-// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr %{{.*}})
+// CHECK-LABEL: define{{.*}} void @pass_agg_float_a16(ptr dead_on_unwind noalias writable sret(%struct.agg_float_a16) align 16 %{{.*}}, ptr dead_on_return %{{.*}})
// Verify that the following are *not* float-like aggregate types
diff --git a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
index 434937a66389c..d76fb4bd1fda6 100644
--- a/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
+++ b/clang/test/CodeGen/SystemZ/systemz-inline-asm.c
@@ -131,7 +131,7 @@ double test_f64(double f, double g) {
long double test_f128(long double f, long double g) {
asm("axbr %0, %2" : "=f" (f) : "0" (f), "f" (g));
return f;
-// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr noundef readonly captures(none) %0, ptr noundef readonly captures(none) %1)
+// CHECK: define{{.*}} void @test_f128(ptr dead_on_unwind noalias writable writeonly sret(fp128) align 8 captures(none) initializes((0, 16)) [[DEST:%.*]], ptr dead_on_return noundef readonly captures(none) %0, ptr dead_on_return noundef readonly captures(none) %1)
// CHECK: %f = load fp128, ptr %0
// CHECK: %g = load fp128, ptr %1
// CHECK: [[RESULT:%.*]] = tail call fp128 asm "axbr $0, $2", "=f,0,f"(fp128 %f, fp128 %g)
diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c
index f87091427df71..b16b10b7b8a21 100644
--- a/clang/test/CodeGen/X86/cx-complex-range.c
+++ b/clang/test/CodeGen/X86/cx-complex-range.c
@@ -1064,7 +1064,7 @@ _Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) {
// PRMTD-NEXT: ret <2 x half> [[TMP33]]
//
// X86WINPRMTD-LABEL: define dso_local i32 @f1(
-// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2
// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2
diff --git a/clang/test/CodeGen/X86/x86_32-arguments-win32.c b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
index 5b81c43f4bbb8..53d040af725b0 100644
--- a/clang/test/CodeGen/X86/x86_32-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_32-arguments-win32.c
@@ -72,10 +72,10 @@ void receive_vec_512(__m512 x, __m512 y, __m512 z, __m512 w, __m512 q) {
void receive_vec_1024(__m1024 x, __m1024 y, __m1024 z, __m1024 w, __m1024 q) {
gv1024 = x + y + z + w + q;
}
-// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr noundef %0, ptr noundef %1)
-// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr noundef %0, ptr noundef %1, ptr noundef %2, ptr noundef %3, ptr noundef %4)
+// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg noundef %x, <8 x float> inreg noundef %y, <8 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg noundef %x, <16 x float> inreg noundef %y, <16 x float> inreg noundef %z, ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// CHECK-LABEL: define dso_local void @receive_vec_1024(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1, ptr dead_on_return noundef %2, ptr dead_on_return noundef %3, ptr dead_on_return noundef %4)
void pass_vec_128(void) {
__m128 z = {0};
@@ -83,13 +83,13 @@ void pass_vec_128(void) {
}
// CHECK-LABEL: define dso_local void @pass_vec_128()
-// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}}, ptr noundef %{{[^,)]*}})
+// CHECK: call void @receive_vec_128(<4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, <4 x float> inreg noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}}, ptr dead_on_return noundef %{{[^,)]*}})
void __fastcall fastcall_indirect_vec(__m128 x, __m128 y, __m128 z, __m128 w, int edx, __m128 q) {
gv128 = x + y + z + w + q;
}
-// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr inreg noundef %0, i32 inreg noundef %edx, ptr noundef %1)
+// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01 at fastcall_indirect_vec@84"(<4 x float> inreg noundef %x, <4 x float> inreg noundef %y, <4 x float> inreg noundef %z, ptr dead_on_return inreg noundef %0, i32 inreg noundef %edx, ptr dead_on_return noundef %1)
struct __declspec(align(1)) Align1 { unsigned long long x; };
struct __declspec(align(4)) Align4 { unsigned long long x; };
@@ -156,4 +156,4 @@ void pass_fixed_align_variadic() {
// correctly in Clang than it is to be bug for bug compatible, so we pass such
// arguments indirectly.
// CHECK-LABEL: define dso_local void @pass_fixed_align_variadic()
-// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr noundef %{{[^)]*}}, i32 noundef 42)
+// CHECK: call void (ptr, ...) @receive_fixed_align_variadic(ptr dead_on_return noundef %{{[^)]*}}, i32 noundef 42)
diff --git a/clang/test/CodeGen/X86/x86_64-arguments-win32.c b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
index 8768e73a854aa..6010e531acb00 100644
--- a/clang/test/CodeGen/X86/x86_64-arguments-win32.c
+++ b/clang/test/CodeGen/X86/x86_64-arguments-win32.c
@@ -21,7 +21,7 @@ void f4(unsigned short a) {}
// CHECK-LABEL: define dso_local void @f5(i64 noundef %a.coerce)
void f5(_Complex float a) {}
-// CHECK-LABEL: define dso_local void @f6(ptr noundef %a)
+// CHECK-LABEL: define dso_local void @f6(ptr dead_on_return noundef %a)
void f6(_Complex double a) {}
// CHECK-LABEL: define dso_local i64 @f7()
diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp
index 1c26d68e434f4..53fc53c2f7296 100644
--- a/clang/test/CodeGen/aapcs64-align.cpp
+++ b/clang/test/CodeGen/aapcs64-align.cpp
@@ -122,8 +122,8 @@ unsigned sizeof_RidiculouslyOverSizedBitfield = sizeof(RidiculouslyOverSizedBitf
unsigned alignof_RidiculouslyOverSizedBitfield = alignof(RidiculouslyOverSizedBitfield);
// CHECK: define{{.*}} void @g9
-// CHECK: call void @f9(i32 noundef 1, ptr noundef nonnull %agg.tmp)
-// CHECK: declare void @f9(i32 noundef, ptr noundef)
+// CHECK: call void @f9(i32 noundef 1, ptr dead_on_return noundef nonnull %agg.tmp)
+// CHECK: declare void @f9(i32 noundef, ptr dead_on_return noundef)
void f9(int a, RidiculouslyOverSizedBitfield b);
void g9() {
RidiculouslyOverSizedBitfield s = {42};
diff --git a/clang/test/CodeGen/arm-aapcs-vfp.c b/clang/test/CodeGen/arm-aapcs-vfp.c
index 6581929f99f14..e60ed1e52c33a 100644
--- a/clang/test/CodeGen/arm-aapcs-vfp.c
+++ b/clang/test/CodeGen/arm-aapcs-vfp.c
@@ -65,7 +65,7 @@ struct big_struct {
float f4;
};
// CHECK: define{{.*}} arm_aapcs_vfpcc void @test_big([5 x i32] %{{.*}})
-// CHECK64: define{{.*}} void @test_big(ptr noundef %{{.*}})
+// CHECK64: define{{.*}} void @test_big(ptr dead_on_return noundef %{{.*}})
// CHECK64: call void @llvm.memcpy
// CHECK64: call void @big_callee(ptr
extern void big_callee(struct big_struct);
diff --git a/clang/test/CodeGen/arm-abi-vector.c b/clang/test/CodeGen/arm-abi-vector.c
index c2a8902007980..93b770878c3fa 100644
--- a/clang/test/CodeGen/arm-abi-vector.c
+++ b/clang/test/CodeGen/arm-abi-vector.c
@@ -177,11 +177,11 @@ double varargs_vec_19c(int fixed, ...) {
double test_19c(__char19 *in) {
// CHECK: test_19c
-// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call arm_aapcscc double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
// APCS-GNU: test_19c
-// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// APCS-GNU: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
// ANDROID: test_19c
-// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// ANDROID: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
return varargs_vec_19c(19, *in);
}
diff --git a/clang/test/CodeGen/arm-swiftcall.c b/clang/test/CodeGen/arm-swiftcall.c
index 677b878c6765d..55c22a45fd8de 100644
--- a/clang/test/CodeGen/arm-swiftcall.c
+++ b/clang/test/CodeGen/arm-swiftcall.c
@@ -261,7 +261,7 @@ TEST(struct_big_1)
// CHECK-LABEL: define{{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret({{.*}})
// Should not be byval.
-// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr{{( %.*)?}})
+// CHECK-LABEL: define{{.*}} void @take_struct_big_1(ptr dead_on_return{{( %.*)?}})
/*****************************************************************************/
/********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGen/arm64-abi-vector.c b/clang/test/CodeGen/arm64-abi-vector.c
index 81e42315c883b..cf50cdd2fe86e 100644
--- a/clang/test/CodeGen/arm64-abi-vector.c
+++ b/clang/test/CodeGen/arm64-abi-vector.c
@@ -128,7 +128,7 @@ double varargs_vec_19c(int fixed, ...) {
double test_19c(__char19 *in) {
// CHECK: test_19c
-// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
return varargs_vec_19c(19, *in);
}
@@ -211,7 +211,7 @@ double varargs_vec_5i(int fixed, ...) {
double test_5i(__int5 *in) {
// CHECK: test_5i
-// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
return varargs_vec_5i(5, *in);
}
@@ -231,7 +231,7 @@ double varargs_vec_3d(int fixed, ...) {
double test_3d(__double3 *in) {
// CHECK: test_3d
-// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
return varargs_vec_3d(3, *in);
}
@@ -291,7 +291,7 @@ double test(__char3 *c3, __char5 *c5, __char9 *c9, __char19 *c19,
__short3 *s3, __short5 *s5, __int3 *i3, __int5 *i5,
__double3 *d3) {
double ret = varargs_vec(3, *c3, *c5, *c9, *c19, *s3, *s5, *i3, *i5, *d3);
-// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr noundef {{%.*}}, ptr noundef {{%.*}})
+// CHECK: call double (i32, ...) @varargs_vec(i32 noundef 3, i32 {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, <2 x i32> {{%.*}}, <4 x i32> {{%.*}}, <4 x i32> {{%.*}}, ptr dead_on_return noundef {{%.*}}, ptr dead_on_return noundef {{%.*}})
return ret;
}
@@ -350,7 +350,7 @@ __attribute__((noinline)) double args_vec_19c(int fixed, __char19 c19) {
double fixed_19c(__char19 *in) {
// CHECK: fixed_19c
-// CHECK: call double @args_vec_19c(i32 noundef 19, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_19c(i32 noundef 19, ptr dead_on_return noundef {{%.*}})
return args_vec_19c(19, *in);
}
@@ -409,7 +409,7 @@ __attribute__((noinline)) double args_vec_5i(int fixed, __int5 c5) {
double fixed_5i(__int5 *in) {
// CHECK: fixed_5i
-// CHECK: call double @args_vec_5i(i32 noundef 5, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_5i(i32 noundef 5, ptr dead_on_return noundef {{%.*}})
return args_vec_5i(5, *in);
}
@@ -424,6 +424,6 @@ __attribute__((noinline)) double args_vec_3d(int fixed, __double3 c3) {
double fixed_3d(__double3 *in) {
// CHECK: fixed_3d
-// CHECK: call double @args_vec_3d(i32 noundef 3, ptr noundef {{%.*}})
+// CHECK: call double @args_vec_3d(i32 noundef 3, ptr dead_on_return noundef {{%.*}})
return args_vec_3d(3, *in);
}
diff --git a/clang/test/CodeGen/arm64-arguments.c b/clang/test/CodeGen/arm64-arguments.c
index 4c4f85d923e78..2e3ab388432f6 100644
--- a/clang/test/CodeGen/arm64-arguments.c
+++ b/clang/test/CodeGen/arm64-arguments.c
@@ -163,7 +163,7 @@ void f32(struct s32 s) { }
// A composite type larger than 16 bytes should be passed indirectly.
struct s33 { char buf[32*32]; };
void f33(struct s33 s) { }
-// CHECK: define{{.*}} void @f33(ptr noundef %s)
+// CHECK: define{{.*}} void @f33(ptr dead_on_return noundef %s)
struct s34 { char c; };
void f34(struct s34 s);
@@ -226,9 +226,9 @@ T_float32x2 f1_0(T_float32x2 a0) { return a0; }
// CHECK: define{{.*}} <4 x float> @f1_1(<4 x float> noundef %{{.*}})
T_float32x4 f1_1(T_float32x4 a0) { return a0; }
// Vector with length bigger than 16-byte is illegal and is passed indirectly.
-// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_2(ptr dead_on_unwind noalias writable sret(<8 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
T_float32x8 f1_2(T_float32x8 a0) { return a0; }
-// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr noundef %0)
+// CHECK: define{{.*}} void @f1_3(ptr dead_on_unwind noalias writable sret(<16 x float>) align 16 %{{.*}}, ptr dead_on_return noundef %0)
T_float32x16 f1_3(T_float32x16 a0) { return a0; }
// Testing alignment with aggregates: HFA, aggregates with size <= 16 bytes and
@@ -278,7 +278,7 @@ struct s37
typedef struct s37 s37_with_align;
int32x4_t f37(int i, s37_with_align s1, s37_with_align s2) {
-// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} <4 x i32> @f37(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
// CHECK: load <4 x i32>, ptr %s1, align 16
// CHECK: load <4 x i32>, ptr %s2, align 16
int32x4_t v = vaddq_s32(*(int32x4_t *)&s1,
@@ -292,7 +292,7 @@ int32x4_t caller37() {
// CHECK: %[[b:.*]] = alloca %struct.s37, align 16
// CHECK: call void @llvm.memcpy
// CHECK: call void @llvm.memcpy
-// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call <4 x i32> @f37(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
return f37(3, g37, g37);
}
@@ -530,7 +530,7 @@ typedef struct s42 s42_no_align;
// passing structs in registers
__attribute__ ((noinline))
int f42(int i, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
// CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -545,14 +545,14 @@ int caller42() {
// CHECK: %[[b:.*]] = alloca %struct.s42, align 4
// CHECK: call void @llvm.memcpy.p0.p0.i64
// CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
return f42(3, g42, g42_2);
}
// passing structs on stack
__attribute__ ((noinline))
int f42_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
int i9, s42_no_align s1, s42_no_align s2) {
-// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f42_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
// CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s42, ptr %s2, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s42, ptr %s1, i32 0, i32 1
@@ -565,7 +565,7 @@ int caller42_stack() {
// CHECK: %[[b:.*]] = alloca %struct.s42, align 4
// CHECK: call void @llvm.memcpy.p0.p0.i64
// CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f42_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
return f42_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g42, g42_2);
}
@@ -583,7 +583,7 @@ typedef struct s43 s43_with_align;
// passing aligned structs in registers
__attribute__ ((noinline))
int f43(int i, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43(i32 noundef %i, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
// CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -598,14 +598,14 @@ int caller43() {
// CHECK: %[[b:.*]] = alloca %struct.s43, align 16
// CHECK: call void @llvm.memcpy.p0.p0.i64
// CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43(i32 noundef 3, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43(i32 noundef 3, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
return f43(3, g43, g43_2);
}
// passing aligned structs on stack
__attribute__ ((noinline))
int f43_stack(int i, int i2, int i3, int i4, int i5, int i6, int i7, int i8,
int i9, s43_with_align s1, s43_with_align s2) {
-// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr noundef %s1, ptr noundef %s2)
+// CHECK: define{{.*}} i32 @f43_stack(i32 noundef %i, i32 noundef %i2, i32 noundef %i3, i32 noundef %i4, i32 noundef %i5, i32 noundef %i6, i32 noundef %i7, i32 noundef %i8, i32 noundef %i9, ptr dead_on_return noundef %s1, ptr dead_on_return noundef %s2)
// CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s43, ptr %s2, i32 0, i32 0
// CHECK: getelementptr inbounds nuw %struct.s43, ptr %s1, i32 0, i32 1
@@ -618,7 +618,7 @@ int caller43_stack() {
// CHECK: %[[b:.*]] = alloca %struct.s43, align 16
// CHECK: call void @llvm.memcpy.p0.p0.i64
// CHECK: call void @llvm.memcpy.p0.p0.i64
-// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr noundef %[[a]], ptr noundef %[[b]])
+// CHECK: call i32 @f43_stack(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6, i32 noundef 7, i32 noundef 8, i32 noundef 9, ptr dead_on_return noundef %[[a]], ptr dead_on_return noundef %[[b]])
return f43_stack(1, 2, 3, 4, 5, 6, 7, 8, 9, g43, g43_2);
}
diff --git a/clang/test/CodeGen/arm64-microsoft-arguments.cpp b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
index 85472645acb3b..a0a81be54325f 100644
--- a/clang/test/CodeGen/arm64-microsoft-arguments.cpp
+++ b/clang/test/CodeGen/arm64-microsoft-arguments.cpp
@@ -29,7 +29,7 @@ S2 f2() {
// Pass and return for type size > 16 bytes.
// CHECK: define {{.*}} void @{{.*}}f3{{.*}}(ptr dead_on_unwind noalias writable sret(%struct.S3) align 4 %agg.result)
-// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr noundef %agg.tmp)
+// CHECK: call void {{.*}}func3{{.*}}(ptr dead_on_unwind writable sret(%struct.S3) align 4 %agg.result, ptr dead_on_return noundef %agg.tmp)
struct S3 {
int a[5];
};
diff --git a/clang/test/CodeGen/armv7k-abi.c b/clang/test/CodeGen/armv7k-abi.c
index 872e6423a4a99..6a781bc04d042 100644
--- a/clang/test/CodeGen/armv7k-abi.c
+++ b/clang/test/CodeGen/armv7k-abi.c
@@ -39,7 +39,7 @@ typedef struct {
double z;
} BigStruct;
-// CHECK: define{{.*}} void @big_struct_indirect(ptr noundef %b)
+// CHECK: define{{.*}} void @big_struct_indirect(ptr dead_on_return noundef %b)
void big_struct_indirect(BigStruct b) {}
// CHECK: define{{.*}} void @return_big_struct_indirect(ptr dead_on_unwind noalias writable sret
diff --git a/clang/test/CodeGen/atomic-arm64.c b/clang/test/CodeGen/atomic-arm64.c
index d2a30a3b6e66f..d539cad0c6a7d 100644
--- a/clang/test/CodeGen/atomic-arm64.c
+++ b/clang/test/CodeGen/atomic-arm64.c
@@ -57,7 +57,7 @@ void test3(pointer_pair_t pair) {
}
// CHECK-LABEL:define{{.*}} void @test4(
-// CHECK-SAME: ptr noundef [[QUAD:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[QUAD:%.*]])
// CHECK: [[QUAD_INDIRECT_ADDR:%.*]] = alloca ptr, align 8
// CHECK-NEXT: [[TEMP:%.*]] = alloca [[QUAD_T:%.*]], align 8
// CHECK-NEXT: store ptr [[QUAD]], ptr [[QUAD_INDIRECT_ADDR]]
diff --git a/clang/test/CodeGen/attr-noundef.cpp b/clang/test/CodeGen/attr-noundef.cpp
index abdf9496bd396..619dbec7678db 100644
--- a/clang/test/CodeGen/attr-noundef.cpp
+++ b/clang/test/CodeGen/attr-noundef.cpp
@@ -27,7 +27,7 @@ struct NoCopy {
NoCopy ret_nocopy() { return {}; }
void pass_nocopy(NoCopy e) {}
// CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
struct Huge {
int a[1024];
@@ -35,7 +35,8 @@ struct Huge {
Huge ret_huge() { return {}; }
void pass_huge(Huge h) {}
// CHECK: [[DEF]] void @{{.*}}ret_huge{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-INTEL: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr noundef
+// CHECK-AARCH: [[DEF]] void @{{.*}}pass_huge{{.*}}(ptr dead_on_return noundef
} // namespace check_structs
//************ Passing unions by value
@@ -59,7 +60,7 @@ union NoCopy {
NoCopy ret_nocopy() { return {}; }
void pass_nocopy(NoCopy e) {}
// CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(ptr dead_on_unwind noalias writable sret({{[^)]+}}) align 4 %
-// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr noundef %
+// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(ptr dead_on_return noundef %
} // namespace check_unions
//************ Passing `this` pointers
diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c
index cf74ab2dcca3f..fba692cac4492 100644
--- a/clang/test/CodeGen/cx-complex-range.c
+++ b/clang/test/CodeGen/cx-complex-range.c
@@ -1520,7 +1520,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
// PRMTD-NEXT: ret { double, double } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD-LABEL: define dso_local void @divd(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1744,7 +1744,7 @@ void mulassignf(_Complex float *a, _Complex float b) {
// PRMTD_FAST-NEXT: ret { double, double } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @divd(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -1938,7 +1938,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
// PRMTD-NEXT: ret void
//
// X86WINPRMTD-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2180,7 +2180,7 @@ _Complex double divd(_Complex double a, _Complex double b) {
// PRMTD_FAST-NEXT: ret void
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2325,7 +2325,7 @@ void divassignd(_Complex double *a, _Complex double b) {
// PRMTD-NEXT: ret { double, double } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD-LABEL: define dso_local void @muld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2457,7 +2457,7 @@ void divassignd(_Complex double *a, _Complex double b) {
// PRMTD_FAST-NEXT: ret { double, double } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @muld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -2594,7 +2594,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
// PRMTD-NEXT: ret void
//
// X86WINPRMTD-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2744,7 +2744,7 @@ _Complex double muld(_Complex double a, _Complex double b) {
// PRMTD_FAST-NEXT: ret void
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignd(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -2922,7 +2922,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
// PRMTD-NEXT: ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD-LABEL: define dso_local void @divld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3190,7 +3190,7 @@ void mulassignd(_Complex double *a, _Complex double b) {
// PRMTD_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @divld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -3432,7 +3432,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
// PRMTD-NEXT: ret void
//
// X86WINPRMTD-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3702,7 +3702,7 @@ _Complex long double divld(_Complex long double a, _Complex long double b) {
// PRMTD_FAST-NEXT: ret void
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @divassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -3895,7 +3895,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
// PRMTD-NEXT: ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD-LABEL: define dso_local void @mulld(
-// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4059,7 +4059,7 @@ void divassignld(_Complex long double *a, _Complex long double b) {
// PRMTD_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[DOTFCA_1_INSERT]]
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @mulld(
-// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr dead_on_return noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[A_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8
@@ -4220,7 +4220,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
// PRMTD-NEXT: ret void
//
// X86WINPRMTD-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4386,7 +4386,7 @@ _Complex long double mulld(_Complex long double a, _Complex long double b) {
// PRMTD_FAST-NEXT: ret void
//
// X86WINPRMTD_STRICT-LABEL: define dso_local void @mulassignld(
-// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: ptr noundef [[A:%.*]], ptr dead_on_return noundef [[B:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[B_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i32 0, i32 0
// X86WINPRMTD_STRICT-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8
@@ -4644,7 +4644,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
// PRMTD-NEXT: ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
//
// X86WINPRMTD-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
// X86WINPRMTD-NEXT: entry:
// X86WINPRMTD-NEXT: [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
// X86WINPRMTD-NEXT: [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
@@ -5052,7 +5052,7 @@ void mulassignld(_Complex long double *a, _Complex long double b) {
// PRMTD_FAST-NEXT: ret <2 x float> [[RETVAL_SROA_0_4_VEC_INSERT]]
//
// X86WINPRMTD_STRICT-LABEL: define dso_local i64 @f1(
-// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
+// X86WINPRMTD_STRICT-SAME: i64 noundef [[A_COERCE:%.*]], ptr dead_on_return noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] {
// X86WINPRMTD_STRICT-NEXT: entry:
// X86WINPRMTD_STRICT-NEXT: [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE]] to i32
// X86WINPRMTD_STRICT-NEXT: [[TMP0:%.*]] = bitcast i32 [[A_SROA_0_0_EXTRACT_TRUNC]] to float
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index fdca4012ee4a4..7cfd992fd48b4 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -33,9 +33,9 @@
// Make sure 128 and 64 bit versions are passed like integers.
void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
// LIN64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
-// WIN32: define dso_local void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// WIN64: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
+// WIN32: define dso_local void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
// NVPTX64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
// NVPTX: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
// SPARCV9: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
@@ -50,22 +50,22 @@ void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
// ARC: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 inreg %{{.+}})
// XCORE: define{{.*}} void @ParamPassing(ptr byval(i128) align 4 %{{.+}}, i64 %{{.+}})
// RISCV64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
// WASM: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
// PPC64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
// PPC32: define{{.*}} void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
// AARCH64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
// AARCH64DARWIN: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
// ARM: define{{.*}} arm_aapcscc void @ParamPassing(ptr byval(i128) align 8 %{{.+}}, i64 %{{.+}})
// LA64: define{{.*}} void @ParamPassing(i128 %{{.+}}, i64 %{{.+}})
-// LA32: define{{.*}} void @ParamPassing(ptr %{{.+}}, i64 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing(ptr dead_on_return %{{.+}}, i64 %{{.+}})
void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
// LIN64: define{{.*}} void @ParamPassing2(i64 %{{.+}}, i64 %{{.+}}, i64 %{{.+}})
-// WIN64: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
-// WIN32: define dso_local void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// WIN64: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
+// WIN32: define dso_local void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
// NVPTX64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
// NVPTX: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
// SPARCV9: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
@@ -80,16 +80,16 @@ void ParamPassing2(_BitInt(127) b, _BitInt(63) c) {}
// ARC: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 inreg %{{.+}})
// XCORE: define{{.*}} void @ParamPassing2(ptr byval(i128) align 4 %{{.+}}, i63 %{{.+}})
// RISCV64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// RISCV32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// RISCV32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
// WASM: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
-// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 signext %{{.+}})
+// SYSTEMZ: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 signext %{{.+}})
// PPC64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
// PPC32: define{{.*}} void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
// AARCH64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
// AARCH64DARWIN: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 %{{.+}})
// ARM: define{{.*}} arm_aapcscc void @ParamPassing2(ptr byval(i128) align 8 %{{.+}}, i63 %{{.+}})
// LA64: define{{.*}} void @ParamPassing2(i127 %{{.+}}, i63 signext %{{.+}})
-// LA32: define{{.*}} void @ParamPassing2(ptr %{{.+}}, i63 %{{.+}})
+// LA32: define{{.*}} void @ParamPassing2(ptr dead_on_return %{{.+}}, i63 %{{.+}})
// Make sure we follow the signext rules for promotable integer types.
void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
@@ -129,10 +129,10 @@ void ParamPassing3(_BitInt(15) a, _BitInt(31) b) {}
// _BitInt widths to alert us to enable the test.
void ParamPassing4(_BitInt(129) a) {}
// LIN64: define{{.*}} void @ParamPassing4(ptr byval([24 x i8]) align 8 %{{.+}})
-// WIN64: define dso_local void @ParamPassing4(ptr %{{.+}})
-// LIN32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// WIN32: define dso_local void @ParamPassing4(ptr %{{.+}})
-// AARCH64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// WIN64: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LIN32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// WIN32: define dso_local void @ParamPassing4(ptr dead_on_return %{{.+}})
+// AARCH64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
// NVPTX64-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
// NVPTX-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
// SPARCV9-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
@@ -154,8 +154,8 @@ void ParamPassing4(_BitInt(129) a) {}
// PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
// AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
// ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// LA64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// LA32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA64: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
+// LA32: define{{.*}} void @ParamPassing4(ptr dead_on_return %{{.+}})
#endif
_BitInt(63) ReturnPassing(void) { return 0; }
diff --git a/clang/test/CodeGen/isfpclass.c b/clang/test/CodeGen/isfpclass.c
index 26dd846a2bf20..ee3a22b40fefd 100644
--- a/clang/test/CodeGen/isfpclass.c
+++ b/clang/test/CodeGen/isfpclass.c
@@ -160,7 +160,7 @@ int4 check_isfpclass_nan_strict_v4f32(float4 x) {
}
// CHECK-LABEL: define dso_local void @check_isfpclass_nan_v4f64
-// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: (ptr dead_on_unwind noalias writable writeonly sret(<4 x i64>) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
// CHECK-NEXT: entry:
// CHECK-NEXT: [[X:%.*]] = load <4 x double>, ptr [[TMP0]], align 16, !tbaa [[TBAA2:![0-9]+]]
// CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x double> [[X]], zeroinitializer
diff --git a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
index eb706154300a2..0124cc5c06d43 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa-indirect-args.c
@@ -30,37 +30,37 @@ long double powl(long double a, long double b);
//
// CHECK-PPC-LABEL: define dso_local ppc_fp128 @test_powl(
// CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]], ppc_fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-PPC: [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-PPC: [[CALL:%.*]] = tail call ppc_fp128 @powl(ppc_fp128 noundef [[A]], ppc_fp128 noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
//
// CHECK-ARM-LABEL: define dso_local double @test_powl(
// CHECK-ARM-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM: [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM: [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
//
// CHECK-ARM-HF-LABEL: define dso_local double @test_powl(
// CHECK-ARM-HF-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-ARM-HF: [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-ARM-HF: [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
//
// CHECK-THUMB-LABEL: define double @test_powl(
// CHECK-THUMB-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-THUMB: [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
+// CHECK-THUMB: [[CALL:%.*]] = tail call double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA3:![0-9]+]]
//
// CHECK-AARCH-LABEL: define dso_local fp128 @test_powl(
// CHECK-AARCH-SAME: fp128 noundef [[A:%.*]], fp128 noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-AARCH: [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR2:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-AARCH: [[CALL:%.*]] = tail call fp128 @powl(fp128 noundef [[A]], fp128 noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
//
// CHECK-SPIR-LABEL: define dso_local spir_func double @test_powl(
// CHECK-SPIR-SAME: double noundef [[A:%.*]], double noundef [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-SPIR: [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR3:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
+// CHECK-SPIR: [[CALL:%.*]] = tail call spir_func double @powl(double noundef [[A]], double noundef [[B]]) #[[ATTR4:[0-9]+]], !tbaa [[TBAA2:![0-9]+]]
//
// CHECK-MINGW32-LABEL: define dso_local void @test_powl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-// CHECK-MINGW32: [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3:![0-9]+]]
-// CHECK-MINGW32: [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32: store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32: store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32: call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]], ptr noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
-// CHECK-MINGW32: [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32: store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret(x86_fp80) align 16 captures(none) initializes((0, 10)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]], ptr dead_on_return noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-MINGW32: [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6:![0-9]+]]
+// CHECK-MINGW32: [[B:%.*]] = load x86_fp80, ptr [[TMP1]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32: store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32: store x86_fp80 [[B]], ptr [[BYVAL_TEMP1:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32: call void @powl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP1]]) #[[ATTR3:[0-9]+]]
+// CHECK-MINGW32: [[TMP2:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32: store x86_fp80 [[TMP2]], ptr [[AGG_RESULT]], align 16, !tbaa [[TBAA6]]
//
long double test_powl(long double a, long double b) {
return powl(a, b);
@@ -93,51 +93,51 @@ long double test_powl(long double a, long double b) {
// CHECK-I686: store x86_fp80 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 4
//
// CHECK-PPC-LABEL: define dso_local void @test_cargl(
-// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-PPC-SAME: ptr dead_on_unwind noalias writable writeonly sret({ ppc_fp128, ppc_fp128 }) align 16 captures(none) initializes((0, 32)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ ppc_fp128, ppc_fp128 }) align 16 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
// CHECK-PPC: [[CLD_REAL:%.*]] = load ppc_fp128, ptr [[CLD]], align 16
// CHECK-PPC: [[CLD_IMAG:%.*]] = load ppc_fp128, ptr [[CLD_IMAGP:%.*]], align 16
// CHECK-PPC: store ppc_fp128 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
// CHECK-PPC: store ppc_fp128 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-PPC: [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-PPC: [[CALL:%.*]] = tail call ppc_fp128 @cargl(ptr noundef nonnull byval({ ppc_fp128, ppc_fp128 }) align 16 [[BYVAL_TEMP]]) #[[ATTR4]]
// CHECK-PPC: store ppc_fp128 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
// CHECK-PPC: store ppc_fp128 [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 16
//
// CHECK-ARM-LABEL: define dso_local void @test_cargl(
-// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM: [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], [2 x i64] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM: [[CALL:%.*]] = tail call double @cargl([2 x i64] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
// CHECK-ARM: store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
// CHECK-ARM: store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
//
// CHECK-ARM-HF-LABEL: define dso_local { double, double } @test_cargl(
-// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-ARM-HF: [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF-SAME: { double, double } noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-ARM-HF: [[CALL:%.*]] = tail call double @cargl({ double, double } noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
//
// CHECK-THUMB-LABEL: define { double, double } @test_cargl(
-// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-THUMB: [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB-SAME: [2 x double] noundef [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-THUMB: [[CALL:%.*]] = tail call double @cargl([2 x double] noundef [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA3]]
//
// CHECK-AARCH-LABEL: define dso_local { fp128, fp128 } @test_cargl(
-// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-// CHECK-AARCH: [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH-SAME: [2 x fp128] noundef alignstack(16) [[CLD_COERCE:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-AARCH: [[CALL:%.*]] = tail call fp128 @cargl([2 x fp128] noundef alignstack(16) [[CLD_COERCE]]) #[[ATTR3]], !tbaa [[TBAA2]]
//
// CHECK-SPIR-LABEL: define dso_local spir_func void @test_cargl(
-// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+// CHECK-SPIR-SAME: ptr dead_on_unwind noalias writable writeonly sret({ double, double }) align 8 captures(none) initializes((0, 16)) [[AGG_RESULT:%.*]], ptr noundef readonly byval({ double, double }) align 8 captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
// CHECK-SPIR: [[CLD_REAL:%.*]] = load double, ptr [[CLD]], align 8
// CHECK-SPIR: [[CLD_IMAG:%.*]] = load double, ptr [[CLD_IMAGP:%.*]], align 8
// CHECK-SPIR: store double [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 8
// CHECK-SPIR: store double [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 8
-// CHECK-SPIR: [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-SPIR: [[CALL:%.*]] = tail call spir_func double @cargl(ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]]) #[[ATTR4]]
// CHECK-SPIR: store double [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 8
// CHECK-SPIR: store double [[MUL_IR:%.*]], ptr [[AGG_RESULT_IMAGP:%.*]], align 8
//
// CHECK-MINGW32-LABEL: define dso_local void @test_cargl(
-// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32-SAME: ptr dead_on_unwind noalias writable writeonly sret({ x86_fp80, x86_fp80 }) align 16 captures(none) initializes((0, 10), (16, 26)) [[AGG_RESULT:%.*]], ptr dead_on_return noundef readonly captures(none) [[CLD:%.*]]) local_unnamed_addr #[[ATTR0]] {
// CHECK-MINGW32: [[CLD_REAL:%.*]] = load x86_fp80, ptr [[CLD]], align 16
// CHECK-MINGW32: [[CLD_IMAG:%.*]] = load x86_fp80, ptr [[CLD_IMAGP:%.*]], align 16
// CHECK-MINGW32: store x86_fp80 [[CLD_REAL]], ptr [[BYVAL_TEMP:%.*]], align 16
// CHECK-MINGW32: store x86_fp80 [[CLD_IMAG]], ptr [[BYVAL_TEMP_IMAGP:%.*]], align 16
-// CHECK-MINGW32: call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
-// CHECK-MINGW32: [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA3]]
+// CHECK-MINGW32: call void @cargl(ptr dead_on_unwind nonnull writable sret(x86_fp80) align 16 [[TMP:%.*]], ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32: [[TMP0:%.*]] = load x86_fp80, ptr [[TMP]], align 16, !tbaa [[TBAA6]]
// CHECK-MINGW32: [[CLD_REAL3:%.*]] = load x86_fp80, ptr [[CLD]], align 16
// CHECK-MINGW32: [[CLD_IMAG5:%.*]] = load x86_fp80, ptr [[CLD_IMAGP]], align 16
// CHECK-MINGW32: store x86_fp80 [[MUL_RL:%.*]], ptr [[AGG_RESULT]], align 16
@@ -166,33 +166,33 @@ int ilogbl(long double a);
//
// CHECK-PPC-LABEL: define dso_local i32 @test_ilogb(
// CHECK-PPC-SAME: ppc_fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-PPC: [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-PPC: [[CALL:%.*]] = tail call i32 @ilogbl(ppc_fp128 noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
//
// CHECK-ARM-LABEL: define dso_local i32 @test_ilogb(
// CHECK-ARM-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM: [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM: [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
//
// CHECK-ARM-HF-LABEL: define dso_local i32 @test_ilogb(
// CHECK-ARM-HF-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-ARM-HF: [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-ARM-HF: [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
//
// CHECK-THUMB-LABEL: define i32 @test_ilogb(
// CHECK-THUMB-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-THUMB: [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA3]]
+// CHECK-THUMB: [[CALL:%.*]] = tail call i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA3]]
//
// CHECK-AARCH-LABEL: define dso_local i32 @test_ilogb(
// CHECK-AARCH-SAME: fp128 noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-AARCH: [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR2]], !tbaa [[TBAA2]]
+// CHECK-AARCH: [[CALL:%.*]] = tail call i32 @ilogbl(fp128 noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
//
// CHECK-SPIR-LABEL: define dso_local spir_func i32 @test_ilogb(
// CHECK-SPIR-SAME: double noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-SPIR: [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR3]], !tbaa [[TBAA2]]
+// CHECK-SPIR: [[CALL:%.*]] = tail call spir_func i32 @ilogbl(double noundef [[A]]) #[[ATTR4]], !tbaa [[TBAA2]]
//
// CHECK-MINGW32-LABEL: define dso_local i32 @test_ilogb(
-// CHECK-MINGW32-SAME: ptr noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
-// CHECK-MINGW32: [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32: store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA3]]
-// CHECK-MINGW32: [[CALL:%.*]] = call i32 @ilogbl(ptr noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
+// CHECK-MINGW32-SAME: ptr dead_on_return noundef readonly captures(none) [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-MINGW32: [[A:%.*]] = load x86_fp80, ptr [[TMP0]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32: store x86_fp80 [[A]], ptr [[BYVAL_TEMP:%.*]], align 16, !tbaa [[TBAA6]]
+// CHECK-MINGW32: [[CALL:%.*]] = call i32 @ilogbl(ptr dead_on_return noundef nonnull [[BYVAL_TEMP]]) #[[ATTR3]]
//
int test_ilogb(long double a) {
return ilogbl(a);
@@ -243,8 +243,8 @@ int test_ilogb(long double a) {
// CHECK-SPIR: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
// CHECK-SPIR: [[META5]] = !{!"Simple C/C++ TBAA"}
//.
-// CHECK-MINGW32: [[TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
-// CHECK-MINGW32: [[META4]] = !{!"long double", [[META5:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META5]] = !{!"omnipotent char", [[META6:![0-9]+]], i64 0}
-// CHECK-MINGW32: [[META6]] = !{!"Simple C/C++ TBAA"}
+// CHECK-MINGW32: [[TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+// CHECK-MINGW32: [[META7]] = !{!"long double", [[META8:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+// CHECK-MINGW32: [[META9]] = !{!"Simple C/C++ TBAA"}
//.
diff --git a/clang/test/CodeGen/mingw-long-double.c b/clang/test/CodeGen/mingw-long-double.c
index 0fc8f01509682..b98929701bc62 100644
--- a/clang/test/CodeGen/mingw-long-double.c
+++ b/clang/test/CodeGen/mingw-long-double.c
@@ -29,15 +29,15 @@ long double TestLD(long double x) {
return x * x;
}
// GNU32: define dso_local x86_fp80 @TestLD(x86_fp80 noundef %x)
-// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr noundef %0)
+// GNU64: define dso_local void @TestLD(ptr dead_on_unwind noalias writable sret(x86_fp80) align 16 %agg.result, ptr dead_on_return noundef %0)
// MSC64: define dso_local double @TestLD(double noundef %x)
long double _Complex TestLDC(long double _Complex x) {
return x * x;
}
// GNU32: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 4 %agg.result, ptr noundef byval({ x86_fp80, x86_fp80 }) align 4 %x)
-// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr noundef %x)
-// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr noundef %x)
+// GNU64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ x86_fp80, x86_fp80 }) align 16 %agg.result, ptr dead_on_return noundef %x)
+// MSC64: define dso_local void @TestLDC(ptr dead_on_unwind noalias writable sret({ double, double }) align 8 %agg.result, ptr dead_on_return noundef %x)
// GNU32: declare dso_local void @__mulxc3
// GNU64: declare dso_local void @__mulxc3
diff --git a/clang/test/CodeGen/ms_abi.c b/clang/test/CodeGen/ms_abi.c
index 528e546f315d5..5d58c9816da78 100644
--- a/clang/test/CodeGen/ms_abi.c
+++ b/clang/test/CodeGen/ms_abi.c
@@ -142,7 +142,7 @@ struct i128 {
};
__attribute__((ms_abi)) struct i128 f7(struct i128 a) {
- // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
- // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr noundef %a)
+ // WIN64: define dso_local void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
+ // FREEBSD: define{{.*}} win64cc void @f7(ptr dead_on_unwind noalias writable sret(%struct.i128) align 8 %agg.result, ptr dead_on_return noundef %a)
return a;
}
diff --git a/clang/test/CodeGen/pass-by-value-noalias.c b/clang/test/CodeGen/pass-by-value-noalias.c
index bc35d13c4df6a..e673ceb80bebe 100644
--- a/clang/test/CodeGen/pass-by-value-noalias.c
+++ b/clang/test/CodeGen/pass-by-value-noalias.c
@@ -11,6 +11,6 @@ struct Foo {
int f;
};
-// WITH_NOALIAS: define{{.*}} void @take(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
void take(struct Foo arg) {}
diff --git a/clang/test/CodeGen/ptrauth-in-c-struct.c b/clang/test/CodeGen/ptrauth-in-c-struct.c
index 2aec31ec3baf9..c74be17b4c837 100644
--- a/clang/test/CodeGen/ptrauth-in-c-struct.c
+++ b/clang/test/CodeGen/ptrauth-in-c-struct.c
@@ -115,7 +115,7 @@ void test_move_assignment_SA(SA *p) {
*p = getSA();
}
-// CHECK: define void @test_parameter_SA(ptr noundef %{{.*}})
+// CHECK: define void @test_parameter_SA(ptr dead_on_return noundef %{{.*}})
// CHECK-NOT: call
// CHECK: ret void
@@ -128,7 +128,7 @@ void test_parameter_SA(SA a) {
// CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
// CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
// CHECK: call void @__copy_constructor_8_8_t0w4_pa1_50_8(ptr %[[AGG_TMP]], ptr %[[V0]])
-// CHECK: call void @calleeSA(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @calleeSA(ptr dead_on_return noundef %[[AGG_TMP]])
// CHECK-NOT: call
// CHECK: ret void
diff --git a/clang/test/CodeGen/regcall.c b/clang/test/CodeGen/regcall.c
index f10da87353fa1..d4b9f00d54d41 100644
--- a/clang/test/CodeGen/regcall.c
+++ b/clang/test/CodeGen/regcall.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
void __regcall v4(int a, struct Large b, int c) {}
// Win32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
// Lin32: define dso_local x86_regcallcc void @__regcall3__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
// Lin64: define dso_local x86_regcallcc void @__regcall3__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
// indirectly. Additional vector arguments can consume the rest of the SSE
// registers.
void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
// X64: define dso_local x86_regcallcc void @__regcall3__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
// Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
// handling to ensure alignment.
void __regcall hfa4(struct HFA5 a) {}
// X32: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall3__hfa4(ptr dead_on_return noundef %a)
// Lin64: define dso_local x86_regcallcc void @__regcall3__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
// Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
// X64: define dso_local x86_regcallcc void @__regcall3__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
// X64: define dso_local x86_regcallcc void @__regcall3__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
struct HFA6 { __m128 f[4]; };
struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
// Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
// Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall3__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/regcall2.c b/clang/test/CodeGen/regcall2.c
index c88d4e485b104..42619369677b2 100644
--- a/clang/test/CodeGen/regcall2.c
+++ b/clang/test/CodeGen/regcall2.c
@@ -20,7 +20,7 @@ double __regcall bar(__sVector a) {
// FIXME: Do we need to change for Windows?
// Win: define dso_local x86_regcallcc void @__regcall3__foo(ptr dead_on_unwind noalias writable sret(%struct.__sVector) align 64 %agg.result, i32 noundef %a) #0
-// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr noundef %a) #0
+// Win: define dso_local x86_regcallcc double @__regcall3__bar(ptr dead_on_return noundef %a) #0
// Win: attributes #0 = { noinline nounwind optnone "min-legal-vector-width"="0" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+avx,+avx2,+avx512f,+avx512vl,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" }
// Lin: define dso_local x86_regcallcc %struct.__sVector @__regcall3__foo(i32 noundef %a) #0
diff --git a/clang/test/CodeGen/regcall4.c b/clang/test/CodeGen/regcall4.c
index 5fbe77fbc7d76..d5fe5d88a0e8c 100644
--- a/clang/test/CodeGen/regcall4.c
+++ b/clang/test/CodeGen/regcall4.c
@@ -28,7 +28,7 @@ struct Large { int a[5]; };
void __regcall v4(int a, struct Large b, int c) {}
// Win32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
// Lin32: define dso_local x86_regcallcc void @__regcall4__v4(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 noundef %c)
-// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// Win64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
// Lin64: define dso_local x86_regcallcc void @__regcall4__v4(i32 noundef %a, [5 x i32] %b.coerce, i32 noundef %c)
void __regcall v5(long long a, int b, int c) {}
@@ -47,7 +47,7 @@ void __regcall hfa1(int a, struct HFA4 b, int c) {}
// indirectly. Additional vector arguments can consume the rest of the SSE
// registers.
void __regcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hfa2(double %a.0, double %a.1, double %a.2, double %a.3, double %b.0, double %b.1, double %b.2, double %b.3, ptr dead_on_return inreg noundef %0)
// X64: define dso_local x86_regcallcc void @__regcall4__hfa2(double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double %{{.*}}, double noundef %c)
// Ensure that we pass builtin types directly while counting them against the
@@ -61,7 +61,7 @@ void __regcall hfa3(double a, double b, double c, double d, double e, struct HFA
// handling to ensure alignment.
void __regcall hfa4(struct HFA5 a) {}
// X32: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef byval(%struct.HFA5) align 4 %{{.*}})
-// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr noundef %a)
+// Win64: define dso_local x86_regcallcc void @__regcall4__hfa4(ptr dead_on_return noundef %a)
// Lin64: define dso_local x86_regcallcc void @__regcall4__hfa4(double %a.coerce0, double %a.coerce1, double %a.coerce2, double %a.coerce3, double %a.coerce4)
// Return HFAs of 4 or fewer elements in registers.
@@ -79,7 +79,7 @@ void __regcall hva1(int a, struct HVA4 b, int c) {}
// X64: define dso_local x86_regcallcc void @__regcall4__hva1(i32 noundef %a, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, i32 noundef %c)
void __regcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {}
-// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %0)
+// X86: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %0)
// X64: define dso_local x86_regcallcc void @__regcall4__hva2(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> noundef %c)
void __regcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {}
@@ -95,6 +95,6 @@ void __regcall odd_size_hva(struct OddSizeHVA a) {}
struct HFA6 { __m128 f[4]; };
struct HFA6 __regcall ret_reg_reused(struct HFA6 a, struct HFA6 b, struct HFA6 c, struct HFA6 d){ struct HFA6 h; return h;}
-// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr inreg noundef %c, ptr inreg noundef %d)
+// X86: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, ptr dead_on_return inreg noundef %c, ptr dead_on_return inreg noundef %d)
// Win64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused(<4 x float> %a.0, <4 x float> %a.1, <4 x float> %a.2, <4 x float> %a.3, <4 x float> %b.0, <4 x float> %b.1, <4 x float> %b.2, <4 x float> %b.3, <4 x float> %c.0, <4 x float> %c.1, <4 x float> %c.2, <4 x float> %c.3, <4 x float> %d.0, <4 x float> %d.1, <4 x float> %d.2, <4 x float> %d.3)
// Lin64: define dso_local x86_regcallcc %struct.HFA6 @__regcall4__ret_reg_reused([4 x <4 x float>] %a.coerce, [4 x <4 x float>] %b.coerce, [4 x <4 x float>] %c.coerce, [4 x <4 x float>] %d.coerce)
diff --git a/clang/test/CodeGen/sparcv9-abi.c b/clang/test/CodeGen/sparcv9-abi.c
index 616e24e7c519d..5a3d64fd37889 100644
--- a/clang/test/CodeGen/sparcv9-abi.c
+++ b/clang/test/CodeGen/sparcv9-abi.c
@@ -80,7 +80,7 @@ struct medium {
int *c, *d;
};
-// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr noundef %x)
+// CHECK-LABEL: define{{.*}} %struct.medium @f_medium(ptr dead_on_return noundef %x)
struct medium f_medium(struct medium x) {
x.a += *x.b;
x.b = 0;
@@ -94,7 +94,7 @@ struct large {
int x;
};
-// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @f_large(ptr dead_on_unwind noalias writable sret(%struct.large) align 8 %agg.result, ptr dead_on_return noundef %x)
struct large f_large(struct large x) {
x.a += *x.b;
x.b = 0;
diff --git a/clang/test/CodeGen/vectorcall.c b/clang/test/CodeGen/vectorcall.c
index cab7fc0972d7b..09b3310c7c4c8 100644
--- a/clang/test/CodeGen/vectorcall.c
+++ b/clang/test/CodeGen/vectorcall.c
@@ -17,7 +17,7 @@ void __vectorcall v3(int a, struct Small b, int c) {}
struct Large { int a[5]; };
void __vectorcall v4(int a, struct Large b, int c) {}
// X86: define dso_local x86_vectorcallcc void @"\01v4@@28"(i32 inreg noundef %a, ptr noundef byval(%struct.Large) align 4 %b, i32 inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr noundef %b, i32 noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01v4@@40"(i32 noundef %a, ptr dead_on_return noundef %b, i32 noundef %c)
void __vectorcall v5(long long a, int b, int c) {}
// X86: define dso_local x86_vectorcallcc void @"\01v5@@16"(i64 noundef %a, i32 inreg noundef %b, i32 inreg noundef %c)
@@ -35,21 +35,21 @@ void __vectorcall hfa1(int a, struct HFA4 b, int c) {}
// indirectly. Additional vector arguments can consume the rest of the SSE
// registers.
void __vectorcall hfa2(struct HFA4 a, struct HFA4 b, double c) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr inreg noundef %b, double inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr noundef %b, double noundef %c)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, double inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa2@@72"(%struct.HFA4 inreg %a.coerce, ptr dead_on_return noundef %b, double noundef %c)
// Ensure that we pass builtin types directly while counting them against the
// SSE register usage.
void __vectorcall hfa3(double a, double b, double c, double d, double e, struct HFA2 f) {}
-// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double inreg noundef %a, double inreg noundef %b, double inreg noundef %c, double inreg noundef %d, double inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa3@@56"(double noundef %a, double noundef %b, double noundef %c, double noundef %d, double noundef %e, ptr dead_on_return noundef %f)
// Aggregates with more than four elements are not HFAs and are passed byval.
// Because they are not classified as homogeneous, they don't get special
// handling to ensure alignment.
void __vectorcall hfa4(struct HFA5 a) {}
// X86: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef byval(%struct.HFA5) align 4 %0)
-// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr noundef %a)
+// X64: define dso_local x86_vectorcallcc void @"\01hfa4@@40"(ptr dead_on_return noundef %a)
// Return HFAs of 4 or fewer elements in registers.
static struct HFA2 g_hfa2;
@@ -68,26 +68,26 @@ v4f32 __vectorcall hva1(int a, struct HVA4 b, int c) {return b.w;}
// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva1@@80"(i32 noundef %a, %struct.HVA4 inreg %b.coerce, i32 noundef %c)
v4f32 __vectorcall hva2(struct HVA4 a, struct HVA4 b, v4f32 c) {return c;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva2@@144"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
v4f32 __vectorcall hva3(v4f32 a, v4f32 b, v4f32 c, v4f32 d, v4f32 e, struct HVA2 f) {return f.x;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr inreg noundef %f)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr noundef %f)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> inreg noundef %a, <4 x float> inreg noundef %b, <4 x float> inreg noundef %c, <4 x float> inreg noundef %d, <4 x float> inreg noundef %e, ptr dead_on_return inreg noundef %f)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva3@@112"(<4 x float> noundef %a, <4 x float> noundef %b, <4 x float> noundef %c, <4 x float> noundef %d, <4 x float> noundef %e, ptr dead_on_return noundef %f)
// vector types have higher priority then HVA structures, So vector types are allocated first
// and HVAs are allocated if enough registers are available
v4f32 __vectorcall hva4(struct HVA4 a, struct HVA2 b, v4f32 c) {return b.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva4@@112"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c)
v4f32 __vectorcall hva5(struct HVA3 a, struct HVA3 b, v4f32 c, struct HVA2 d) {return d.y;}
-// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
-// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
+// X86: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return inreg noundef %b, <4 x float> inreg noundef %c, %struct.HVA2 inreg %d.coerce)
+// X64: define dso_local x86_vectorcallcc <4 x float> @"\01hva5@@144"(%struct.HVA3 inreg %a.coerce, ptr dead_on_return noundef %b, <4 x float> noundef %c, %struct.HVA2 inreg %d.coerce)
struct HVA4 __vectorcall hva6(struct HVA4 a, struct HVA4 b) { return b;}
-// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr inreg noundef %b)
-// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr noundef %b)
+// X86: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return inreg noundef %b)
+// X64: define dso_local x86_vectorcallcc %struct.HVA4 @"\01hva6@@128"(%struct.HVA4 inreg %a.coerce, ptr dead_on_return noundef %b)
struct HVA5 __vectorcall hva7(void) {struct HVA5 a = {}; return a;}
// X86: define dso_local x86_vectorcallcc void @"\01hva7@@0"(ptr dead_on_unwind noalias writable sret(%struct.HVA5) align 16 %agg.result)
@@ -108,8 +108,8 @@ void __vectorcall odd_size_hva(struct OddSizeHVA a) {}
// consider 'p7' as a register. Instead p5 gets put into the register on the second pass.
// x86 should pass p2, p6 and p7 in registers, then p1 in the second pass.
struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3, int p4, struct HFA2 p5, float p6, float p7, int p8){ return p1;}
-// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr inreg noundef %p3, i32 inreg noundef %p4, ptr noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
-// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
+// X86: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@84"(%struct.HFA2 inreg %p1.coerce, float inreg noundef %p2, ptr dead_on_return inreg noundef %p3, i32 inreg noundef %p4, ptr dead_on_return noundef %p5, float inreg noundef %p6, float inreg noundef %p7, i32 noundef %p8)
+// X64: define dso_local x86_vectorcallcc %struct.HFA2 @"\01AddParticles@@104"(%struct.HFA2 inreg %p1.coerce, float noundef %p2, ptr dead_on_return noundef %p3, i32 noundef %p4, %struct.HFA2 inreg %p5.coerce, float noundef %p6, float noundef %p7, i32 noundef %p8)
// Vectorcall in both architectures allows passing of an HVA as long as there is room,
// even if it is not one of the first 6 arguments. First pass puts p4 into a
@@ -117,8 +117,8 @@ struct HFA2 __vectorcall AddParticles(struct HFA2 p1, float p2, struct HFA4 p3,
// in a register, does NOT put p7 in a register (since there's no room), then puts
// p8 in a register.
void __vectorcall HVAAnywhere(struct HFA2 p1, int p2, int p3, float p4, int p5, int p6, struct HFA4 p7, struct HFA2 p8, float p9){}
-// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
-// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
+// X86: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@88"(%struct.HFA2 inreg %p1.coerce, i32 inreg noundef %p2, i32 inreg noundef %p3, float inreg noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float inreg noundef %p9)
+// X64: define dso_local x86_vectorcallcc void @"\01HVAAnywhere@@112"(%struct.HFA2 inreg %p1.coerce, i32 noundef %p2, i32 noundef %p3, float noundef %p4, i32 noundef %p5, i32 noundef %p6, ptr dead_on_return noundef %p7, %struct.HFA2 inreg %p8.coerce, float noundef %p9)
#ifndef __x86_64__
// This covers the three ways XMM values can be passed on 32-bit x86:
@@ -137,9 +137,9 @@ void __vectorcall vectorcall_indirect_vec(
// X86-SAME: double inreg noundef %xmm3,
// X86-SAME: double inreg noundef %xmm4,
// X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
// X86-SAME: i32 inreg noundef %edx,
-// X86-SAME: ptr noundef %1)
+// X86-SAME: ptr dead_on_return noundef %1)
void __vectorcall vectorcall_indirect_fp(
double xmm0, double xmm1, double xmm2, double xmm3, double xmm4,
@@ -153,7 +153,7 @@ void __vectorcall vectorcall_indirect_fp(
// X86-SAME: double inreg noundef %xmm3,
// X86-SAME: double inreg noundef %xmm4,
// X86-SAME: <4 x float> inreg noundef %xmm5,
-// X86-SAME: ptr inreg noundef %0,
+// X86-SAME: ptr dead_on_return inreg noundef %0,
// X86-SAME: i32 inreg noundef %edx,
// X86-SAME: double noundef %mem)
#endif
diff --git a/clang/test/CodeGen/win-fp128.c b/clang/test/CodeGen/win-fp128.c
index 328a7aaa7df57..8d223741bc93e 100644
--- a/clang/test/CodeGen/win-fp128.c
+++ b/clang/test/CodeGen/win-fp128.c
@@ -6,7 +6,7 @@ __float128 fp128_ret(void) { return 0; }
// CHECK-GNU64: define dso_local <2 x i64> @fp128_ret()
__float128 fp128_args(__float128 a, __float128 b) { return a * b; }
-// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr noundef %0, ptr noundef %1)
+// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
void fp128_vararg(int a, ...) {
// CHECK-GNU64-LABEL: define dso_local void @fp128_vararg
diff --git a/clang/test/CodeGen/win64-i128.c b/clang/test/CodeGen/win64-i128.c
index e10b2be0530eb..2d83889d8f89b 100644
--- a/clang/test/CodeGen/win64-i128.c
+++ b/clang/test/CodeGen/win64-i128.c
@@ -12,8 +12,8 @@ int128_t foo(void) { return 0; }
int128_t bar(int128_t a, int128_t b) { return a * b; }
-// GNU64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
-// MSC64: define dso_local <2 x i64> @bar(ptr noundef %0, ptr noundef %1)
+// GNU64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
+// MSC64: define dso_local <2 x i64> @bar(ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
void vararg(int a, ...) {
// GNU64-LABEL: define{{.*}} void @vararg
diff --git a/clang/test/CodeGen/windows-swiftcall.c b/clang/test/CodeGen/windows-swiftcall.c
index 41569c2606622..8716f25b9ddfb 100644
--- a/clang/test/CodeGen/windows-swiftcall.c
+++ b/clang/test/CodeGen/windows-swiftcall.c
@@ -219,7 +219,7 @@ TEST(struct_big_1)
// CHECK-LABEL: define {{.*}} void @return_struct_big_1({{.*}} dead_on_unwind noalias writable sret
// Should not be byval.
-// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr noundef{{( %.*)?}})
+// CHECK-LABEL: define {{.*}} void @take_struct_big_1(ptr dead_on_return noundef{{( %.*)?}})
/*****************************************************************************/
/********************************* TYPE MERGING ******************************/
diff --git a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
index 152be26948f28..1709c88563267 100644
--- a/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
+++ b/clang/test/CodeGenCXX/aarch64-mangle-sve-vectors.cpp
@@ -576,7 +576,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
// CHECK-NEXT: [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
// CHECK-NEXT: [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
// CHECK-NEXT: store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// CHECK-NEXT: call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// CHECK-NEXT: call void @_Z1f10svboolx4_tS_(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
// CHECK-NEXT: store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
// CHECK-NEXT: [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
// CHECK-NEXT: [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
@@ -1125,7 +1125,7 @@ void f(__clang_svmfloat8x4_t, __clang_svmfloat8x4_t);
// COMPAT_17-NEXT: [[COERCE74_EXTRACT2:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 2
// COMPAT_17-NEXT: [[COERCE74_EXTRACT3:%.*]] = extractvalue { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } [[COERCE74_TUPLE]], 3
// COMPAT_17-NEXT: store { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } zeroinitializer, ptr [[BYVAL_TEMP]], align 2
-// COMPAT_17-NEXT: call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr noundef [[BYVAL_TEMP]])
+// COMPAT_17-NEXT: call void @_Z1f10svboolx4_t10svboolx4_t(<vscale x 16 x i1> [[COERCE74_EXTRACT0]], <vscale x 16 x i1> [[COERCE74_EXTRACT1]], <vscale x 16 x i1> [[COERCE74_EXTRACT2]], <vscale x 16 x i1> [[COERCE74_EXTRACT3]], ptr dead_on_return noundef [[BYVAL_TEMP]])
// COMPAT_17-NEXT: store { <vscale x 16 x i8>, <vscale x 16 x i8> } zeroinitializer, ptr [[COERCE75]], align 16
// COMPAT_17-NEXT: [[COERCE75_TUPLE:%.*]] = load { <vscale x 16 x i8>, <vscale x 16 x i8> }, ptr [[COERCE75]], align 16
// COMPAT_17-NEXT: [[COERCE75_EXTRACT0:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[COERCE75_TUPLE]], 0
diff --git a/clang/test/CodeGenCXX/arm-cc.cpp b/clang/test/CodeGenCXX/arm-cc.cpp
index 68e1b7e4e1e46..939615fcc69e0 100644
--- a/clang/test/CodeGenCXX/arm-cc.cpp
+++ b/clang/test/CodeGenCXX/arm-cc.cpp
@@ -17,4 +17,4 @@ void baz() {
}
// CHECK: declare void @_Z3fooPv(ptr dead_on_unwind writable sret(%class.SMLoc) align 4, ptr noundef)
-// CHECK: declare void @_Z3zed5SMLoc(ptr noundef)
+// CHECK: declare void @_Z3zed5SMLoc(ptr dead_on_return noundef)
diff --git a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
index c341bd2b855ff..f2d602b3b523e 100644
--- a/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
+++ b/clang/test/CodeGenCXX/attr-target-mv-inalloca.cpp
@@ -55,18 +55,18 @@ void usage() {
// WINDOWS-NEXT: ret i32 %[[RET]]
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
// WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
// WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
// WINDOWS64: ret i32 %[[LOAD]]
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.sse4.2"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
// WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
// WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
// WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 1
// WINDOWS64: ret i32 %[[ADD]]
-// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr noundef %[[O:[0-9a-zA-Z]+]])
+// WINDOWS64: define dso_local noundef i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr dead_on_return noundef %[[O:[0-9a-zA-Z]+]])
// WINDOWS64: %[[X:[0-9a-zA-Z]+]] = getelementptr inbounds nuw %struct.Foo, ptr %[[O]], i32 0, i32 0
// WINDOWS64: %[[LOAD:[0-9a-zA-Z]+]] = load i32, ptr %[[X]]
// WINDOWS64: %[[ADD:[0-9a-zA-Z]+]] = add nsw i32 %[[LOAD]], 2
@@ -75,7 +75,7 @@ void usage() {
// WINDOWS64: define dso_local void @"?usage@@YAXXZ"()
// WINDOWS64: %[[F:[0-9a-zA-Z]+]] = alloca %struct.Foo
// WINDOWS64: %[[ARG:[0-9a-zA-Z.]+]] = alloca %struct.Foo
-// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr noundef %[[ARG]])
+// WINDOWS64: %[[CALL:[0-9a-zA-Z]+]] = call noundef i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr dead_on_return noundef %[[ARG]])
// WINDOWS64: define weak_odr dso_local i32 @"?bar@@YAHUFoo@@@Z.resolver"(ptr %0)
// WINDOWS64: %[[RET:[0-9a-zA-Z]+]] = musttail call i32 @"?bar@@YAHUFoo@@@Z.arch_ivybridge"(ptr %0)
diff --git a/clang/test/CodeGenCXX/copy-initialization.cpp b/clang/test/CodeGenCXX/copy-initialization.cpp
index aa0c6395f158d..4e6194cc040c2 100644
--- a/clang/test/CodeGenCXX/copy-initialization.cpp
+++ b/clang/test/CodeGenCXX/copy-initialization.cpp
@@ -12,7 +12,7 @@ struct Bar {
void f(Foo);
-// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr noundef %foo)
+// CHECK-LABEL: define{{.*}} void @_Z1g3Foo(ptr dead_on_return noundef %foo)
void g(Foo foo) {
// CHECK: call void @_ZN3BarC1Ev
// CHECK: @_ZNK3BarcvRK3FooEv
diff --git a/clang/test/CodeGenCXX/debug-info.cpp b/clang/test/CodeGenCXX/debug-info.cpp
index 8594a897ef7c0..9cf26ba83ba3e 100644
--- a/clang/test/CodeGenCXX/debug-info.cpp
+++ b/clang/test/CodeGenCXX/debug-info.cpp
@@ -4,7 +4,7 @@
// CHECK: @_ZN6pr96081xE ={{.*}} global ptr null, align 8, !dbg [[X:![0-9]+]]
// CHECK: define{{.*}} void @_ZN7pr147634funcENS_3fooE
-// CHECK-SAME: ptr noundef [[param:%.*]])
+// CHECK-SAME: ptr dead_on_return noundef [[param:%.*]])
// CHECK-NEXT: entry:
// CHECK-NEXT: alloca ptr, align 8
// CHECK-NEXT: [[param_addr_storage:%.*]] = alloca ptr, align 8
diff --git a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
index c7d3a017414ef..c8f5a0f7c2ea6 100644
--- a/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
+++ b/clang/test/CodeGenCXX/empty-nontrivially-copyable.cpp
@@ -13,7 +13,7 @@ struct Empty {
};
bool foo(Empty e) {
-// CHECK: @_Z3foo5Empty(ptr noundef %e)
+// CHECK: @_Z3foo5Empty(ptr dead_on_return noundef %e)
// CHECK: call {{.*}} @_ZN5Empty5checkEv(ptr {{[^,]*}} %e)
return e.check();
}
@@ -21,6 +21,6 @@ bool foo(Empty e) {
void caller(Empty &e) {
// CHECK: @_Z6callerR5Empty(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %e)
// CHECK: call {{.*}} @_ZN5EmptyC1ERKS_(ptr {{[^,]*}} [[NEWTMP:%.*]], ptr
-// CHECK: call {{.*}} @_Z3foo5Empty(ptr noundef [[NEWTMP]])
+// CHECK: call {{.*}} @_Z3foo5Empty(ptr dead_on_return noundef [[NEWTMP]])
foo(e);
}
diff --git a/clang/test/CodeGenCXX/fastcall.cpp b/clang/test/CodeGenCXX/fastcall.cpp
index 4c94c1623ee16..405917f7a14bb 100644
--- a/clang/test/CodeGenCXX/fastcall.cpp
+++ b/clang/test/CodeGenCXX/fastcall.cpp
@@ -15,6 +15,6 @@ struct S1 {
void __attribute__((fastcall)) foo2(S1 a, int b);
void bar2(S1 a, int b) {
// CHECK-LABEL: define{{.*}} void @_Z4bar22S1i
- // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr inreg %{{.*}}, i32 inreg %
+ // CHECK: call x86_fastcallcc void @_Z4foo22S1i(ptr dead_on_return inreg %{{.*}}, i32 inreg %
foo2(a, b);
}
diff --git a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
index 63ffc6b5bfac8..5ebeb8aad4c18 100644
--- a/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
+++ b/clang/test/CodeGenCXX/homogeneous-aggregates.cpp
@@ -41,8 +41,8 @@ struct D5 : I1, I2, I3 {}; // homogeneous aggregate
// PPC: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
// ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, [3 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
-// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D12D1(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
+// X64: define dso_local x86_vectorcallcc void @"\01_Z7func_D12D1@@24"(ptr dead_on_unwind noalias writable sret(%struct.D1) align 8 %agg.result, ptr dead_on_return noundef %x)
D1 CC func_D1(D1 x) { return x; }
// PPC: define{{.*}} [3 x double] @_Z7func_D22D2([3 x double] %x.coerce)
@@ -53,7 +53,7 @@ D2 CC func_D2(D2 x) { return x; }
// PPC: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
// ARM32: define{{.*}} arm_aapcs_vfpcc void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, [4 x i64] %x.coerce)
-// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr noundef %x)
+// ARM64: define{{.*}} void @_Z7func_D32D3(ptr dead_on_unwind noalias writable sret(%struct.D3) align 8 %agg.result, ptr dead_on_return noundef %x)
D3 CC func_D3(D3 x) { return x; }
// PPC: define{{.*}} [4 x double] @_Z7func_D42D4([4 x double] %x.coerce)
@@ -201,7 +201,7 @@ struct NonHFA {
virtual void f1();
};
double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at polymorphic@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
}
namespace trivial_copy_assignment {
struct HFA {
@@ -221,7 +221,7 @@ struct NonHFA {
NonHFA &operator=(const NonHFA&);
};
double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_copy_assignment@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
}
namespace user_provided_ctor {
struct HFA {
@@ -251,7 +251,7 @@ struct NonHFA {
~NonHFA();
};
double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at non_trivial_dtor@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
}
namespace non_empty_base {
struct non_empty_base { double d; };
@@ -272,7 +272,7 @@ struct NonHFA {
empty e;
};
double foo(NonHFA v) { return v.x + v.y; }
-// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr noundef %{{.*}})
+// WOA64: define dso_local noundef double @"?foo at empty_field@@YANUNonHFA at 1@@Z"(ptr dead_on_return noundef %{{.*}})
}
namespace non_empty_field {
struct non_empty { double d; };
diff --git a/clang/test/CodeGenCXX/inalloca-lambda.cpp b/clang/test/CodeGenCXX/inalloca-lambda.cpp
index dc78aa2773f89..0d527e1a02e5f 100644
--- a/clang/test/CodeGenCXX/inalloca-lambda.cpp
+++ b/clang/test/CodeGenCXX/inalloca-lambda.cpp
@@ -22,7 +22,7 @@ void test() {
// CHECK: %[[V:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
// CHECK: %call = call x86_thiscallcc noundef i32
// CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[V]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[V]])
// CHECK: define internal noundef i32
// CHECK-SAME: @"?__invoke@<lambda_0>@?0??test@@YAXXZ at CA?A?<auto>@@UA@@@Z"
@@ -31,12 +31,12 @@ void test() {
// CHECK: %[[VAR:.*]] = getelementptr inbounds nuw <{ %struct.A }>, ptr %[[ARG]], i32 0, i32 0
// CHECK: %call = call x86_thiscallcc noundef i32
// CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %unused.capture, ptr noundef %[[VAR]])
+// CHECK-SAME: (ptr noundef %unused.capture, ptr dead_on_return noundef %[[VAR]])
// CHECK: ret i32 %call
// CHECK: define internal x86_thiscallcc noundef i32
// CHECK-SAME: @"?__impl@<lambda_0>@?0??test@@YAXXZ at QBE?A?<auto>@@UA@@@Z"
-// CHECK-SAME: (ptr noundef %this, ptr noundef %[[ARG:.*]])
+// CHECK-SAME: (ptr noundef %this, ptr dead_on_return noundef %[[ARG:.*]])
// CHECK: %this.addr = alloca ptr, align 4
// CHECK: store ptr %this, ptr %this.addr, align 4
// CHECK: %this1 = load ptr, ptr %this.addr, align 4
diff --git a/clang/test/CodeGenCXX/inalloca-overaligned.cpp b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
index 3751751ec0bca..305b8c5dccd10 100644
--- a/clang/test/CodeGenCXX/inalloca-overaligned.cpp
+++ b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
@@ -57,7 +57,7 @@ int receive_both(Both o) {
}
// CHECK-LABEL: define dso_local noundef i32 @"?receive_both@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
int pass_both() {
gvi32 = receive_both(Both());
@@ -67,7 +67,7 @@ int pass_both() {
// CHECK-LABEL: define dso_local noundef i32 @"?pass_both@@Y{{.*}}"
// CHECK: [[TMP:%[^ ]*]] = alloca %struct.Both, align 8
// CHECK: call x86_thiscallcc noundef ptr @"??0Both@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receive_both@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
int receive_inalloca_both(NonTrivial nt, Both o) {
return nt.x + o.x + o.y;
@@ -101,11 +101,11 @@ struct [[trivial_abi]] alignas(8) MyPtr {
int receiveMyPtr(MyPtr o) { return *o.ptr; }
// CHECK-LABEL: define dso_local noundef i32 @"?receiveMyPtr@@Y{{.*}}"
-// CHECK-SAME: (ptr noundef %o)
+// CHECK-SAME: (ptr dead_on_return noundef %o)
int passMyPtr() { return receiveMyPtr(MyPtr()); }
// CHECK-LABEL: define dso_local noundef i32 @"?passMyPtr@@Y{{.*}}"
// CHECK: [[TMP:%[^ ]*]] = alloca %struct.MyPtr, align 8
// CHECK: call x86_thiscallcc noundef ptr @"??0MyPtr@@QAE at XZ"(ptr {{[^,]*}} [[TMP]])
-// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr noundef [[TMP]])
+// CHECK: call noundef i32 @"?receiveMyPtr@@Y{{.*}}"(ptr dead_on_return noundef [[TMP]])
diff --git a/clang/test/CodeGenCXX/inalloca-vector.cpp b/clang/test/CodeGenCXX/inalloca-vector.cpp
index d1bacb4f0dc8c..2db4c49df116a 100644
--- a/clang/test/CodeGenCXX/inalloca-vector.cpp
+++ b/clang/test/CodeGenCXX/inalloca-vector.cpp
@@ -56,7 +56,7 @@ void __fastcall fastcall_receive_vec(__m128 x, __m128 y, __m128 z, __m128 w, int
// CHECK-SAME: (<4 x float> inreg noundef %x,
// CHECK-SAME: <4 x float> inreg noundef %y,
// CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
// CHECK-SAME: i32 inreg noundef %edx,
// CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
@@ -73,6 +73,6 @@ void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
// CHECK-SAME: <4 x float> inreg noundef %x,
// CHECK-SAME: <4 x float> inreg noundef %y,
// CHECK-SAME: <4 x float> inreg noundef %z,
-// CHECK-SAME: ptr inreg noundef %0,
+// CHECK-SAME: ptr dead_on_return inreg noundef %0,
// CHECK-SAME: i32 inreg noundef %edx,
// CHECK-SAME: ptr inalloca(<{ ptr, %struct.NonTrivial }>) %1)
diff --git a/clang/test/CodeGenCXX/inheriting-constructor.cpp b/clang/test/CodeGenCXX/inheriting-constructor.cpp
index 100ca269d7f3c..21751bea055dc 100644
--- a/clang/test/CodeGenCXX/inheriting-constructor.cpp
+++ b/clang/test/CodeGenCXX/inheriting-constructor.cpp
@@ -166,7 +166,7 @@ namespace inalloca_nonvirt {
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
- // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+ // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
// WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
@@ -202,7 +202,7 @@ namespace inalloca_nonvirt {
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
- // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+ // WIN64: call {{.*}} @"??0A at inalloca_nonvirt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
// WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
}
@@ -253,7 +253,7 @@ namespace inalloca_virt {
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG3]], i32 3)
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
// WIN64: br i1
- // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+ // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
// WIN64: br
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
@@ -302,7 +302,7 @@ namespace inalloca_virt {
// WIN64: call {{.*}} @"??0Q@@QEAA at H@Z"(ptr {{[^,]*}} %[[ARG1]], i32 1)
// WIN64: br i1
// WIN64: store {{.*}} @"??_8C at inalloca_virt@@7B@"
- // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr %[[ARG1]], i32 2, ptr %[[ARG3]], ptr{{.*}} %[[TMP]])
+ // WIN64: call {{.*}} @"??0A at inalloca_virt@@QEAA at UQ@@H0$$QEAU2@@Z"(ptr{{.*}}, ptr dead_on_return %[[ARG1]], i32 2, ptr dead_on_return %[[ARG3]], ptr{{.*}} %[[TMP]])
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
// WIN64: call {{.*}} @"??0Z@@QEAA at XZ"(
// WIN64: call void @"??1Q@@QEAA at XZ"(ptr {{[^,]*}} %[[TMP]])
diff --git a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
index ff511c0243801..f06cda8b7684e 100644
--- a/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
+++ b/clang/test/CodeGenCXX/member-function-pointer-calls.cpp
@@ -16,7 +16,7 @@ int f(A* a, int (A::*fp)()) {
// CHECK-NOT: }
// CHECK: ret i32 1
// MINGW64-LABEL: define dso_local noundef i32 @_Z2g1v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
int g1() {
A a;
return f(&a, &A::vf1);
@@ -26,7 +26,7 @@ int g1() {
// CHECK-NOT: }
// CHECK: ret i32 2
// MINGW64-LABEL: define dso_local noundef i32 @_Z2g2v()
-// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr noundef %{{.*}})
+// MINGW64: call noundef i32 @_Z1fP1AMS_FivE(ptr noundef %{{.*}}, ptr dead_on_return noundef %{{.*}})
int g2() {
A a;
return f(&a, &A::vf2);
diff --git a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
index b551df747c073..63a4d5525336b 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-arg-order.cpp
@@ -24,7 +24,7 @@ void foo(A a, A b, A c) {
// X86: ret void
// X64-LABEL: define dso_local void @"?foo@@YAXUA@@00 at Z"
-// X64: (ptr noundef %[[a:[^,]*]], ptr noundef %[[b:[^,]*]], ptr noundef %[[c:[^)]*]])
+// X64: (ptr dead_on_return noundef %[[a:[^,]*]], ptr dead_on_return noundef %[[b:[^,]*]], ptr dead_on_return noundef %[[c:[^)]*]])
// X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[a]])
// X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[b]])
// X64: call void @"??1A@@QEAA at XZ"(ptr {{[^,]*}} %[[c]])
@@ -64,7 +64,7 @@ void call_foo() {
// X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg2:[^,]*]], i32 noundef 2)
// X64: invoke noundef ptr @"??0A@@QEAA at H@Z"(ptr {{[^,]*}} %[[arg1:[^,]*]], i32 noundef 1)
// X64: call void @"?foo@@YAXUA@@00 at Z"
-// X64: (ptr noundef %[[arg1]], ptr noundef %[[arg2]], ptr noundef %[[arg3]])
+// X64: (ptr dead_on_return noundef %[[arg1]], ptr dead_on_return noundef %[[arg2]], ptr dead_on_return noundef %[[arg3]])
// X64: ret void
//
// lpad2:
diff --git a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
index 5654db3ba8151..813abb03a7810 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-byval-thunks.cpp
@@ -22,10 +22,10 @@ C::C() {} // force emission
// CHECK32-NEXT: ret void
// CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@byval_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64: (ptr noundef %this, ptr noundef %x)
+// CHECK64: (ptr noundef %this, ptr dead_on_return noundef %x)
// CHECK64: getelementptr i8, ptr %{{.*}}, i32 -8
// CHECK64: call void @"?foo at C@byval_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64: (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64: (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
// CHECK64-NOT: call
// CHECK64: ret void
}
@@ -54,10 +54,10 @@ C::C() {} // force emission
// CHECK32-NEXT: ret void
// CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@stdcall_thunk@@W7EAAXUAgg at 2@@Z"
-// CHECK64: (ptr noundef %this, ptr noundef %x)
+// CHECK64: (ptr noundef %this, ptr dead_on_return noundef %x)
// CHECK64: getelementptr i8, ptr %{{.*}}, i32 -8
// CHECK64: call void @"?foo at C@stdcall_thunk@@UEAAXUAgg at 2@@Z"
-// CHECK64: (ptr {{[^,]*}} %{{.*}}, ptr noundef %x)
+// CHECK64: (ptr {{[^,]*}} %{{.*}}, ptr dead_on_return noundef %x)
// CHECK64-NOT: call
// CHECK64: ret void
}
@@ -86,10 +86,10 @@ C::C() {} // force emission
// CHECK32-NEXT: ret ptr %[[rv]]
// CHECK64-LABEL: define linkonce_odr dso_local void @"?foo at C@sret_thunk@@W7EAA?AUAgg at 2@U32@@Z"
-// CHECK64: (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64: (ptr noundef %this, ptr dead_on_unwind noalias writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
// CHECK64: getelementptr i8, ptr %{{.*}}, i32 -8
// CHECK64: call void @"?foo at C@sret_thunk@@UEAA?AUAgg at 2@U32@@Z"
-// CHECK64: (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr noundef %x)
+// CHECK64: (ptr {{[^,]*}} %{{.*}}, ptr dead_on_unwind writable sret(%"struct.sret_thunk::Agg") align 4 %agg.result, ptr dead_on_return noundef %x)
// CHECK64-NOT: call
// CHECK64: ret void
}
diff --git a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
index 806bc5b63ef02..8defb68c668b2 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
@@ -428,7 +428,7 @@ bool nullTestDataUnspecified(int Unspecified::*mp) {
// Pass this large type indirectly.
// X64-LABEL: define dso_local noundef zeroext i1 @"?nullTestDataUnspecified@@
-// X64: (ptr noundef %0)
+// X64: (ptr dead_on_return noundef %0)
}
bool nullTestFunctionUnspecified(void (Unspecified::*mp)()) {
@@ -590,7 +590,7 @@ bool unspecFuncMemptrEq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
// CHECK: }
// X64-LABEL: define dso_local noundef zeroext i1 @"?unspecFuncMemptrEq@@
-// X64: (ptr noundef %0, ptr noundef %1)
+// X64: (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
}
bool unspecFuncMemptrNeq(void (Unspecified::*l)(), void (Unspecified::*r)()) {
@@ -635,7 +635,7 @@ bool unspecDataMemptrEq(int Unspecified::*l, int Unspecified::*r) {
// CHECK: }
// X64-LABEL: define dso_local noundef zeroext i1 @"?unspecDataMemptrEq@@
-// X64: (ptr noundef %0, ptr noundef %1)
+// X64: (ptr dead_on_return noundef %0, ptr dead_on_return noundef %1)
}
void (Multiple::*convertB2FuncToMultiple(void (B2::*mp)()))() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
index 6e8ba3953b2cf..767bf168633ae 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-sret-and-byval.cpp
@@ -175,7 +175,7 @@ void multibyte_arg(Multibyte s) {}
void packed_arg(Packed s) {}
// LINUX-LABEL: define{{.*}} void @_Z10packed_arg6Packed(ptr noundef byval(%struct.Packed) align 4 %s)
// WIN32: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef byval(%struct.Packed) align 4 %s)
-// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?packed_arg@@YAXUPacked@@@Z"(ptr dead_on_return noundef %s)
// Test that dtors are invoked in the callee.
void small_arg_with_dtor(SmallWithDtor s) {}
@@ -190,7 +190,7 @@ void small_arg_with_dtor(SmallWithDtor s) {}
// WOA64: }
// FIXME: MSVC incompatible!
-// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr noundef %s) {{.*}} {
+// WOA: define dso_local arm_aapcs_vfpcc void @"?small_arg_with_dtor@@YAXUSmallWithDtor@@@Z"(ptr dead_on_return noundef %s) {{.*}} {
// WOA: call arm_aapcs_vfpcc void @"??1SmallWithDtor@@QAA at XZ"(ptr {{[^,]*}} %s)
// WOA: }
@@ -220,7 +220,7 @@ void ref_small_arg_with_dtor(const SmallWithDtor &s) { }
// WIN64-LABEL: define dso_local void @"?ref_small_arg_with_dtor@@YAXAEBUSmallWithDtor@@@Z"(ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %s)
void big_arg_with_dtor(BigWithDtor s) {}
-// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %s)
+// WIN64-LABEL: define dso_local void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %s)
// WIN64: call void @"??1BigWithDtor@@QEAA at XZ"
// WIN64: }
@@ -231,7 +231,7 @@ void call_big_arg_with_dtor() {
// larger than 8 bytes and is passed indirectly.
// WIN64-LABEL: define dso_local void @"?call_big_arg_with_dtor@@YAXXZ"()
// WIN64: call noundef ptr @"??0BigWithDtor@@QEAA at XZ"
-// WIN64: call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr noundef %{{.*}})
+// WIN64: call void @"?big_arg_with_dtor@@YAXUBigWithDtor@@@Z"(ptr dead_on_return noundef %{{.*}})
// WIN64-NOT: call void @"??1BigWithDtor@@QEAA at XZ"
// WIN64: ret void
@@ -259,22 +259,22 @@ void eh_cleanup_arg_with_dtor() {
// WIN32: }
void small_arg_with_vftable(SmallWithVftable s) {}
-// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z22small_arg_with_vftable16SmallWithVftable(ptr dead_on_return noundef %s)
// WIN32: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr inalloca(<{ %struct.SmallWithVftable }>) %0)
-// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?small_arg_with_vftable@@YAXUSmallWithVftable@@@Z"(ptr dead_on_return noundef %s)
void medium_arg_with_copy_ctor(MediumWithCopyCtor s) {}
-// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr noundef %s)
+// LINUX-LABEL: define{{.*}} void @_Z25medium_arg_with_copy_ctor18MediumWithCopyCtor(ptr dead_on_return noundef %s)
// WIN32: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr inalloca(<{ %struct.MediumWithCopyCtor }>) %0)
-// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
-// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA: define dso_local arm_aapcs_vfpcc void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
+// WOA64: define dso_local void @"?medium_arg_with_copy_ctor@@YAXUMediumWithCopyCtor@@@Z"(ptr dead_on_return noundef %s)
void big_arg(Big s) {}
// LINUX-LABEL: define{{.*}} void @_Z7big_arg3Big(ptr noundef byval(%struct.Big) align 4 %s)
// WIN32: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef byval(%struct.Big) align 4 %s)
-// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr noundef %s)
+// WIN64: define dso_local void @"?big_arg@@YAXUBig@@@Z"(ptr dead_on_return noundef %s)
// PR27607: We would attempt to load i32 value out of the reference instead of
// just loading the pointer from the struct during argument expansion.
@@ -346,7 +346,7 @@ class Class {
void thiscall_method_arg(Big s) {}
// LINUX: define {{.*}} void @_ZN5Class19thiscall_method_argE3Big(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
// WIN32: define {{.*}} void @"?thiscall_method_arg at Class@@QAEXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef byval(%struct.Big) align 4 %s)
- // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr noundef %s)
+ // WIN64: define linkonce_odr dso_local void @"?thiscall_method_arg at Class@@QEAAXUBig@@@Z"(ptr {{[^,]*}} %this, ptr dead_on_return noundef %s)
};
void use_class() {
diff --git a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
index 9e37e71e257fd..b7653632cf882 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-unknown-arch.cpp
@@ -18,4 +18,4 @@ A B::foo(A x) {
return x;
}
-// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr noundef %x)
+// CHECK-LABEL: define{{.*}} void @"?foo at B@@QEAA?AUA@@U2@@Z"(ptr {{[^,]*}} %this, ptr dead_on_unwind noalias writable sret(%struct.A) align 4 %agg.result, ptr dead_on_return noundef %x)
diff --git a/clang/test/CodeGenCXX/ms-property.cpp b/clang/test/CodeGenCXX/ms-property.cpp
index 744de224b2f9a..d9fbf46dacb25 100644
--- a/clang/test/CodeGenCXX/ms-property.cpp
+++ b/clang/test/CodeGenCXX/ms-property.cpp
@@ -105,7 +105,7 @@ int main(int argc, char **argv) {
// CHECK: [[ARGC:%.+]] = load i32, ptr %
// CHECK: [[P1_X_ARGC_0:%.+]] = call noundef i32 @"?GetX at S@@QEAAHHH at Z"(ptr {{[^,]*}} [[P1]], i32 noundef [[ARGC]], i32 noundef 0)
// CHECK: [[CAST:%.+]] = trunc i32 [[P1_X_ARGC_0]] to i8
- // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr noundef %{{.+}})
+ // CHECK: [[P2_Y_p1_X_ARGC_0_T:%.+]] = call noundef i8 @"?GetY@?$St at M@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[P2_2]], i8 noundef [[CAST]], ptr dead_on_return noundef %{{.+}})
// CHECK: [[CAST:%.+]] = sitofp i8 [[P2_Y_p1_X_ARGC_0_T]] to float
// CHECK: [[J:%.+]] = load i32, ptr %
// CHECK: [[CAST1:%.+]] = sitofp i32 [[J]] to float
@@ -124,6 +124,6 @@ int main(int argc, char **argv) {
// CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
// CHECK: call void @"?PutY@?$St at H@@QEAAXDHN at Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, i32 noundef %{{.+}}, double noundef %{{.+}}
// CHECK: call noundef i32 @"?GetX@?$St at H@@QEAAHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}} i32 noundef %{{.+}})
-// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr noundef %{{.+}})
+// CHECK: call noundef i8 @"?GetY@?$St at H@@QEAADDVTest1@@@Z"(ptr {{[^,]*}} [[BAR]], i8 noundef %{{.+}}, ptr dead_on_return noundef %{{.+}})
// CHECK: call noundef i32 @"?PutX@?$St at H@@QEAAHHHH at Z"(ptr {{[^,]*}} [[BAR]], i32 noundef %{{.+}}, i32 noundef %{{.+}}, i32 noundef %{{.+}})
#endif //HEADER
diff --git a/clang/test/CodeGenCXX/nrvo.cpp b/clang/test/CodeGenCXX/nrvo.cpp
index 1141bc35de582..5b0fc914120e3 100644
--- a/clang/test/CodeGenCXX/nrvo.cpp
+++ b/clang/test/CodeGenCXX/nrvo.cpp
@@ -2197,7 +2197,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
// CHECK-EH-11-NEXT: br i1 [[CMP9]], label [[IF_THEN10:%.*]], label [[IF_END11:%.*]]
// CHECK-EH-11: if.then10:
// CHECK-EH-11-NEXT: store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4
-// CHECK-EH-11-NEXT: br label [[CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK-EH-11-NEXT: br label [[CLEANUP]], !llvm.loop [[LOOP4:![0-9]+]]
// CHECK-EH-11: if.end11:
// CHECK-EH-11-NEXT: [[TMP4:%.*]] = load i32, ptr [[I_ADDR]], align 4
// CHECK-EH-11-NEXT: [[CMP12:%.*]] = icmp eq i32 [[TMP4]], 3
@@ -2239,7 +2239,7 @@ void test16() { // http://wg21.link/p2025r2#ex-9
// CHECK-EH-11-NEXT: i32 2, label [[IMPOSSIBLE]]
// CHECK-EH-11-NEXT: ]
// CHECK-EH-11: cleanup.cont:
-// CHECK-EH-11-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP3]]
+// CHECK-EH-11-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP4]]
// CHECK-EH-11: while.end:
// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(ptr noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]])
// CHECK-EH-11-NEXT: br label [[RETURN]]
diff --git a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
index 773cf6b81c3b2..947379d9b8b92 100644
--- a/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
+++ b/clang/test/CodeGenCXX/pass-by-value-noalias.cpp
@@ -14,8 +14,8 @@ struct Foo {
// Make sure noalias is added to indirect arguments with trivially copyable types
// if -fpass-by-value-is-noalias is provided.
-// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take3Foo(ptr dead_on_return noundef %arg)
void take(Foo arg) {}
int G;
@@ -38,8 +38,8 @@ struct NonTrivial {
// Make sure noalias is not added to indirect arguments that are not trivially
// copyable even if -fpass-by-value-is-noalias is provided.
-// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take10NonTrivial(ptr dead_on_return noundef %arg)
void take(NonTrivial arg) {}
// Escape examples. Pointers to the objects passed to take() may escape, depending on whether a temporary copy is created or not (e.g. due to NRVO).
@@ -54,8 +54,8 @@ struct A {
};
A *p;
-// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr noalias noundef %arg)
-// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noalias noundef %arg)
+// NO_NOALIAS: define{{.*}} void @_Z4take1A(ptr dead_on_return noundef %arg)
void take(A arg) {}
// WITH_NOALIAS: define{{.*}} void @_Z7CreateAPP1A(ptr dead_on_unwind noalias writable sret(%struct.A) align 1 %agg.result, ptr noundef %where)
diff --git a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
index daeea77774ec8..0310535362e3d 100644
--- a/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
+++ b/clang/test/CodeGenCXX/ptrauth-qualifier-struct.cpp
@@ -24,7 +24,7 @@ struct __attribute__((trivial_abi)) TrivialSA {
// Check that TrivialSA is passed indirectly despite being annotated with
// 'trivial_abi'.
-// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr noundef %{{.*}})
+// CHECK: define {{.*}}void @_Z18testParamTrivialSA9TrivialSA(ptr dead_on_return noundef %{{.*}})
void testParamTrivialSA(TrivialSA a) {
}
diff --git a/clang/test/CodeGenCXX/regparm.cpp b/clang/test/CodeGenCXX/regparm.cpp
index b9735485db8de..a31394016fb55 100644
--- a/clang/test/CodeGenCXX/regparm.cpp
+++ b/clang/test/CodeGenCXX/regparm.cpp
@@ -11,7 +11,7 @@ struct S1 {
};
void __attribute__((regparm(3))) foo2(S1 a, int b);
-// CHECK: declare void @_Z4foo22S1i(ptr inreg noundef, i32 inreg noundef)
+// CHECK: declare void @_Z4foo22S1i(ptr dead_on_return inreg noundef, i32 inreg noundef)
void bar2(S1 a, int b) {
foo2(a, b);
}
diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp
index b8cc0d1cc6528..eacbde594e517 100644
--- a/clang/test/CodeGenCXX/trivial_abi.cpp
+++ b/clang/test/CodeGenCXX/trivial_abi.cpp
@@ -140,7 +140,7 @@ void testIgnoredSmall() {
testReturnSmall();
}
-// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[A:.*]])
// CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[A]])
// CHECK: ret void
// CHECK: }
@@ -163,7 +163,7 @@ Large testReturnLarge() {
// CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE]], align 8
// CHECK: %[[CALL:.*]] = call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[T]])
// CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN5LargeC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(520) %[[T]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
// CHECK: %[[CALL2:.*]] = call noundef ptr @_ZN5LargeD1Ev(ptr {{[^,]*}} %[[T]])
// CHECK: ret void
// CHECK: }
@@ -176,7 +176,7 @@ void testCallLarge0() {
// CHECK: define{{.*}} void @_Z14testCallLarge1v()
// CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_LARGE:.*]], align 8
// CHECK: call void @_Z15testReturnLargev(ptr dead_on_unwind writable sret(%[[STRUCT_LARGE]]) align 8 %[[AGG_TMP]])
-// CHECK: call void @_Z14testParamLarge5Large(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z14testParamLarge5Large(ptr dead_on_return noundef %[[AGG_TMP]])
// CHECK: ret void
// CHECK: }
@@ -244,7 +244,7 @@ void testExceptionSmall() {
// CHECK: call noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP]])
// CHECK: invoke noundef ptr @_ZN5LargeC1Ev(ptr {{[^,]*}} %[[AGG_TMP1]])
-// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @_Z20calleeExceptionLarge5LargeS_(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
// CHECK-NEXT: ret void
// CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenCXX/uncopyable-args.cpp b/clang/test/CodeGenCXX/uncopyable-args.cpp
index 31192b65cc362..2d09732f9d3c2 100644
--- a/clang/test/CodeGenCXX/uncopyable-args.cpp
+++ b/clang/test/CodeGenCXX/uncopyable-args.cpp
@@ -59,12 +59,12 @@ void bar() {
// CHECK-LABEL: define{{.*}} void @_ZN9move_ctor3barEv()
// CHECK: call void @_Z{{.*}}C1Ev(
// CHECK-NOT: call
-// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
// OLDABI: call void @_ZN9move_ctor3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr dead_on_return noundef)
// OLDABI-LABEL: declare void @_ZN9move_ctor3fooENS_1AE(ptr)
-// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_ctor@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
}
namespace all_deleted {
@@ -81,12 +81,12 @@ void bar() {
// CHECK-LABEL: define{{.*}} void @_ZN11all_deleted3barEv()
// CHECK: call void @_Z{{.*}}C1Ev(
// CHECK-NOT: call
-// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
// OLDABI: call void @_ZN11all_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr dead_on_return noundef)
// OLDABI-LABEL: declare void @_ZN11all_deleted3fooENS_1AE(ptr)
-// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at all_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
}
namespace implicitly_deleted {
@@ -102,14 +102,14 @@ void bar() {
// CHECK-LABEL: define{{.*}} void @_ZN18implicitly_deleted3barEv()
// CHECK: call void @_Z{{.*}}C1Ev(
// CHECK-NOT: call
-// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
// OLDABI: call void @_ZN18implicitly_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr dead_on_return noundef)
// OLDABI-LABEL: declare void @_ZN18implicitly_deleted3fooENS_1AE(ptr)
// In MSVC 2013, the copy ctor is not deleted by a move assignment. In MSVC 2015, it is.
// WIN64-18-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(i64
-// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-19-LABEL: declare dso_local void @"?foo at implicitly_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
}
namespace one_deleted {
@@ -125,12 +125,12 @@ void bar() {
// CHECK-LABEL: define{{.*}} void @_ZN11one_deleted3barEv()
// CHECK: call void @_Z{{.*}}C1Ev(
// CHECK-NOT: call
-// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef %{{.*}})
// OLDABI: call void @_ZN11one_deleted3fooENS_1AE(ptr %{{.*}})
-// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr dead_on_return noundef)
// OLDABI-LABEL: declare void @_ZN11one_deleted3fooENS_1AE(ptr)
-// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at one_deleted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
}
namespace copy_defaulted {
@@ -170,7 +170,7 @@ void bar() {
// CHECK: call void @_ZN14move_defaulted3fooENS_1AE(ptr %{{.*}})
// CHECK-LABEL: declare void @_ZN14move_defaulted3fooENS_1AE(ptr)
-// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at move_defaulted@@YAXUA at 1@@Z"(ptr dead_on_return noundef)
}
namespace trivial_defaulted {
@@ -207,12 +207,12 @@ void bar() {
}
// CHECK-LABEL: define{{.*}} void @_ZN14two_copy_ctors3barEv()
// CHECK: call void @_Z{{.*}}C1Ev(
-// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef %{{.*}})
+// NEWABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef %{{.*}})
// OLDABI: call void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
-// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef)
+// NEWABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr dead_on_return noundef)
// OLDABI-LABEL: declare void @_ZN14two_copy_ctors3fooENS_1BE(ptr noundef byval
-// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr noundef)
+// WIN64-LABEL: declare dso_local void @"?foo at two_copy_ctors@@YAXUB at 1@@Z"(ptr dead_on_return noundef)
}
namespace definition_only {
diff --git a/clang/test/CodeGenCXX/wasm-args-returns.cpp b/clang/test/CodeGenCXX/wasm-args-returns.cpp
index fbb152ac1bb3d..7b1c27bf0808d 100644
--- a/clang/test/CodeGenCXX/wasm-args-returns.cpp
+++ b/clang/test/CodeGenCXX/wasm-args-returns.cpp
@@ -46,17 +46,17 @@ struct copy_ctor {
copy_ctor(copy_ctor const &);
};
test(copy_ctor);
-// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward9copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.copy_ctor) align 8 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
//
// CHECK: declare ptr @_ZN9copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 8 dereferenceable(8))
//
// CHECK: define void @_Z14test_copy_ctorv()
// CHECK: %[[tmp:.*]] = alloca %struct.copy_ctor, align 8
// CHECK: call void @_Z13def_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.copy_ctor) align 8 %[[tmp]])
-// CHECK: call void @_Z3use9copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use9copy_ctor(ptr dead_on_return nonnull %[[tmp]])
// CHECK: ret void
//
-// CHECK: declare void @_Z3use9copy_ctor(ptr)
+// CHECK: declare void @_Z3use9copy_ctor(ptr dead_on_return)
// CHECK: declare void @_Z13def_copy_ctorv(ptr dead_on_unwind writable sret(%struct.copy_ctor) align 8)
struct __attribute__((aligned(16))) aligned_copy_ctor {
@@ -64,17 +64,17 @@ struct __attribute__((aligned(16))) aligned_copy_ctor {
aligned_copy_ctor(aligned_copy_ctor const &);
};
test(aligned_copy_ctor);
-// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr nonnull %{{.*}})
+// CHECK: define void @_Z7forward17aligned_copy_ctor(ptr dead_on_unwind noalias {{[^,]*}} sret(%struct.aligned_copy_ctor) align 16 %{{.*}}, ptr dead_on_return nonnull %{{.*}})
//
// CHECK: declare ptr @_ZN17aligned_copy_ctorC1ERKS_(ptr {{[^,]*}} returned {{[^,]*}}, ptr nonnull align 16 dereferenceable(16))
//
// CHECK: define void @_Z22test_aligned_copy_ctorv()
// CHECK: %[[tmp:.*]] = alloca %struct.aligned_copy_ctor, align 16
// CHECK: call void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind nonnull writable sret(%struct.aligned_copy_ctor) align 16 %[[tmp]])
-// CHECK: call void @_Z3use17aligned_copy_ctor(ptr nonnull %[[tmp]])
+// CHECK: call void @_Z3use17aligned_copy_ctor(ptr dead_on_return nonnull %[[tmp]])
// CHECK: ret void
//
-// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr)
+// CHECK: declare void @_Z3use17aligned_copy_ctor(ptr dead_on_return)
// CHECK: declare void @_Z21def_aligned_copy_ctorv(ptr dead_on_unwind writable sret(%struct.aligned_copy_ctor) align 16)
struct empty {};
diff --git a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
index 9927d0b24799e..78c45753b5bbd 100644
--- a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
+++ b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
@@ -18,7 +18,7 @@ struct NonTrivial {
SWIFTCALL int receiveNonTrivial(NonTrivial o) { return o.o; }
-// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %o)
+// CHECK-LABEL: define dso_local swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %o)
int passNonTrivial() {
return receiveNonTrivial({});
@@ -26,4 +26,4 @@ int passNonTrivial() {
// CHECK-LABEL: define dso_local noundef i32 @"?passNonTrivial@@YAHXZ"()
// CHECK-NOT: stacksave
-// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr noundef %{{.*}})
+// CHECK: call swiftcc noundef i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(ptr dead_on_return noundef %{{.*}})
diff --git a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
index d2a954ae26a04..ba8a04b52716e 100644
--- a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
+++ b/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
@@ -42,7 +42,7 @@ void testStrongException(void) {
// CHECK: call void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP]])
// CHECK: invoke void @genWeak(ptr dead_on_unwind writable sret(%[[STRUCT_WEAK]]) align 8 %[[AGG_TMP1]])
-// CHECK: call void @calleeWeak(ptr noundef %[[AGG_TMP]], ptr noundef %[[AGG_TMP1]])
+// CHECK: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]], ptr dead_on_return noundef %[[AGG_TMP1]])
// CHECK: ret void
// CHECK: landingpad { ptr, i32 }
diff --git a/clang/test/CodeGenObjC/pass-by-value-noalias.m b/clang/test/CodeGenObjC/pass-by-value-noalias.m
index ed94d4c80b525..86a4ba36a1b8f 100644
--- a/clang/test/CodeGenObjC/pass-by-value-noalias.m
+++ b/clang/test/CodeGenObjC/pass-by-value-noalias.m
@@ -17,6 +17,6 @@ @interface Bar
Bar *__weak f;
};
-// WITH_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
-// NO_NOALIAS: define{{.*}} void @take(ptr noundef %arg)
+// WITH_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
+// NO_NOALIAS: define{{.*}} void @take(ptr dead_on_return noundef %arg)
void take(struct Foo arg) {}
diff --git a/clang/test/CodeGenObjC/weak-in-c-struct.m b/clang/test/CodeGenObjC/weak-in-c-struct.m
index be80edd1ff11d..6809360d03da1 100644
--- a/clang/test/CodeGenObjC/weak-in-c-struct.m
+++ b/clang/test/CodeGenObjC/weak-in-c-struct.m
@@ -130,7 +130,7 @@ void test_move_assignment_Weak(Weak *p) {
*p = getWeak();
}
-// COMMON: define{{.*}} void @test_parameter_Weak(ptr noundef %[[A:.*]])
+// COMMON: define{{.*}} void @test_parameter_Weak(ptr dead_on_return noundef %[[A:.*]])
// COMMON: call void @__destructor_{{.*}}(ptr %[[A]])
void test_parameter_Weak(Weak a) {
@@ -142,7 +142,7 @@ void test_parameter_Weak(Weak a) {
// COMMON: store ptr %[[A]], ptr %[[A_ADDR]]
// COMMON: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]]
// COMMON: call void @__copy_constructor_{{.*}}(ptr %[[AGG_TMP]], ptr %[[V0]])
-// COMMON: call void @calleeWeak(ptr noundef %[[AGG_TMP]])
+// COMMON: call void @calleeWeak(ptr dead_on_return noundef %[[AGG_TMP]])
// COMMON-NEXT: ret
void test_argument_Weak(Weak *a) {
@@ -164,7 +164,7 @@ Weak test_return_Weak(Weak *a) {
// COMMON: %[[AGG_TMP:.*]] = alloca %[[STRUCT_WEAK]]
// COMMON: br i1
-// COMMON: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// COMMON: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
// COMMON: br
// COMMON: call void @__destructor_{{.*}}(ptr %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
index 4d76796d86d1f..3a043c4892981 100644
--- a/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/objc-struct-cxx-abi.mm
@@ -75,7 +75,7 @@ - (void)passStrongWeak:(StrongWeak)a;
- (void)passNonTrivial:(NonTrivial)a;
@end
-// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %{{.*}})
// CHECK: call noundef ptr @_ZN10StrongWeakD1Ev(
// CHECK-NEXT: ret void
@@ -88,7 +88,7 @@ void testParamStrongWeak(StrongWeak a) {
// CHECK: store ptr %[[A]], ptr %[[A_ADDR]], align 8
// CHECK: %[[V0:.*]] = load ptr, ptr %[[A_ADDR]], align 8
// CHECK: %[[CALL:.*]] = call noundef ptr @_ZN10StrongWeakC1ERKS_(ptr {{[^,]*}} %[[AGG_TMP]], ptr noundef nonnull align 8 dereferenceable(16) %[[V0]])
-// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr noundef %[[AGG_TMP]])
+// CHECK: call void @_Z19testParamStrongWeak10StrongWeak(ptr dead_on_return noundef %[[AGG_TMP]])
// CHECK-NOT: call
// CHECK: ret void
@@ -107,13 +107,13 @@ StrongWeak testReturnStrongWeak(StrongWeak *a) {
return *a;
}
-// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z27testParamContainsStrongWeak18ContainsStrongWeak(ptr dead_on_return noundef %[[A:.*]])
// CHECK: call noundef ptr @_ZN18ContainsStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
void testParamContainsStrongWeak(ContainsStrongWeak a) {
}
-// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr noundef %[[A:.*]])
+// CHECK: define{{.*}} void @_Z26testParamDerivedStrongWeak17DerivedStrongWeak(ptr dead_on_return noundef %[[A:.*]])
// CHECK: call noundef ptr @_ZN17DerivedStrongWeakD1Ev(ptr {{[^,]*}} %[[A]])
void testParamDerivedStrongWeak(DerivedStrongWeak a) {
@@ -163,7 +163,7 @@ Strong testReturnStrong(Strong *a) {
return *a;
}
-// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr noundef %{{.*}})
+// CHECK: define{{.*}} void @_Z21testParamWeakTemplate1SIU6__weakP11objc_objectE(ptr dead_on_return noundef %{{.*}})
// CHECK: call noundef ptr @_ZN1SIU6__weakP11objc_objectED1Ev(
// CHECK-NEXT: ret void
@@ -237,7 +237,7 @@ void test0(C *c) {
// CHECK: %[[AGG_TMP:.*]] = alloca %[[STRUCT_STRONGWEAK]], align 8
// CHECK: br i1
-// CHECK: call void @objc_msgSend({{.*}}, ptr noundef %[[AGG_TMP]])
+// CHECK: call void @objc_msgSend({{.*}}, ptr dead_on_return noundef %[[AGG_TMP]])
// CHECK: br
// CHECK: %[[CALL1:.*]] = call noundef ptr @_ZN10StrongWeakD1Ev(ptr noundef nonnull align 8 dereferenceable(16) %[[AGG_TMP]])
diff --git a/clang/test/CodeGenObjCXX/property-objects.mm b/clang/test/CodeGenObjCXX/property-objects.mm
index 7ae20f66177bd..8354794254933 100644
--- a/clang/test/CodeGenObjCXX/property-objects.mm
+++ b/clang/test/CodeGenObjCXX/property-objects.mm
@@ -60,7 +60,7 @@ - (struct CGRect)extent {return bounds;}
// CHECK-LABEL: define{{.*}} i32 @main
// CHECK: call void @_ZN1SC1ERKS_(ptr {{[^,]*}} [[AGGTMP:%[a-zA-Z0-9\.]+]], ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) {{%[a-zA-Z0-9\.]+}})
-// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef [[AGGTMP]])
+// CHECK: call void @objc_msgSend(ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr noundef {{%[a-zA-Z0-9\.]+}}, ptr dead_on_return noundef [[AGGTMP]])
// CHECK-NEXT: ret i32 0
int main() {
I *i;
diff --git a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
index e5cb71bad47c0..9428940d6da48 100644
--- a/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
+++ b/clang/test/CodeGenObjCXX/ptrauth-struct-cxx-abi.mm
@@ -15,7 +15,7 @@
// Check that AddrDiscStrong0 is destructed in the callee.
-// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr noundef %[[A:.*]])
+// CHECK: define void @_Z24testParamAddrDiscStrong015AddrDiscStrong0(ptr dead_on_return noundef %[[A:.*]])
// CHECK: call noundef ptr @_ZN15AddrDiscStrong0D1Ev(ptr noundef nonnull align {{[0-9]+}} dereferenceable(16) %[[A]])
// CHECK: ret void
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 738b7ab7f2b75..d42e51d04ab9d 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -4,6 +4,7 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+typedef unsigned int uint;
typedef half __attribute__((ext_vector_type(2))) half2;
// CHECK-LABEL: @test_setprio_inc_wg(
@@ -42,6 +43,24 @@ void test_s_wait_tensorcnt() {
__builtin_amdgcn_s_wait_tensorcnt(0);
}
+// CHECK-LABEL: @test_prng_b32(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT: [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT: store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT: store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.prng.b32(i32 [[TMP0]])
+// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT: store i32 [[TMP1]], ptr addrspace(1) [[TMP2]], align 4
+// CHECK-NEXT: ret void
+//
+void test_prng_b32(global uint* out, uint a) {
+ *out = __builtin_amdgcn_prng_b32(a);
+}
+
// CHECK-LABEL: @test_tanh_f32(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
@@ -119,6 +138,25 @@ void test_rcp_bf16(global __bf16* out, __bf16 a)
*out = __builtin_amdgcn_rcp_bf16(a);
}
+// CHECK-LABEL: @test_sqrt_bf16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT: [[A_ADDR:%.*]] = alloca bfloat, align 2, addrspace(5)
+// CHECK-NEXT: [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT: [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT: store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT: store bfloat [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT: [[TMP0:%.*]] = load bfloat, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT: [[TMP1:%.*]] = call bfloat @llvm.amdgcn.sqrt.bf16(bfloat [[TMP0]])
+// CHECK-NEXT: [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT: store bfloat [[TMP1]], ptr addrspace(1) [[TMP2]], align 2
+// CHECK-NEXT: ret void
+//
+void test_sqrt_bf16(global __bf16* out, __bf16 a)
+{
+ *out = __builtin_amdgcn_sqrt_bf16(a);
+}
+
// CHECK-LABEL: @test_rsq_bf16(
// CHECK-NEXT: entry:
// CHECK-NEXT: [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 5f9383fbed8f4..8cf8f04bb6b48 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -107,3 +107,22 @@
// CHECK-AARCH64-MULTILIB-CUSTOM-FLAG: --target=aarch64-unknown-none-eabi
// CHECK-MULTILIB-CUSTOM-FLAG-DAG: -fmultilib-flag=foo
// CHECK-MULTILIB-CUSTOM-FLAG-DAG: -fmultilib-flag=bar
+
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi | FileCheck --check-prefixes=CHECK-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -frwpi | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fropi -frwpi | FileCheck --check-prefixes=CHECK-ROPI,CHECK-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fno-ropi -fno-rwpi | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-NO-PIC %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpic | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIC | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIC2 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fpie | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE1 %s
+// RUN: %clang -multi-lib-config=%S/Inputs/multilib/empty.yaml -print-multi-flags-experimental --target=arm-none-eabi -march=armv7a -fPIE | FileCheck --check-prefixes=CHECK-NO-ROPI,CHECK-NO-RWPI,CHECK-PIE2 %s
+// CHECK-PIC2: -fPIC
+// CHECK-PIE2: -fPIE
+// CHECK-NO-PIC: -fno-pic
+// CHECK-NO-ROPI: -fno-ropi
+// CHECK-NO-RWPI: -fno-rwpi
+// CHECK-PIC1: -fpic
+// CHECK-PIE1: -fpie
+// CHECK-ROPI: -fropi
+// CHECK-RWPI: -frwpi
diff --git a/clang/test/Headers/stdarg.cpp b/clang/test/Headers/stdarg.cpp
index 20bf17caf15f7..bfc3af11a23b6 100644
--- a/clang/test/Headers/stdarg.cpp
+++ b/clang/test/Headers/stdarg.cpp
@@ -15,8 +15,8 @@
#include <stdarg.h>
-// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
-// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr noundef %list)
+// AARCH64-C: define {{.*}} @f(i32 noundef %n, ptr dead_on_return noundef %list)
+// AARCH64-CXX: define {{.*}} @_Z1fiSt9__va_list(i32 noundef %n, ptr dead_on_return noundef %list)
// X86_64-C: define {{.*}} @f(i32 noundef %n, ptr noundef %list)
// X86_64-CXX: define {{.*}} @_Z1fiP13__va_list_tag(i32 noundef %n, ptr noundef %list)
// PPC64-C: define {{.*}} @f(i32 noundef signext %n, ptr noundef %list)
diff --git a/clang/test/OpenMP/for_firstprivate_codegen.cpp b/clang/test/OpenMP/for_firstprivate_codegen.cpp
index 0255e1e3d4aea..83b5939799642 100644
--- a/clang/test/OpenMP/for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/for_firstprivate_codegen.cpp
@@ -427,7 +427,7 @@ int main() {
// CHECK1-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
// CHECK1-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
// CHECK1-NEXT: store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 4, ptr @_Z5tmainIiET_v.omp_outlined, ptr [[T_VAR]], ptr [[VEC]], ptr [[S_ARR]], ptr [[TMP0]])
// CHECK1-NEXT: store i32 0, ptr [[RETVAL]], align 4
// CHECK1-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR]], i32 0, i32 0
@@ -469,7 +469,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -543,12 +543,12 @@ int main() {
// CHECK1-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: store ptr [[TMP3]], ptr [[TMP]], align 8
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: store ptr [[TMP4]], ptr [[_TMP1]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -572,7 +572,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP6]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE6]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done6:
-// CHECK1-NEXT: [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK1-NEXT: [[TMP7:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR7]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP7]], ptr noundef [[AGG_TMP8]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP8]]) #[[ATTR2]]
@@ -611,7 +611,7 @@ int main() {
// CHECK1-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
// CHECK1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC4]], i64 0, i64 [[IDXPROM]]
// CHECK1-NEXT: store i32 [[TMP16]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8
+// CHECK1-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP9]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: [[TMP19:%.*]] = load i32, ptr [[I]], align 4
// CHECK1-NEXT: [[IDXPROM11:%.*]] = sext i32 [[TMP19]] to i64
// CHECK1-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR5]], i64 0, i64 [[IDXPROM11]]
@@ -711,7 +711,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1044,8 +1044,8 @@ int main() {
// CHECK4-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
// CHECK4-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
// CHECK4-NEXT: store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr @g1, align 8
+// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr @g1, align 8, !nonnull [[META3]], !align [[META4]]
// CHECK4-NEXT: store ptr [[TMP1]], ptr [[TMP]], align 8
// CHECK4-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK4-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1086,7 +1086,7 @@ int main() {
// CHECK4-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
// CHECK4-NEXT: store i32 [[ADD]], ptr [[I]], align 4
// CHECK4-NEXT: store i32 1, ptr [[G]], align 4
-// CHECK4-NEXT: [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT: [[TMP13:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK4-NEXT: store volatile i32 1, ptr [[TMP13]], align 4
// CHECK4-NEXT: store i32 2, ptr [[SIVAR3]], align 4
// CHECK4-NEXT: [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1103,7 +1103,7 @@ int main() {
// CHECK4-NEXT: [[TMP14:%.*]] = load volatile i32, ptr [[G]], align 4
// CHECK4-NEXT: store volatile i32 [[TMP14]], ptr [[BLOCK_CAPTURED]], align 8
// CHECK4-NEXT: [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 5
-// CHECK4-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK4-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK4-NEXT: store ptr [[TMP15]], ptr [[BLOCK_CAPTURED5]], align 8
// CHECK4-NEXT: [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[BLOCK]], i32 0, i32 7
// CHECK4-NEXT: [[TMP16:%.*]] = load i32, ptr [[SIVAR3]], align 4
@@ -1137,7 +1137,7 @@ int main() {
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
// CHECK4-NEXT: store i32 2, ptr [[BLOCK_CAPTURE_ADDR]], align 8
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR1:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
-// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8
+// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR1]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK4-NEXT: store i32 2, ptr [[TMP0]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, i32, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
// CHECK4-NEXT: store i32 4, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
diff --git a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
index 93e71b9a8312e..441e809dc59e5 100644
--- a/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/parallel_firstprivate_codegen.cpp
@@ -350,9 +350,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
// CHECK1-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
// CHECK1-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -524,7 +524,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
// CHECK1-NEXT: store i8 [[BF_SET]], ptr [[B]], align 4
// CHECK1-NEXT: [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: store ptr [[TMP0]], ptr [[C]], align 4
// CHECK1-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
// CHECK1-NEXT: store ptr [[A3]], ptr [[A2]], align 4
@@ -535,22 +535,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
// CHECK1-NEXT: store i32 [[BF_CAST]], ptr [[B4]], align 4
// CHECK1-NEXT: [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: store ptr [[TMP1]], ptr [[C7]], align 4
// CHECK1-NEXT: [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
// CHECK1-NEXT: store ptr [[E9]], ptr [[E]], align 4
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[A_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
// CHECK1-NEXT: store i32 [[TMP5]], ptr [[B_CASTED]], align 4
// CHECK1-NEXT: [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK1-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK1-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
// CHECK1-NEXT: store i32 [[TMP8]], ptr [[C_CASTED]], align 4
// CHECK1-NEXT: [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK1-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
// CHECK1-NEXT: ret void
//
@@ -578,25 +578,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store i32 [[C]], ptr [[C_ADDR]], align 4
// CHECK1-NEXT: store ptr [[E]], ptr [[E_ADDR]], align 4
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 4
// CHECK1-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
// CHECK1-NEXT: store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
// CHECK1-NEXT: store ptr [[E3]], ptr [[_TMP4]], align 4
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
// CHECK1-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1
// CHECK1-NEXT: store i32 [[INC]], ptr [[TMP3]], align 4
// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK1-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
// CHECK1-NEXT: store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
// CHECK1-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
// CHECK1-NEXT: store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4
+// CHECK1-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i32 0, i32 2
// CHECK1-NEXT: store i32 1111, ptr [[ARRAYIDX]], align 4
// CHECK1-NEXT: ret void
@@ -656,7 +656,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -738,9 +738,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
// CHECK1-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
// CHECK1-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -840,7 +840,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store i32 0, ptr [[A]], align 4
// CHECK1-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
// CHECK1-NEXT: store ptr [[A3]], ptr [[A2]], align 4
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
// CHECK1-NEXT: store i32 [[TMP1]], ptr [[A_CASTED]], align 4
// CHECK1-NEXT: [[TMP2:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -862,7 +862,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK1-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 4
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK1-NEXT: [[INC:%.*]] = add nsw i32 [[TMP2]], 1
// CHECK1-NEXT: store i32 [[INC]], ptr [[TMP1]], align 4
@@ -896,7 +896,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -965,7 +965,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK3-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
// CHECK3-NEXT: store i8 [[BF_SET]], ptr [[B]], align 4
// CHECK3-NEXT: [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
// CHECK3-NEXT: store ptr [[TMP0]], ptr [[C]], align 4
// CHECK3-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
// CHECK3-NEXT: store ptr [[A3]], ptr [[A2]], align 4
@@ -976,22 +976,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK3-NEXT: [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
// CHECK3-NEXT: store i32 [[BF_CAST]], ptr [[B4]], align 4
// CHECK3-NEXT: [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: store ptr [[TMP1]], ptr [[C7]], align 4
// CHECK3-NEXT: [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
// CHECK3-NEXT: store ptr [[E9]], ptr [[E]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[A_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
// CHECK3-NEXT: [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
// CHECK3-NEXT: store i32 [[TMP5]], ptr [[B_CASTED]], align 4
// CHECK3-NEXT: [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK3-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK3-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
// CHECK3-NEXT: store i32 [[TMP8]], ptr [[C_CASTED]], align 4
// CHECK3-NEXT: [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK3-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK3-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
// CHECK3-NEXT: ret void
//
@@ -1020,22 +1020,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK3-NEXT: store i32 [[C]], ptr [[C_ADDR]], align 4
// CHECK3-NEXT: store ptr [[E]], ptr [[E_ADDR]], align 4
// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 4
// CHECK3-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
// CHECK3-NEXT: store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
// CHECK3-NEXT: store ptr [[E3]], ptr [[_TMP4]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK3-NEXT: store ptr [[TMP0]], ptr [[TMP3]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: store ptr [[TMP5]], ptr [[TMP4]], align 4
// CHECK3-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK3-NEXT: store ptr [[B_ADDR]], ptr [[TMP6]], align 4
// CHECK3-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK3-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: store ptr [[TMP8]], ptr [[TMP7]], align 4
// CHECK3-NEXT: call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 4 dereferenceable(16) [[REF_TMP]])
// CHECK3-NEXT: ret void
@@ -1053,32 +1053,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK3-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 4
// CHECK3-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4
+// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
// CHECK3-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1
// CHECK3-NEXT: store i32 [[INC]], ptr [[TMP3]], align 4
// CHECK3-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4
+// CHECK3-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
// CHECK3-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
// CHECK3-NEXT: store i32 [[DEC]], ptr [[TMP6]], align 4
// CHECK3-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4
+// CHECK3-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
// CHECK3-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
// CHECK3-NEXT: store i32 [[DIV]], ptr [[TMP9]], align 4
// CHECK3-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK3-NEXT: [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4
+// CHECK3-NEXT: [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
// CHECK3-NEXT: store i32 [[TMP13]], ptr [[A_CASTED]], align 4
// CHECK3-NEXT: [[TMP14:%.*]] = load i32, ptr [[A_CASTED]], align 4
// CHECK3-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK3-NEXT: [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4
+// CHECK3-NEXT: [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
// CHECK3-NEXT: store i32 [[TMP17]], ptr [[B_CASTED]], align 4
// CHECK3-NEXT: [[TMP18:%.*]] = load i32, ptr [[B_CASTED]], align 4
// CHECK3-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK3-NEXT: [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4
+// CHECK3-NEXT: [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
// CHECK3-NEXT: store i32 [[TMP21]], ptr [[C_CASTED]], align 4
// CHECK3-NEXT: [[TMP22:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1106,14 +1106,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 4
// CHECK3-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK3-NEXT: [[INC:%.*]] = add nsw i32 [[TMP2]], 1
// CHECK3-NEXT: store i32 [[INC]], ptr [[TMP1]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK3-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
// CHECK3-NEXT: store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META4]], !align [[META5]]
// CHECK3-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
// CHECK3-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
// CHECK3-NEXT: store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1252,7 +1252,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
// CHECK4-NEXT: store i8 [[BF_SET]], ptr [[B]], align 4
// CHECK4-NEXT: [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4
+// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 4, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
// CHECK4-NEXT: store ptr [[TMP0]], ptr [[C]], align 4
// CHECK4-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
// CHECK4-NEXT: store ptr [[A3]], ptr [[A2]], align 4
@@ -1263,22 +1263,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
// CHECK4-NEXT: store i32 [[BF_CAST]], ptr [[B4]], align 4
// CHECK4-NEXT: [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4
+// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: store ptr [[TMP1]], ptr [[C7]], align 4
// CHECK4-NEXT: [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
// CHECK4-NEXT: store ptr [[E9]], ptr [[E]], align 4
-// CHECK4-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4
+// CHECK4-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
// CHECK4-NEXT: store i32 [[TMP3]], ptr [[A_CASTED]], align 4
// CHECK4-NEXT: [[TMP4:%.*]] = load i32, ptr [[A_CASTED]], align 4
// CHECK4-NEXT: [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
// CHECK4-NEXT: store i32 [[TMP5]], ptr [[B_CASTED]], align 4
// CHECK4-NEXT: [[TMP6:%.*]] = load i32, ptr [[B_CASTED]], align 4
-// CHECK4-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4
+// CHECK4-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
// CHECK4-NEXT: store i32 [[TMP8]], ptr [[C_CASTED]], align 4
// CHECK4-NEXT: [[TMP9:%.*]] = load i32, ptr [[C_CASTED]], align 4
-// CHECK4-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 4
+// CHECK4-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i32 [[TMP4]], i32 [[TMP6]], i32 [[TMP9]], ptr [[TMP10]])
// CHECK4-NEXT: ret void
//
@@ -1307,11 +1307,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: store i32 [[C]], ptr [[C_ADDR]], align 4
// CHECK4-NEXT: store ptr [[E]], ptr [[E_ADDR]], align 4
// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
-// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4
+// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 4
// CHECK4-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
// CHECK4-NEXT: store ptr [[TMP1]], ptr [[_TMP2]], align 4
-// CHECK4-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4
+// CHECK4-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[E3]], ptr align 4 [[TMP2]], i32 16, i1 false)
// CHECK4-NEXT: store ptr [[E3]], ptr [[_TMP4]], align 4
// CHECK4-NEXT: [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 0
@@ -1327,13 +1327,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 5
// CHECK4-NEXT: store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK4-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 7
// CHECK4-NEXT: [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK4-NEXT: store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[BLOCK]], i32 0, i32 8
-// CHECK4-NEXT: [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT: [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 4
// CHECK4-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
// CHECK4-NEXT: [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 4
@@ -1354,7 +1354,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
// CHECK4-NEXT: [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4
+// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
// CHECK4-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
// CHECK4-NEXT: store i32 [[INC]], ptr [[TMP0]], align 4
@@ -1363,12 +1363,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
// CHECK4-NEXT: store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT: [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4
+// CHECK4-NEXT: [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
// CHECK4-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
// CHECK4-NEXT: store i32 [[DIV]], ptr [[TMP3]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK4-NEXT: [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4
+// CHECK4-NEXT: [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
// CHECK4-NEXT: store i32 [[TMP6]], ptr [[A_CASTED]], align 4
// CHECK4-NEXT: [[TMP7:%.*]] = load i32, ptr [[A_CASTED]], align 4
@@ -1377,7 +1377,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: store i32 [[TMP8]], ptr [[B_CASTED]], align 4
// CHECK4-NEXT: [[TMP9:%.*]] = load i32, ptr [[B_CASTED]], align 4
// CHECK4-NEXT: [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, i32, ptr }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 8
-// CHECK4-NEXT: [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4
+// CHECK4-NEXT: [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
// CHECK4-NEXT: store i32 [[TMP11]], ptr [[C_CASTED]], align 4
// CHECK4-NEXT: [[TMP12:%.*]] = load i32, ptr [[C_CASTED]], align 4
@@ -1405,14 +1405,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK4-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 4
// CHECK4-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 4
-// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK4-NEXT: [[INC:%.*]] = add nsw i32 [[TMP2]], 1
// CHECK4-NEXT: store i32 [[INC]], ptr [[TMP1]], align 4
// CHECK4-NEXT: [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK4-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
// CHECK4-NEXT: store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK4-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4
+// CHECK4-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 4, !nonnull [[META6]], !align [[META7]]
// CHECK4-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
// CHECK4-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
// CHECK4-NEXT: store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -1530,9 +1530,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK9-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
// CHECK9-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
// CHECK9-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1704,7 +1704,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
// CHECK9-NEXT: store i8 [[BF_SET]], ptr [[B]], align 4
// CHECK9-NEXT: [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: store ptr [[TMP0]], ptr [[C]], align 8
// CHECK9-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
// CHECK9-NEXT: store ptr [[A3]], ptr [[A2]], align 8
@@ -1715,22 +1715,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
// CHECK9-NEXT: store i32 [[BF_CAST]], ptr [[B4]], align 4
// CHECK9-NEXT: [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: store ptr [[TMP1]], ptr [[C7]], align 8
// CHECK9-NEXT: [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
// CHECK9-NEXT: store ptr [[E9]], ptr [[E]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
// CHECK9-NEXT: store i32 [[TMP3]], ptr [[A_CASTED]], align 4
// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
// CHECK9-NEXT: [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
// CHECK9-NEXT: store i32 [[TMP5]], ptr [[B_CASTED]], align 4
// CHECK9-NEXT: [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK9-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK9-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
// CHECK9-NEXT: store i32 [[TMP8]], ptr [[C_CASTED]], align 4
// CHECK9-NEXT: [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK9-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK9-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
// CHECK9-NEXT: ret void
//
@@ -1758,25 +1758,25 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store i64 [[C]], ptr [[C_ADDR]], align 8
// CHECK9-NEXT: store ptr [[E]], ptr [[E_ADDR]], align 8
// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 8
// CHECK9-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
// CHECK9-NEXT: store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
// CHECK9-NEXT: store ptr [[E3]], ptr [[_TMP4]], align 8
-// CHECK9-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
// CHECK9-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1
// CHECK9-NEXT: store i32 [[INC]], ptr [[TMP3]], align 4
// CHECK9-NEXT: [[TMP5:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK9-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP5]], -1
// CHECK9-NEXT: store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK9-NEXT: [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK9-NEXT: [[TMP6:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
// CHECK9-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP7]], 1
// CHECK9-NEXT: store i32 [[DIV]], ptr [[TMP6]], align 4
-// CHECK9-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8
+// CHECK9-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP4]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP8]], i64 0, i64 2
// CHECK9-NEXT: store i32 1111, ptr [[ARRAYIDX]], align 4
// CHECK9-NEXT: ret void
@@ -1836,7 +1836,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK9-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK9-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK9-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK9-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1918,9 +1918,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
// CHECK9-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
// CHECK9-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -2020,7 +2020,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store i32 0, ptr [[A]], align 4
// CHECK9-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SST]], ptr [[THIS1]], i32 0, i32 0
// CHECK9-NEXT: store ptr [[A3]], ptr [[A2]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
// CHECK9-NEXT: store i32 [[TMP1]], ptr [[A_CASTED]], align 4
// CHECK9-NEXT: [[TMP2:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2042,7 +2042,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store i64 [[A]], ptr [[A_ADDR]], align 8
// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK9-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK9-NEXT: [[INC:%.*]] = add nsw i32 [[TMP2]], 1
// CHECK9-NEXT: store i32 [[INC]], ptr [[TMP1]], align 4
@@ -2076,7 +2076,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK9-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK9-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK9-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK9-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK9-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2145,7 +2145,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
// CHECK11-NEXT: store i8 [[BF_SET]], ptr [[B]], align 4
// CHECK11-NEXT: [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK11-NEXT: store ptr [[TMP0]], ptr [[C]], align 8
// CHECK11-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
// CHECK11-NEXT: store ptr [[A3]], ptr [[A2]], align 8
@@ -2156,22 +2156,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
// CHECK11-NEXT: store i32 [[BF_CAST]], ptr [[B4]], align 4
// CHECK11-NEXT: [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: store ptr [[TMP1]], ptr [[C7]], align 8
// CHECK11-NEXT: [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
// CHECK11-NEXT: store ptr [[E9]], ptr [[E]], align 8
-// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
// CHECK11-NEXT: store i32 [[TMP3]], ptr [[A_CASTED]], align 4
// CHECK11-NEXT: [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
// CHECK11-NEXT: [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
// CHECK11-NEXT: store i32 [[TMP5]], ptr [[B_CASTED]], align 4
// CHECK11-NEXT: [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK11-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK11-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
// CHECK11-NEXT: store i32 [[TMP8]], ptr [[C_CASTED]], align 4
// CHECK11-NEXT: [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK11-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK11-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1:[0-9]+]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
// CHECK11-NEXT: ret void
//
@@ -2200,22 +2200,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store i64 [[C]], ptr [[C_ADDR]], align 8
// CHECK11-NEXT: store ptr [[E]], ptr [[E_ADDR]], align 8
// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 8
// CHECK11-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
// CHECK11-NEXT: store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
// CHECK11-NEXT: store ptr [[E3]], ptr [[_TMP4]], align 8
// CHECK11-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK11-NEXT: store ptr [[TMP0]], ptr [[TMP3]], align 8
// CHECK11-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK11-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: store ptr [[TMP5]], ptr [[TMP4]], align 8
// CHECK11-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK11-NEXT: store ptr [[B_ADDR]], ptr [[TMP6]], align 8
// CHECK11-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 3
-// CHECK11-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT: [[TMP8:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: store ptr [[TMP8]], ptr [[TMP7]], align 8
// CHECK11-NEXT: call void @_ZZN2SSC1ERiENKUlvE_clEv(ptr nonnull align 8 dereferenceable(32) [[REF_TMP]])
// CHECK11-NEXT: ret void
@@ -2233,32 +2233,32 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0:%.*]], ptr [[THIS1]], i32 0, i32 0
// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
// CHECK11-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
+// CHECK11-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
// CHECK11-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1
// CHECK11-NEXT: store i32 [[INC]], ptr [[TMP3]], align 4
// CHECK11-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8
+// CHECK11-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP5]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
// CHECK11-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP7]], -1
// CHECK11-NEXT: store i32 [[DEC]], ptr [[TMP6]], align 4
// CHECK11-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8
+// CHECK11-NEXT: [[TMP9:%.*]] = load ptr, ptr [[TMP8]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
// CHECK11-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP10]], 1
// CHECK11-NEXT: store i32 [[DIV]], ptr [[TMP9]], align 4
// CHECK11-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 1
-// CHECK11-NEXT: [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8
+// CHECK11-NEXT: [[TMP12:%.*]] = load ptr, ptr [[TMP11]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4
// CHECK11-NEXT: store i32 [[TMP13]], ptr [[A_CASTED]], align 4
// CHECK11-NEXT: [[TMP14:%.*]] = load i64, ptr [[A_CASTED]], align 8
// CHECK11-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 2
-// CHECK11-NEXT: [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8
+// CHECK11-NEXT: [[TMP16:%.*]] = load ptr, ptr [[TMP15]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
// CHECK11-NEXT: store i32 [[TMP17]], ptr [[B_CASTED]], align 4
// CHECK11-NEXT: [[TMP18:%.*]] = load i64, ptr [[B_CASTED]], align 8
// CHECK11-NEXT: [[TMP19:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[THIS1]], i32 0, i32 3
-// CHECK11-NEXT: [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8
+// CHECK11-NEXT: [[TMP20:%.*]] = load ptr, ptr [[TMP19]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
// CHECK11-NEXT: store i32 [[TMP21]], ptr [[C_CASTED]], align 4
// CHECK11-NEXT: [[TMP22:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2286,14 +2286,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK11-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 8
// CHECK11-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK11-NEXT: [[INC:%.*]] = add nsw i32 [[TMP2]], 1
// CHECK11-NEXT: store i32 [[INC]], ptr [[TMP1]], align 4
// CHECK11-NEXT: [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK11-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
// CHECK11-NEXT: store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK11-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK11-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK11-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
// CHECK11-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
// CHECK11-NEXT: store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2432,7 +2432,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 0
// CHECK12-NEXT: store i8 [[BF_SET]], ptr [[B]], align 4
// CHECK12-NEXT: [[C:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK12-NEXT: [[TMP0:%.*]] = load ptr, ptr [[D_ADDR]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
// CHECK12-NEXT: store ptr [[TMP0]], ptr [[C]], align 8
// CHECK12-NEXT: [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 0
// CHECK12-NEXT: store ptr [[A3]], ptr [[A2]], align 8
@@ -2443,22 +2443,22 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: [[BF_CAST:%.*]] = sext i8 [[BF_ASHR]] to i32
// CHECK12-NEXT: store i32 [[BF_CAST]], ptr [[B4]], align 4
// CHECK12-NEXT: [[C8:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 2
-// CHECK12-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8
+// CHECK12-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C8]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: store ptr [[TMP1]], ptr [[C7]], align 8
// CHECK12-NEXT: [[E9:%.*]] = getelementptr inbounds nuw [[STRUCT_SS]], ptr [[THIS1]], i32 0, i32 3
// CHECK12-NEXT: store ptr [[E9]], ptr [[E]], align 8
-// CHECK12-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8
+// CHECK12-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A2]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
// CHECK12-NEXT: store i32 [[TMP3]], ptr [[A_CASTED]], align 4
// CHECK12-NEXT: [[TMP4:%.*]] = load i64, ptr [[A_CASTED]], align 8
// CHECK12-NEXT: [[TMP5:%.*]] = load i32, ptr [[B4]], align 4
// CHECK12-NEXT: store i32 [[TMP5]], ptr [[B_CASTED]], align 4
// CHECK12-NEXT: [[TMP6:%.*]] = load i64, ptr [[B_CASTED]], align 8
-// CHECK12-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8
+// CHECK12-NEXT: [[TMP7:%.*]] = load ptr, ptr [[C7]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
// CHECK12-NEXT: store i32 [[TMP8]], ptr [[C_CASTED]], align 4
// CHECK12-NEXT: [[TMP9:%.*]] = load i64, ptr [[C_CASTED]], align 8
-// CHECK12-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 8
+// CHECK12-NEXT: [[TMP10:%.*]] = load ptr, ptr [[E]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 5, ptr @_ZN2SSC2ERi.omp_outlined, ptr [[THIS1]], i64 [[TMP4]], i64 [[TMP6]], i64 [[TMP9]], ptr [[TMP10]])
// CHECK12-NEXT: ret void
//
@@ -2487,11 +2487,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: store i64 [[C]], ptr [[C_ADDR]], align 8
// CHECK12-NEXT: store ptr [[E]], ptr [[E_ADDR]], align 8
// CHECK12-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
-// CHECK12-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK12-NEXT: [[TMP1:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 8
// CHECK12-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
// CHECK12-NEXT: store ptr [[TMP1]], ptr [[_TMP2]], align 8
-// CHECK12-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8
+// CHECK12-NEXT: [[TMP2:%.*]] = load ptr, ptr [[_TMP2]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[E3]], ptr align 4 [[TMP2]], i64 16, i1 false)
// CHECK12-NEXT: store ptr [[E3]], ptr [[_TMP4]], align 8
// CHECK12-NEXT: [[BLOCK_ISA:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 0
@@ -2507,13 +2507,13 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: [[BLOCK_CAPTURED_THIS_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 5
// CHECK12-NEXT: store ptr [[TMP0]], ptr [[BLOCK_CAPTURED_THIS_ADDR]], align 8
// CHECK12-NEXT: [[BLOCK_CAPTURED:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 6
-// CHECK12-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: store ptr [[TMP3]], ptr [[BLOCK_CAPTURED]], align 8
// CHECK12-NEXT: [[BLOCK_CAPTURED5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 8
// CHECK12-NEXT: [[TMP4:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK12-NEXT: store i32 [[TMP4]], ptr [[BLOCK_CAPTURED5]], align 8
// CHECK12-NEXT: [[BLOCK_CAPTURED6:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[BLOCK]], i32 0, i32 7
-// CHECK12-NEXT: [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT: [[TMP5:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: store ptr [[TMP5]], ptr [[BLOCK_CAPTURED6]], align 8
// CHECK12-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw [[STRUCT___BLOCK_LITERAL_GENERIC:%.*]], ptr [[BLOCK]], i32 0, i32 3
// CHECK12-NEXT: [[TMP7:%.*]] = load ptr, ptr [[TMP6]], align 8
@@ -2534,7 +2534,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: [[BLOCK_CAPTURED_THIS:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 5
// CHECK12-NEXT: [[THIS:%.*]] = load ptr, ptr [[BLOCK_CAPTURED_THIS]], align 8
// CHECK12-NEXT: [[BLOCK_CAPTURE_ADDR:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT: [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8
+// CHECK12-NEXT: [[TMP0:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4
// CHECK12-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
// CHECK12-NEXT: store i32 [[INC]], ptr [[TMP0]], align 4
@@ -2543,12 +2543,12 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP2]], -1
// CHECK12-NEXT: store i32 [[DEC]], ptr [[BLOCK_CAPTURE_ADDR1]], align 8
// CHECK12-NEXT: [[BLOCK_CAPTURE_ADDR2:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT: [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8
+// CHECK12-NEXT: [[TMP3:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR2]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
// CHECK12-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP4]], 1
// CHECK12-NEXT: store i32 [[DIV]], ptr [[TMP3]], align 4
// CHECK12-NEXT: [[BLOCK_CAPTURE_ADDR3:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 6
-// CHECK12-NEXT: [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8
+// CHECK12-NEXT: [[TMP5:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR3]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
// CHECK12-NEXT: store i32 [[TMP6]], ptr [[A_CASTED]], align 4
// CHECK12-NEXT: [[TMP7:%.*]] = load i64, ptr [[A_CASTED]], align 8
@@ -2557,7 +2557,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: store i32 [[TMP8]], ptr [[B_CASTED]], align 4
// CHECK12-NEXT: [[TMP9:%.*]] = load i64, ptr [[B_CASTED]], align 8
// CHECK12-NEXT: [[BLOCK_CAPTURE_ADDR5:%.*]] = getelementptr inbounds nuw <{ ptr, i32, i32, ptr, ptr, ptr, ptr, ptr, i32 }>, ptr [[DOTBLOCK_DESCRIPTOR]], i32 0, i32 7
-// CHECK12-NEXT: [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8
+// CHECK12-NEXT: [[TMP10:%.*]] = load ptr, ptr [[BLOCK_CAPTURE_ADDR5]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
// CHECK12-NEXT: store i32 [[TMP11]], ptr [[C_CASTED]], align 4
// CHECK12-NEXT: [[TMP12:%.*]] = load i64, ptr [[C_CASTED]], align 8
@@ -2585,14 +2585,14 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK12-NEXT: [[TMP0:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK12-NEXT: store ptr [[A_ADDR]], ptr [[TMP]], align 8
// CHECK12-NEXT: store ptr [[C_ADDR]], ptr [[_TMP1]], align 8
-// CHECK12-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK12-NEXT: [[TMP1:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK12-NEXT: [[INC:%.*]] = add nsw i32 [[TMP2]], 1
// CHECK12-NEXT: store i32 [[INC]], ptr [[TMP1]], align 4
// CHECK12-NEXT: [[TMP3:%.*]] = load i32, ptr [[B_ADDR]], align 4
// CHECK12-NEXT: [[DEC:%.*]] = add nsw i32 [[TMP3]], -1
// CHECK12-NEXT: store i32 [[DEC]], ptr [[B_ADDR]], align 4
-// CHECK12-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8
+// CHECK12-NEXT: [[TMP4:%.*]] = load ptr, ptr [[_TMP1]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK12-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
// CHECK12-NEXT: [[DIV:%.*]] = sdiv i32 [[TMP5]], 1
// CHECK12-NEXT: store i32 [[DIV]], ptr [[TMP4]], align 4
@@ -2660,11 +2660,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK17-NEXT: store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
// CHECK17-NEXT: store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT: [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT: [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK17-NEXT: [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
// CHECK17-NEXT: [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5:![0-9]+]]
// CHECK17-NEXT: [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
// CHECK17-NEXT: store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
// CHECK17-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2751,8 +2751,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK17-NEXT: [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK17-NEXT: [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT: [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META3]], !align [[META5]]
+// CHECK17-NEXT: [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK17-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
// CHECK17-NEXT: [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
// CHECK17-NEXT: [[TMP8:%.*]] = add nuw i64 [[TMP7]], 127
diff --git a/clang/test/OpenMP/sections_firstprivate_codegen.cpp b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
index 7c6d1839fb10e..32c5826e6f75d 100644
--- a/clang/test/OpenMP/sections_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/sections_firstprivate_codegen.cpp
@@ -448,7 +448,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -517,10 +517,10 @@ int main() {
// CHECK1-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
@@ -668,7 +668,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -996,7 +996,7 @@ int main() {
// CHECK4-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
// CHECK4-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
// CHECK4-NEXT: store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK4-NEXT: store i32 0, ptr [[DOTOMP_SECTIONS_LB_]], align 4
// CHECK4-NEXT: store i32 1, ptr [[DOTOMP_SECTIONS_UB_]], align 4
// CHECK4-NEXT: store i32 1, ptr [[DOTOMP_SECTIONS_ST_]], align 4
diff --git a/clang/test/OpenMP/single_firstprivate_codegen.cpp b/clang/test/OpenMP/single_firstprivate_codegen.cpp
index 27cd220adf225..31ea1ca4952fc 100644
--- a/clang/test/OpenMP/single_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/single_firstprivate_codegen.cpp
@@ -403,7 +403,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -467,10 +467,10 @@ int main() {
// CHECK1-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[T_VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
// CHECK1-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
// CHECK1-NEXT: [[TMP6:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP5]])
@@ -585,7 +585,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META3]], !align [[META4]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -908,7 +908,7 @@ int main() {
// CHECK4-NEXT: store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
// CHECK4-NEXT: store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
// CHECK4-NEXT: store ptr [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8
+// CHECK4-NEXT: [[TMP0:%.*]] = load ptr, ptr [[SIVAR_ADDR]], align 8, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]]
// CHECK4-NEXT: [[TMP1:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
// CHECK4-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
// CHECK4-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_single(ptr @[[GLOB1]], i32 [[TMP2]])
diff --git a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
index aa50d8fb3aabd..a171827a18646 100644
--- a/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_firstprivate_codegen.cpp
@@ -354,9 +354,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -396,9 +396,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -550,12 +550,12 @@ int main() {
// CHECK1-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
// CHECK1-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
// CHECK1-NEXT: store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK1-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK1-NEXT: store ptr [[VEC]], ptr [[TMP4]], align 8
// CHECK1-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -655,7 +655,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -711,14 +711,14 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK1-NEXT: ret void
//
@@ -752,9 +752,9 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -776,7 +776,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done4:
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -815,7 +815,7 @@ int main() {
// CHECK1-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
// CHECK1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
// CHECK1-NEXT: store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK1-NEXT: [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
// CHECK1-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -912,7 +912,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1155,9 +1155,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1197,9 +1197,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1349,12 +1349,12 @@ int main() {
// CHECK3-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
// CHECK3-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
// CHECK3-NEXT: store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK3-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK3-NEXT: store ptr [[VEC]], ptr [[TMP4]], align 4
// CHECK3-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1454,7 +1454,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1510,14 +1510,14 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK3-NEXT: ret void
//
@@ -1551,9 +1551,9 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1575,7 +1575,7 @@ int main() {
// CHECK3-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK3-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK3: omp.arraycpy.done4:
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1613,7 +1613,7 @@ int main() {
// CHECK3-NEXT: [[TMP14:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
// CHECK3-NEXT: store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
// CHECK3-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1709,7 +1709,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1877,7 +1877,7 @@ int main() {
// CHECK9-NEXT: [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK9-NEXT: store i32 [[TMP0]], ptr [[G_CASTED]], align 4
// CHECK9-NEXT: [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
// CHECK9-NEXT: [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
// CHECK9-NEXT: store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1943,13 +1943,13 @@ int main() {
// CHECK9-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
// CHECK9-NEXT: store i32 [[ADD]], ptr [[I]], align 4
// CHECK9-NEXT: store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: store volatile i32 1, ptr [[TMP8]], align 4
// CHECK9-NEXT: store i32 2, ptr [[SIVAR_ADDR]], align 4
// CHECK9-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK9-NEXT: store ptr [[G_ADDR]], ptr [[TMP9]], align 8
// CHECK9-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: store ptr [[TMP11]], ptr [[TMP10]], align 8
// CHECK9-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK9-NEXT: store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
index 78e40e54671ac..678770520f677 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -415,9 +415,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -459,9 +459,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -616,9 +616,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -741,12 +741,12 @@ int main() {
// CHECK1-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
// CHECK1-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
// CHECK1-NEXT: store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK1-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK1-NEXT: store ptr [[VEC]], ptr [[TMP4]], align 8
// CHECK1-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -846,7 +846,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -902,14 +902,14 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK1-NEXT: ret void
//
@@ -944,9 +944,9 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -968,7 +968,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done4:
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1005,7 +1005,7 @@ int main() {
// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
// CHECK1-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK1: omp.inner.for.inc:
@@ -1082,9 +1082,9 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1112,7 +1112,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done5:
-// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1151,7 +1151,7 @@ int main() {
// CHECK1-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
// CHECK1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
// CHECK1-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[I]], align 4
// CHECK1-NEXT: [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
// CHECK1-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1233,7 +1233,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1476,9 +1476,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1520,9 +1520,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1675,9 +1675,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1796,12 +1796,12 @@ int main() {
// CHECK3-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
// CHECK3-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
// CHECK3-NEXT: store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK3-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK3-NEXT: store ptr [[VEC]], ptr [[TMP4]], align 4
// CHECK3-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1901,7 +1901,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1957,14 +1957,14 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK3-NEXT: ret void
//
@@ -1999,9 +1999,9 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2023,7 +2023,7 @@ int main() {
// CHECK3-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK3-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK3: omp.arraycpy.done4:
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2058,7 +2058,7 @@ int main() {
// CHECK3-NEXT: [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT: [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
// CHECK3-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK3: omp.inner.for.inc:
@@ -2135,9 +2135,9 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2163,7 +2163,7 @@ int main() {
// CHECK3-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
// CHECK3-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK3: omp.arraycpy.done4:
-// CHECK3-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2201,7 +2201,7 @@ int main() {
// CHECK3-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
// CHECK3-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP18:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
// CHECK3-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2282,7 +2282,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2450,7 +2450,7 @@ int main() {
// CHECK5-NEXT: [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK5-NEXT: store i32 [[TMP0]], ptr [[G_CASTED]], align 4
// CHECK5-NEXT: [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
// CHECK5-NEXT: [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
// CHECK5-NEXT: store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
// CHECK5-NEXT: [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2520,7 +2520,7 @@ int main() {
// CHECK5-NEXT: [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK5-NEXT: store i32 [[TMP11]], ptr [[G_CASTED]], align 4
// CHECK5-NEXT: [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK5-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK5-NEXT: [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
// CHECK5-NEXT: store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
// CHECK5-NEXT: [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2607,13 +2607,13 @@ int main() {
// CHECK5-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
// CHECK5-NEXT: store i32 [[ADD]], ptr [[I]], align 4
// CHECK5-NEXT: store i32 1, ptr [[G_ADDR]], align 4
-// CHECK5-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK5-NEXT: store volatile i32 1, ptr [[TMP10]], align 4
// CHECK5-NEXT: store i32 2, ptr [[SIVAR_ADDR]], align 4
// CHECK5-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK5-NEXT: store ptr [[G_ADDR]], ptr [[TMP11]], align 8
// CHECK5-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK5-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK5-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK5-NEXT: store ptr [[TMP13]], ptr [[TMP12]], align 8
// CHECK5-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK5-NEXT: store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
@@ -2659,9 +2659,9 @@ int main() {
// CHECK13-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK13-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK13-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK13-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -2703,9 +2703,9 @@ int main() {
// CHECK13-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK13-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK13-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
// CHECK13-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -2860,9 +2860,9 @@ int main() {
// CHECK13-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK13-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK13-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK13-NEXT: [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -2987,14 +2987,14 @@ int main() {
// CHECK13-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK13-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK13-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK13-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK13-NEXT: ret void
//
@@ -3029,9 +3029,9 @@ int main() {
// CHECK13-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK13-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK13-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3053,7 +3053,7 @@ int main() {
// CHECK13-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK13-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK13: omp.arraycpy.done4:
-// CHECK13-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK13-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK13-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3090,7 +3090,7 @@ int main() {
// CHECK13-NEXT: [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK13-NEXT: store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
// CHECK13-NEXT: [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK13-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK13-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
// CHECK13-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK13: omp.inner.for.inc:
@@ -3167,9 +3167,9 @@ int main() {
// CHECK13-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK13-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK13-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK13-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK13-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3197,7 +3197,7 @@ int main() {
// CHECK13-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
// CHECK13-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
// CHECK13: omp.arraycpy.done5:
-// CHECK13-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK13-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
// CHECK13-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
// CHECK13-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR3]]
@@ -3236,7 +3236,7 @@ int main() {
// CHECK13-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
// CHECK13-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
// CHECK13-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK13-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK13-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: [[TMP18:%.*]] = load i32, ptr [[I]], align 4
// CHECK13-NEXT: [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
// CHECK13-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -3321,7 +3321,7 @@ int main() {
// CHECK13-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK13-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK13-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK13-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK13-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3352,7 +3352,7 @@ int main() {
// CHECK13-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK13-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK13-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK13-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK13-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK13-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK13-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -3379,9 +3379,9 @@ int main() {
// CHECK15-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK15-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9:![0-9]+]], !align [[META10:![0-9]+]]
+// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK15-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK15-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -3423,9 +3423,9 @@ int main() {
// CHECK15-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK15-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK15-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
// CHECK15-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -3578,9 +3578,9 @@ int main() {
// CHECK15-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK15-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK15-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK15-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -3701,14 +3701,14 @@ int main() {
// CHECK15-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK15-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK15-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK15-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK15-NEXT: ret void
//
@@ -3743,9 +3743,9 @@ int main() {
// CHECK15-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK15-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK15-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -3767,7 +3767,7 @@ int main() {
// CHECK15-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK15-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK15: omp.arraycpy.done4:
-// CHECK15-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK15-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK15-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3802,7 +3802,7 @@ int main() {
// CHECK15-NEXT: [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK15-NEXT: store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
// CHECK15-NEXT: [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK15-NEXT: [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT: [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l81.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
// CHECK15-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK15: omp.inner.for.inc:
@@ -3879,9 +3879,9 @@ int main() {
// CHECK15-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK15-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
+// CHECK15-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK15-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK15-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -3907,7 +3907,7 @@ int main() {
// CHECK15-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
// CHECK15-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK15: omp.arraycpy.done4:
-// CHECK15-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK15-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK15-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
// CHECK15-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR3]]
@@ -3945,7 +3945,7 @@ int main() {
// CHECK15-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK15-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
// CHECK15-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK15-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK15-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: [[TMP18:%.*]] = load i32, ptr [[I]], align 4
// CHECK15-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
// CHECK15-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -4029,7 +4029,7 @@ int main() {
// CHECK15-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK15-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK15-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK15-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK15-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4060,7 +4060,7 @@ int main() {
// CHECK15-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK15-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK15-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK15-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META9]], !align [[META10]]
// CHECK15-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK15-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK15-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -4089,7 +4089,7 @@ int main() {
// CHECK17-NEXT: [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK17-NEXT: store i32 [[TMP0]], ptr [[G_CASTED]], align 4
// CHECK17-NEXT: [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
// CHECK17-NEXT: [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
// CHECK17-NEXT: store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
// CHECK17-NEXT: [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4159,7 +4159,7 @@ int main() {
// CHECK17-NEXT: [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK17-NEXT: store i32 [[TMP11]], ptr [[G_CASTED]], align 4
// CHECK17-NEXT: [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK17-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
// CHECK17-NEXT: [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
// CHECK17-NEXT: store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
// CHECK17-NEXT: [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -4246,13 +4246,13 @@ int main() {
// CHECK17-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
// CHECK17-NEXT: store i32 [[ADD]], ptr [[I]], align 4
// CHECK17-NEXT: store i32 1, ptr [[G_ADDR]], align 4
-// CHECK17-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
// CHECK17-NEXT: store volatile i32 1, ptr [[TMP10]], align 4
// CHECK17-NEXT: store i32 2, ptr [[SIVAR_ADDR]], align 4
// CHECK17-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK17-NEXT: store ptr [[G_ADDR]], ptr [[TMP11]], align 8
// CHECK17-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK17-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK17-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META6]], !align [[META7]]
// CHECK17-NEXT: store ptr [[TMP13]], ptr [[TMP12]], align 8
// CHECK17-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK17-NEXT: store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
index 294fcba7872b3..f3c9565a17656 100644
--- a/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_firstprivate_codegen.cpp
@@ -357,9 +357,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -399,9 +399,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -553,14 +553,14 @@ int main() {
// CHECK1-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
// CHECK1-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
// CHECK1-NEXT: store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK1-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK1-NEXT: store i64 [[TMP2]], ptr [[TMP6]], align 8
// CHECK1-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -660,7 +660,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -716,14 +716,14 @@ int main() {
// CHECK1-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK1-NEXT: ret void
//
@@ -757,9 +757,9 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -781,7 +781,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done4:
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -820,7 +820,7 @@ int main() {
// CHECK1-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP14]] to i64
// CHECK1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i64 0, i64 [[IDXPROM]]
// CHECK1-NEXT: store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK1-NEXT: [[IDXPROM9:%.*]] = sext i32 [[TMP16]] to i64
// CHECK1-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i64 0, i64 [[IDXPROM9]]
@@ -917,7 +917,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1160,9 +1160,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1202,9 +1202,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1354,14 +1354,14 @@ int main() {
// CHECK3-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
// CHECK3-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
// CHECK3-NEXT: store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK3-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK3-NEXT: store i32 [[TMP2]], ptr [[TMP6]], align 4
// CHECK3-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1461,7 +1461,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1517,14 +1517,14 @@ int main() {
// CHECK3-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB2]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK3-NEXT: ret void
//
@@ -1558,9 +1558,9 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1582,7 +1582,7 @@ int main() {
// CHECK3-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK3-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK3: omp.arraycpy.done4:
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -1620,7 +1620,7 @@ int main() {
// CHECK3-NEXT: [[TMP14:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP14]]
// CHECK3-NEXT: store i32 [[TMP13]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT: [[TMP15:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP16]]
// CHECK3-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP15]], i32 4, i1 false)
@@ -1716,7 +1716,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1884,7 +1884,7 @@ int main() {
// CHECK9-NEXT: [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK9-NEXT: store i32 [[TMP0]], ptr [[G_CASTED]], align 4
// CHECK9-NEXT: [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
// CHECK9-NEXT: [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
// CHECK9-NEXT: store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -1950,13 +1950,13 @@ int main() {
// CHECK9-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
// CHECK9-NEXT: store i32 [[ADD]], ptr [[I]], align 4
// CHECK9-NEXT: store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: store volatile i32 1, ptr [[TMP8]], align 4
// CHECK9-NEXT: store i32 2, ptr [[SIVAR_ADDR]], align 4
// CHECK9-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK9-NEXT: store ptr [[G_ADDR]], ptr [[TMP9]], align 8
// CHECK9-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP11:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: store ptr [[TMP11]], ptr [[TMP10]], align 8
// CHECK9-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK9-NEXT: store ptr [[SIVAR_ADDR]], ptr [[TMP12]], align 8
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
index d742b0a85af42..037aa12d57226 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_firstprivate_codegen.cpp
@@ -391,9 +391,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7:![0-9]+]], !align [[META8:![0-9]+]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -435,9 +435,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -592,9 +592,9 @@ int main() {
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK1-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK1-NEXT: [[TMP3:%.*]] = load i64, ptr [[DOTPREVIOUS_LB__ADDR]], align 8
@@ -717,14 +717,14 @@ int main() {
// CHECK1-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i64 1
// CHECK1-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef signext 2)
// CHECK1-NEXT: store ptr [[TEST]], ptr [[VAR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK1-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP2:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
-// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK1-NEXT: store i64 [[TMP2]], ptr [[TMP6]], align 8
// CHECK1-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -824,7 +824,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -880,14 +880,14 @@ int main() {
// CHECK1-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i64 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK1-NEXT: ret void
//
@@ -922,9 +922,9 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -946,7 +946,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done4:
-// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -983,7 +983,7 @@ int main() {
// CHECK1-NEXT: [[TMP16:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK1-NEXT: store i32 [[TMP16]], ptr [[T_VAR_CASTED]], align 4
// CHECK1-NEXT: [[TMP17:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
-// CHECK1-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8
+// CHECK1-NEXT: [[TMP18:%.*]] = load ptr, ptr [[_TMP7]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i64 [[TMP13]], i64 [[TMP15]], ptr [[VEC2]], i64 [[TMP17]], ptr [[S_ARR3]], ptr [[TMP18]])
// CHECK1-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK1: omp.inner.for.inc:
@@ -1060,9 +1060,9 @@ int main() {
// CHECK1-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK1-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
+// CHECK1-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 8
// CHECK1-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK1-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -1090,7 +1090,7 @@ int main() {
// CHECK1-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
// CHECK1-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE5]], label [[OMP_ARRAYCPY_BODY]]
// CHECK1: omp.arraycpy.done5:
-// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK1-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]])
// CHECK1-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR6]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP7]])
// CHECK1-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP7]]) #[[ATTR2]]
@@ -1129,7 +1129,7 @@ int main() {
// CHECK1-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
// CHECK1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC3]], i64 0, i64 [[IDXPROM]]
// CHECK1-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8
+// CHECK1-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP8]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[TMP18:%.*]] = load i32, ptr [[I]], align 4
// CHECK1-NEXT: [[IDXPROM10:%.*]] = sext i32 [[TMP18]] to i64
// CHECK1-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR4]], i64 0, i64 [[IDXPROM10]]
@@ -1211,7 +1211,7 @@ int main() {
// CHECK1-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK1-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK1-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK1-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META7]], !align [[META8]]
// CHECK1-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK1-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK1-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1454,9 +1454,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1498,9 +1498,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_STRIDE]], align 4
@@ -1653,9 +1653,9 @@ int main() {
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[DOTPREVIOUS_LB__ADDR]], align 4
@@ -1774,14 +1774,14 @@ int main() {
// CHECK3-NEXT: [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[S_ARR]], i32 1
// CHECK3-NEXT: call void @_ZN1SIiEC1Ei(ptr noundef nonnull align 4 dereferenceable(4) [[ARRAYINIT_ELEMENT]], i32 noundef 2)
// CHECK3-NEXT: store ptr [[TEST]], ptr [[VAR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VAR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP0]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[T_VAR]], align 4
// CHECK3-NEXT: store i32 [[TMP1]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP2:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP3:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_BASEPTRS]], i32 0, i32 0
// CHECK3-NEXT: store i32 [[TMP2]], ptr [[TMP6]], align 4
// CHECK3-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x ptr], ptr [[DOTOFFLOAD_PTRS]], i32 0, i32 0
@@ -1881,7 +1881,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1937,14 +1937,14 @@ int main() {
// CHECK3-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP5:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_teams(ptr @[[GLOB3]], i32 4, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined, ptr [[TMP0]], i32 [[TMP4]], ptr [[TMP1]], ptr [[TMP5]])
// CHECK3-NEXT: ret void
//
@@ -1979,9 +1979,9 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_COMB_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_COMB_UB]], align 4
@@ -2003,7 +2003,7 @@ int main() {
// CHECK3-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP3]]
// CHECK3-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK3: omp.arraycpy.done4:
-// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP4:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP4]], ptr noundef [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2038,7 +2038,7 @@ int main() {
// CHECK3-NEXT: [[TMP14:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store i32 [[TMP14]], ptr [[T_VAR_CASTED]], align 4
// CHECK3-NEXT: [[TMP15:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
-// CHECK3-NEXT: [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT: [[TMP16:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB3]], i32 6, ptr @{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z5tmainIiET_v_l56.omp_outlined.omp_outlined, i32 [[TMP12]], i32 [[TMP13]], ptr [[VEC2]], i32 [[TMP15]], ptr [[S_ARR3]], ptr [[TMP16]])
// CHECK3-NEXT: br label [[OMP_INNER_FOR_INC:%.*]]
// CHECK3: omp.inner.for.inc:
@@ -2115,9 +2115,9 @@ int main() {
// CHECK3-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK3-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK3-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: store ptr [[TMP2]], ptr [[TMP]], align 4
// CHECK3-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
// CHECK3-NEXT: store i32 1, ptr [[DOTOMP_UB]], align 4
@@ -2143,7 +2143,7 @@ int main() {
// CHECK3-NEXT: [[OMP_ARRAYCPY_DONE:%.*]] = icmp eq ptr [[OMP_ARRAYCPY_DEST_ELEMENT]], [[TMP5]]
// CHECK3-NEXT: br i1 [[OMP_ARRAYCPY_DONE]], label [[OMP_ARRAYCPY_DONE4]], label [[OMP_ARRAYCPY_BODY]]
// CHECK3: omp.arraycpy.done4:
-// CHECK3-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4
+// CHECK3-NEXT: [[TMP6:%.*]] = load ptr, ptr [[TMP]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: call void @_ZN2StC1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN1SIiEC1ERKS0_2St(ptr noundef nonnull align 4 dereferenceable(4) [[VAR5]], ptr noundef nonnull align 4 dereferenceable(4) [[TMP6]], ptr noundef [[AGG_TMP6]])
// CHECK3-NEXT: call void @_ZN2StD1Ev(ptr noundef nonnull align 4 dereferenceable(8) [[AGG_TMP6]]) #[[ATTR2]]
@@ -2181,7 +2181,7 @@ int main() {
// CHECK3-NEXT: [[TMP16:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[VEC2]], i32 0, i32 [[TMP16]]
// CHECK3-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
-// CHECK3-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4
+// CHECK3-NEXT: [[TMP17:%.*]] = load ptr, ptr [[_TMP7]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[TMP18:%.*]] = load i32, ptr [[I]], align 4
// CHECK3-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR3]], i32 0, i32 [[TMP18]]
// CHECK3-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[ARRAYIDX9]], ptr align 4 [[TMP17]], i32 4, i1 false)
@@ -2262,7 +2262,7 @@ int main() {
// CHECK3-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK3-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK3-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK3-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK3-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK3-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK3-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -2430,7 +2430,7 @@ int main() {
// CHECK9-NEXT: [[TMP0:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK9-NEXT: store i32 [[TMP0]], ptr [[G_CASTED]], align 4
// CHECK9-NEXT: [[TMP1:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5:![0-9]+]], !align [[META6:![0-9]+]]
// CHECK9-NEXT: [[TMP3:%.*]] = load volatile i32, ptr [[TMP2]], align 4
// CHECK9-NEXT: store i32 [[TMP3]], ptr [[G1_CASTED]], align 4
// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2500,7 +2500,7 @@ int main() {
// CHECK9-NEXT: [[TMP11:%.*]] = load i32, ptr [[G_ADDR]], align 4
// CHECK9-NEXT: store i32 [[TMP11]], ptr [[G_CASTED]], align 4
// CHECK9-NEXT: [[TMP12:%.*]] = load i64, ptr [[G_CASTED]], align 8
-// CHECK9-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: [[TMP14:%.*]] = load volatile i32, ptr [[TMP13]], align 4
// CHECK9-NEXT: store i32 [[TMP14]], ptr [[G1_CASTED]], align 4
// CHECK9-NEXT: [[TMP15:%.*]] = load i64, ptr [[G1_CASTED]], align 8
@@ -2587,13 +2587,13 @@ int main() {
// CHECK9-NEXT: [[ADD:%.*]] = add nsw i32 0, [[MUL]]
// CHECK9-NEXT: store i32 [[ADD]], ptr [[I]], align 4
// CHECK9-NEXT: store i32 1, ptr [[G_ADDR]], align 4
-// CHECK9-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: store volatile i32 1, ptr [[TMP10]], align 4
// CHECK9-NEXT: store i32 2, ptr [[SIVAR_ADDR]], align 4
// CHECK9-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 0
// CHECK9-NEXT: store ptr [[G_ADDR]], ptr [[TMP11]], align 8
// CHECK9-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 1
-// CHECK9-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8
+// CHECK9-NEXT: [[TMP13:%.*]] = load ptr, ptr [[TMP]], align 8, !nonnull [[META5]], !align [[META6]]
// CHECK9-NEXT: store ptr [[TMP13]], ptr [[TMP12]], align 8
// CHECK9-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw [[CLASS_ANON_0]], ptr [[REF_TMP]], i32 0, i32 2
// CHECK9-NEXT: store ptr [[SIVAR_ADDR]], ptr [[TMP14]], align 8
diff --git a/clang/test/OpenMP/teams_firstprivate_codegen.cpp b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
index 6f21c9e31bd8d..fec8fcb78f91e 100644
--- a/clang/test/OpenMP/teams_firstprivate_codegen.cpp
+++ b/clang/test/OpenMP/teams_firstprivate_codegen.cpp
@@ -458,9 +458,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK9-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11:![0-9]+]], !align [[META12:![0-9]+]]
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
// CHECK9-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK9-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -493,9 +493,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
// CHECK9-NEXT: store i64 [[SIVAR]], ptr [[SIVAR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
// CHECK9-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i64 8, i1 false)
// CHECK9-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
// CHECK9-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i64 2
@@ -808,7 +808,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK9-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK9-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
// CHECK9-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK9-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK9-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -872,9 +872,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 8
// CHECK9-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
// CHECK9-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK9-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK9-NEXT: [[TMP4:%.*]] = load i64, ptr [[T_VAR_CASTED]], align 8
@@ -902,9 +902,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store i64 [[T_VAR]], ptr [[T_VAR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 8
// CHECK9-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8
-// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8
-// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
+// CHECK9-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
// CHECK9-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i64 8, i1 false)
// CHECK9-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
// CHECK9-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i64 2
@@ -1032,7 +1032,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK9-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 8
// CHECK9-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK9-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8
+// CHECK9-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 8, !nonnull [[META11]], !align [[META12]]
// CHECK9-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK9-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK9-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1250,9 +1250,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK11-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK11-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12:![0-9]+]], !align [[META13:![0-9]+]]
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
// CHECK11-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK11-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK11-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1285,9 +1285,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK11-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
// CHECK11-NEXT: store i32 [[SIVAR]], ptr [[SIVAR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
// CHECK11-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[VEC1]], ptr align 4 [[TMP0]], i32 8, i1 false)
// CHECK11-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S], ptr [[S_ARR2]], i32 0, i32 0
// CHECK11-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1600,7 +1600,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK11-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK11-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
// CHECK11-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S]], ptr [[TMP0]], i32 0, i32 0
// CHECK11-NEXT: [[TMP1:%.*]] = load float, ptr [[F2]], align 4
// CHECK11-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1664,9 +1664,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store ptr [[VEC]], ptr [[VEC_ADDR]], align 4
// CHECK11-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK11-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
// CHECK11-NEXT: [[TMP3:%.*]] = load i32, ptr [[T_VAR_ADDR]], align 4
// CHECK11-NEXT: store i32 [[TMP3]], ptr [[T_VAR_CASTED]], align 4
// CHECK11-NEXT: [[TMP4:%.*]] = load i32, ptr [[T_VAR_CASTED]], align 4
@@ -1694,9 +1694,9 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store i32 [[T_VAR]], ptr [[T_VAR_ADDR]], align 4
// CHECK11-NEXT: store ptr [[S_ARR]], ptr [[S_ARR_ADDR]], align 4
// CHECK11-NEXT: store ptr [[VAR]], ptr [[VAR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4
-// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4
-// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[VEC_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP1:%.*]] = load ptr, ptr [[S_ARR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
+// CHECK11-NEXT: [[TMP2:%.*]] = load ptr, ptr [[VAR_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
// CHECK11-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 128 [[VEC1]], ptr align 128 [[TMP0]], i32 8, i1 false)
// CHECK11-NEXT: [[ARRAY_BEGIN:%.*]] = getelementptr inbounds [2 x %struct.S.0], ptr [[S_ARR2]], i32 0, i32 0
// CHECK11-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_S_0]], ptr [[ARRAY_BEGIN]], i32 2
@@ -1824,7 +1824,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK11-NEXT: store ptr [[T]], ptr [[T_INDIRECT_ADDR]], align 4
// CHECK11-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK11-NEXT: [[F:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0:%.*]], ptr [[THIS1]], i32 0, i32 0
-// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4
+// CHECK11-NEXT: [[TMP0:%.*]] = load ptr, ptr [[S_ADDR]], align 4, !nonnull [[META12]], !align [[META13]]
// CHECK11-NEXT: [[F2:%.*]] = getelementptr inbounds nuw [[STRUCT_S_0]], ptr [[TMP0]], i32 0, i32 0
// CHECK11-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
// CHECK11-NEXT: [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_ST:%.*]], ptr [[T]], i32 0, i32 0
@@ -1997,7 +1997,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK17-NEXT: [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
// CHECK17-NEXT: [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 8
// CHECK17-NEXT: [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
// CHECK17-NEXT: [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 8
@@ -2031,11 +2031,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK17-NEXT: store i64 [[VLA2]], ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: store i64 [[VLA4]], ptr [[VLA_ADDR5]], align 8
// CHECK17-NEXT: store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT: [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT: [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10:![0-9]+]]
// CHECK17-NEXT: [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
// CHECK17-NEXT: [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK17-NEXT: [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
// CHECK17-NEXT: store ptr [[TMP5]], ptr [[SAVED_STACK]], align 8
// CHECK17-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2235,7 +2235,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK17-NEXT: [[TMP0:%.*]] = load i64, ptr [[VLA_ADDR]], align 8
// CHECK17-NEXT: [[TMP1:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
+// CHECK17-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK17-NEXT: [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 8
// CHECK17-NEXT: [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 8
@@ -2273,8 +2273,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK17-NEXT: [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
// CHECK17-NEXT: [[TMP2:%.*]] = load i64, ptr [[VLA_ADDR3]], align 8
// CHECK17-NEXT: [[TMP3:%.*]] = load i64, ptr [[VLA_ADDR5]], align 8
-// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8
-// CHECK17-NEXT: [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8
+// CHECK17-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 8, !nonnull [[META8]], !align [[META9]]
+// CHECK17-NEXT: [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 8, !nonnull [[META8]], !align [[META10]]
// CHECK17-NEXT: [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
// CHECK17-NEXT: store ptr [[TMP6]], ptr [[SAVED_STACK]], align 8
// CHECK17-NEXT: [[TMP7:%.*]] = mul nuw i64 [[TMP2]], [[TMP3]]
@@ -2460,7 +2460,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK19-NEXT: [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
// CHECK19-NEXT: [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
// CHECK19-NEXT: [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8:![0-9]+]], !align [[META9:![0-9]+]]
// CHECK19-NEXT: [[TMP4:%.*]] = load ptr, ptr [[S_ADDR]], align 4
// CHECK19-NEXT: [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
// CHECK19-NEXT: [[TMP6:%.*]] = load ptr, ptr [[A_ADDR]], align 4
@@ -2494,11 +2494,11 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK19-NEXT: store i32 [[VLA2]], ptr [[VLA_ADDR3]], align 4
// CHECK19-NEXT: store i32 [[VLA4]], ptr [[VLA_ADDR5]], align 4
// CHECK19-NEXT: store ptr [[VLA26]], ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT: [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT: [[TMP0:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK19-NEXT: [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
// CHECK19-NEXT: [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
// CHECK19-NEXT: [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK19-NEXT: [[TMP5:%.*]] = call ptr @llvm.stacksave.p0()
// CHECK19-NEXT: store ptr [[TMP5]], ptr [[SAVED_STACK]], align 4
// CHECK19-NEXT: [[TMP6:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
@@ -2696,7 +2696,7 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK19-NEXT: [[TMP0:%.*]] = load i32, ptr [[VLA_ADDR]], align 4
// CHECK19-NEXT: [[TMP1:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
// CHECK19-NEXT: [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
+// CHECK19-NEXT: [[TMP3:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK19-NEXT: [[TMP4:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK19-NEXT: [[TMP5:%.*]] = load ptr, ptr [[VLA1_ADDR]], align 4
// CHECK19-NEXT: [[TMP6:%.*]] = load ptr, ptr [[S_ADDR]], align 4
@@ -2734,8 +2734,8 @@ void array_func(float a[3], St s[2], int n, long double vla1[n]) {
// CHECK19-NEXT: [[TMP1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 4
// CHECK19-NEXT: [[TMP2:%.*]] = load i32, ptr [[VLA_ADDR3]], align 4
// CHECK19-NEXT: [[TMP3:%.*]] = load i32, ptr [[VLA_ADDR5]], align 4
-// CHECK19-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4
-// CHECK19-NEXT: [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4
+// CHECK19-NEXT: [[TMP4:%.*]] = load ptr, ptr [[VLA2_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
+// CHECK19-NEXT: [[TMP5:%.*]] = load ptr, ptr [[N_ADDR]], align 4, !nonnull [[META8]], !align [[META9]]
// CHECK19-NEXT: [[TMP6:%.*]] = call ptr @llvm.stacksave.p0()
// CHECK19-NEXT: store ptr [[TMP6]], ptr [[SAVED_STACK]], align 4
// CHECK19-NEXT: [[TMP7:%.*]] = mul nuw i32 [[TMP2]], [[TMP3]]
diff --git a/clang/test/Sema/dllexport.c b/clang/test/Sema/dllexport.c
index 3f911fb095c0f..5f6ff36e290e9 100644
--- a/clang/test/Sema/dllexport.c
+++ b/clang/test/Sema/dllexport.c
@@ -2,6 +2,10 @@
// RUN: %clang_cc1 -triple x86_64-win32 -fsyntax-only -fms-extensions -verify -std=c11 %s
// RUN: %clang_cc1 -triple i686-mingw32 -fsyntax-only -fms-extensions -verify -std=c11 %s
// RUN: %clang_cc1 -triple x86_64-mingw32 -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple i686-windows-itanium -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c11 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5 -fsyntax-only -fms-extensions -verify -std=c99 %s
+// RUN: %clang_cc1 -triple x86_64-sie-ps5 -fsyntax-only -fms-extensions -verify -std=c11 %s
// Invalid usage.
__declspec(dllexport) typedef int typedef1;
diff --git a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
index 46c3670848529..135865c8450f5 100644
--- a/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
+++ b/clang/test/SemaCXX/overload-resolution-deferred-templates.cpp
@@ -283,3 +283,31 @@ void f() {
}
#endif
+
+namespace GH147374 {
+
+struct String {};
+template <typename T> void operator+(T, String &&) = delete;
+
+struct Bar {
+ void operator+(String) const; // expected-note {{candidate function}}
+ friend void operator+(Bar, String) {}; // expected-note {{candidate function}}
+};
+
+struct Baz {
+ void operator+(String); // expected-note {{candidate function}}
+ friend void operator+(Baz, String) {}; // expected-note {{candidate function}}
+};
+
+void test() {
+ Bar a;
+ String b;
+ a + b;
+ //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Bar' and 'String')}}
+
+ Baz z;
+ z + b;
+ //expected-error at -1 {{use of overloaded operator '+' is ambiguous (with operand types 'Baz' and 'String')}}
+}
+
+}
diff --git a/clang/test/SemaTemplate/concepts-using-decl.cpp b/clang/test/SemaTemplate/concepts-using-decl.cpp
index fca69dea5c88f..41f7b6d2f8faa 100644
--- a/clang/test/SemaTemplate/concepts-using-decl.cpp
+++ b/clang/test/SemaTemplate/concepts-using-decl.cpp
@@ -176,3 +176,24 @@ void func() {
f.foo<10, 10>(); // expected-error {{no matching member function for call to 'foo'}}
}
} // namespace heads_without_concepts.
+
+namespace GH146614 {
+
+template <typename T>
+struct base {
+ template <typename A>
+ void foo(A x)
+ requires (requires{x;})
+ {}
+};
+
+
+struct child : base<int> {
+ using base<int>::foo;
+ template <typename A>
+ void foo(A x)
+ requires (false)
+ {}
+};
+
+}
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 4241d12601242..5f0783f869bf6 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -1466,8 +1466,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
assert(falseTarget && "missing conditional branch false block");
mlir::Location loc = toLocation();
mlir::Value bcc = builder->createConvert(loc, builder->getI1Type(), cond);
- builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget, std::nullopt,
- falseTarget, std::nullopt);
+ builder->create<mlir::cf::CondBranchOp>(loc, bcc, trueTarget,
+ mlir::ValueRange{}, falseTarget,
+ mlir::ValueRange{});
}
void genConditionalBranch(mlir::Value cond,
Fortran::lower::pft::Evaluation *trueTarget,
@@ -2556,8 +2557,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
builder->setInsertionPointToEnd(loopWrapperOp.getBody());
auto loopOp = builder->create<fir::DoConcurrentLoopOp>(
loc, nestLBs, nestUBs, nestSts, /*loopAnnotation=*/nullptr,
- /*local_vars=*/std::nullopt,
- /*local_syms=*/nullptr, /*reduce_vars=*/std::nullopt,
+ /*local_vars=*/mlir::ValueRange{},
+ /*local_syms=*/nullptr, /*reduce_vars=*/mlir::ValueRange{},
/*reduce_byref=*/nullptr, /*reduce_syms=*/nullptr,
/*reduce_attrs=*/nullptr);
@@ -3810,9 +3811,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
mlir::Block *selectCaseBlock = insertBlock(blockList[0]);
mlir::Block *assumedSizeBlock =
rankStarBlock ? rankStarBlock : defaultBlock;
- builder->create<mlir::cf::CondBranchOp>(loc, isAssumedSize,
- assumedSizeBlock, std::nullopt,
- selectCaseBlock, std::nullopt);
+ builder->create<mlir::cf::CondBranchOp>(
+ loc, isAssumedSize, assumedSizeBlock, mlir::ValueRange{},
+ selectCaseBlock, mlir::ValueRange{});
startBlock(selectCaseBlock);
}
// Create fir.select_case for the other rank cases.
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 1850b67898126..b8ab5d09e3e08 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -303,7 +303,7 @@ createStringLitOp(fir::FirOpBuilder &builder, mlir::Location loc,
mlir::NamedAttribute sizeAttr(sizeTag, builder.getI64IntegerAttr(len));
llvm::SmallVector<mlir::NamedAttribute> attrs = {dataAttr, sizeAttr};
return builder.create<fir::StringLitOp>(
- loc, llvm::ArrayRef<mlir::Type>{type}, std::nullopt, attrs);
+ loc, llvm::ArrayRef<mlir::Type>{type}, mlir::ValueRange{}, attrs);
}
}
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 0a1cd67789772..281ab229d1b6a 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1003,9 +1003,9 @@ class ScalarExprLowering {
},
[&](const fir::MutableBoxValue &toBox) {
if (toBox.isPointer()) {
- Fortran::lower::associateMutableBox(converter, loc, toBox, expr,
- /*lbounds=*/std::nullopt,
- stmtCtx);
+ Fortran::lower::associateMutableBox(
+ converter, loc, toBox, expr,
+ /*lbounds=*/mlir::ValueRange{}, stmtCtx);
return;
}
// For allocatable components, a deep copy is needed.
@@ -3604,8 +3604,9 @@ class ArrayExprLowering {
mlir::Value castTo =
builder.createConvert(loc, fir::HeapType::get(seqTy), load);
mlir::Value shapeOp = builder.genShape(loc, shape);
- return builder.create<fir::ArrayLoadOp>(
- loc, seqTy, castTo, shapeOp, /*slice=*/mlir::Value{}, std::nullopt);
+ return builder.create<fir::ArrayLoadOp>(loc, seqTy, castTo, shapeOp,
+ /*slice=*/mlir::Value{},
+ mlir::ValueRange{});
};
// Custom lowering of the element store to deal with the extra indirection
// to the lazy allocated buffer.
@@ -4207,7 +4208,7 @@ class ArrayExprLowering {
auto addr =
builder->create<fir::ArrayCoorOp>(loc, eleRefTy, tmp, shape,
/*slice=*/mlir::Value{}, indices,
- /*typeParams=*/std::nullopt);
+ /*typeParams=*/mlir::ValueRange{});
auto load = builder->create<fir::LoadOp>(loc, addr);
return builder->createConvert(loc, i1Ty, load);
};
@@ -4522,17 +4523,18 @@ class ArrayExprLowering {
fir::isRecordWithAllocatableMember(eleTy))
TODO(loc, "creating an array temp where the element type has "
"allocatable members");
- mlir::Value temp = !seqTy.hasDynamicExtents()
- ? builder.create<fir::AllocMemOp>(loc, type)
- : builder.create<fir::AllocMemOp>(
- loc, type, ".array.expr", std::nullopt, shape);
+ mlir::Value temp =
+ !seqTy.hasDynamicExtents()
+ ? builder.create<fir::AllocMemOp>(loc, type)
+ : builder.create<fir::AllocMemOp>(loc, type, ".array.expr",
+ mlir::ValueRange{}, shape);
fir::FirOpBuilder *bldr = &converter.getFirOpBuilder();
stmtCtx.attachCleanup(
[bldr, loc, temp]() { bldr->create<fir::FreeMemOp>(loc, temp); });
mlir::Value shapeOp = genShapeOp(shape);
return builder.create<fir::ArrayLoadOp>(loc, seqTy, temp, shapeOp,
/*slice=*/mlir::Value{},
- std::nullopt);
+ mlir::ValueRange{});
}
static fir::ShapeOp genShapeOp(mlir::Location loc, fir::FirOpBuilder &builder,
@@ -6483,7 +6485,7 @@ class ArrayExprLowering {
mlir::Value initBuffSz =
builder.createIntegerConstant(loc, idxTy, clInitialBufferSize);
mem = builder.create<fir::AllocMemOp>(
- loc, eleTy, /*typeparams=*/std::nullopt, initBuffSz);
+ loc, eleTy, /*typeparams=*/mlir::ValueRange{}, initBuffSz);
builder.create<fir::StoreOp>(loc, initBuffSz, buffSize);
}
} else {
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 6cda742874ccf..cacf4e249aa28 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -813,6 +813,10 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
baseTy = boxTy.getEleTy();
baseTy = fir::unwrapRefType(baseTy);
+
+ if (mlir::isa<fir::SequenceType>(baseTy))
+ TODO(loc, "array of derived-type with device component");
+
auto recTy =
mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
assert(recTy && "expected fir::RecordType");
@@ -823,7 +827,7 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
if (Fortran::semantics::IsDeviceAllocatable(sym)) {
unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
mlir::Type fieldTy;
- std::vector<mlir::Value> coordinates;
+ llvm::SmallVector<mlir::Value> coordinates;
if (fieldIdx != std::numeric_limits<unsigned>::max()) {
// Field found in the base record type.
@@ -866,8 +870,24 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
TODO(loc, "device resident component in complex derived-type "
"hierarchy");
- mlir::Value comp = builder.create<fir::CoordinateOp>(
- loc, builder.getRefType(fieldTy), fir::getBase(exv), coordinates);
+ mlir::Value base = fir::getBase(exv);
+ mlir::Value comp;
+ if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
+ mlir::Value box = builder.create<fir::LoadOp>(loc, base);
+ mlir::Value addr = builder.create<fir::BoxAddrOp>(loc, box);
+ llvm::SmallVector<mlir::Value> lenParams;
+ assert(coordinates.size() == 1 && "expect one coordinate");
+ auto field = mlir::dyn_cast<fir::FieldIndexOp>(
+ coordinates[0].getDefiningOp());
+ comp = builder.create<hlfir::DesignateOp>(
+ loc, builder.getRefType(fieldTy), addr,
+ /*component=*/field.getFieldName(),
+ /*componentShape=*/mlir::Value{},
+ hlfir::DesignateOp::Subscripts{});
+ } else {
+ comp = builder.create<fir::CoordinateOp>(
+ loc, builder.getRefType(fieldTy), base, coordinates);
+ }
cuf::DataAttributeAttr dataAttr =
Fortran::lower::translateSymbolCUFDataAttribute(
builder.getContext(), sym);
diff --git a/flang/lib/Lower/Runtime.cpp b/flang/lib/Lower/Runtime.cpp
index 2be5ef76e46b8..5f73335242336 100644
--- a/flang/lib/Lower/Runtime.cpp
+++ b/flang/lib/Lower/Runtime.cpp
@@ -134,7 +134,7 @@ void Fortran::lower::genFailImageStatement(
mlir::Location loc = converter.getCurrentLocation();
mlir::func::FuncOp callee =
fir::runtime::getRuntimeFunc<mkRTKey(FailImageStatement)>(loc, builder);
- builder.create<fir::CallOp>(loc, callee, std::nullopt);
+ builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
genUnreachable(builder, loc);
}
@@ -199,7 +199,7 @@ void Fortran::lower::genPauseStatement(
mlir::Location loc = converter.getCurrentLocation();
mlir::func::FuncOp callee =
fir::runtime::getRuntimeFunc<mkRTKey(PauseStatement)>(loc, builder);
- builder.create<fir::CallOp>(loc, callee, std::nullopt);
+ builder.create<fir::CallOp>(loc, callee, mlir::ValueRange{});
}
void Fortran::lower::genPointerAssociate(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Lower/VectorSubscripts.cpp b/flang/lib/Lower/VectorSubscripts.cpp
index 389a89ddcf102..c7b3e11728cea 100644
--- a/flang/lib/Lower/VectorSubscripts.cpp
+++ b/flang/lib/Lower/VectorSubscripts.cpp
@@ -122,7 +122,7 @@ class VectorSubscriptBoxBuilder {
TODO(loc, "threading length parameters in field index op");
fir::FirOpBuilder &builder = converter.getFirOpBuilder();
componentPath.emplace_back(builder.create<fir::FieldIndexOp>(
- loc, fldTy, componentName, recTy, /*typeParams*/ std::nullopt));
+ loc, fldTy, componentName, recTy, /*typeParams=*/mlir::ValueRange{}));
return fir::unwrapSequenceType(recTy.getType(componentName));
}
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index acd5a88a2582d..5b1dbc4435d6c 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -620,7 +620,7 @@ fir::StringLitOp fir::FirOpBuilder::createStringLitOp(mlir::Location loc,
mlir::NamedAttribute sizeAttr(sizeTag, getI64IntegerAttr(data.size()));
llvm::SmallVector<mlir::NamedAttribute> attrs{dataAttr, sizeAttr};
return create<fir::StringLitOp>(loc, llvm::ArrayRef<mlir::Type>{type},
- std::nullopt, attrs);
+ mlir::ValueRange{}, attrs);
}
mlir::Value fir::FirOpBuilder::genShape(mlir::Location loc,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
index 773d6408079cc..04703f7911176 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Intrinsics.cpp
@@ -59,7 +59,8 @@ mlir::Value fir::runtime::genCpuTime(fir::FirOpBuilder &builder,
mlir::Location loc) {
mlir::func::FuncOp func =
fir::runtime::getRuntimeFunc<mkRTKey(CpuTime)>(loc, builder);
- return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+ return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+ .getResult(0);
}
void fir::runtime::genDateAndTime(fir::FirOpBuilder &builder,
@@ -280,7 +281,8 @@ void fir::runtime::genRename(fir::FirOpBuilder &builder, mlir::Location loc,
mlir::Value fir::runtime::genTime(fir::FirOpBuilder &builder,
mlir::Location loc) {
auto func = fir::runtime::getRuntimeFunc<mkRTKey(time)>(loc, builder);
- return builder.create<fir::CallOp>(loc, func, std::nullopt).getResult(0);
+ return builder.create<fir::CallOp>(loc, func, mlir::ValueRange{})
+ .getResult(0);
}
/// generate runtime call to transfer intrinsic with no size argument
diff --git a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
index 411181cc6dd1c..9b5e43b80b1f6 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Stop.cpp
@@ -25,7 +25,7 @@ void fir::runtime::genExit(fir::FirOpBuilder &builder, mlir::Location loc,
void fir::runtime::genAbort(fir::FirOpBuilder &builder, mlir::Location loc) {
mlir::func::FuncOp abortFunc =
fir::runtime::getRuntimeFunc<mkRTKey(Abort)>(loc, builder);
- builder.create<fir::CallOp>(loc, abortFunc, std::nullopt);
+ builder.create<fir::CallOp>(loc, abortFunc, mlir::ValueRange{});
}
void fir::runtime::genReportFatalUserError(fir::FirOpBuilder &builder,
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 03cc92e975b19..c5cf01ed98357 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -405,7 +405,7 @@ void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
mlir::Location loc = forallMaskOp.getLoc();
mlir::Value mask = generateYieldedScalarValue(forallMaskOp.getMaskRegion(),
builder.getI1Type());
- auto ifOp = builder.create<fir::IfOp>(loc, std::nullopt, mask, false);
+ auto ifOp = builder.create<fir::IfOp>(loc, mlir::TypeRange{}, mask, false);
builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
constructStack.push_back(ifOp);
}
@@ -530,7 +530,7 @@ void OrderedAssignmentRewriter::generateMaskIfOp(mlir::Value cdt) {
mlir::Location loc = cdt.getLoc();
cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
cdt = builder.createConvert(loc, builder.getI1Type(), cdt);
- auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), std::nullopt, cdt,
+ auto ifOp = builder.create<fir::IfOp>(cdt.getLoc(), mlir::TypeRange{}, cdt,
/*withElseRegion=*/false);
constructStack.push_back(ifOp.getOperation());
builder.setInsertionPointToStart(&ifOp.getThenRegion().front());
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 1f8edf851de9b..bc4fcd8b0112e 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -222,7 +222,7 @@ void AllocaReplaceImpl::genIndirectDeallocation(
rewriter.create<fir::ConvertOp>(loc, intPtrTy, ptrVal);
mlir::Value isAllocated = rewriter.create<mlir::arith::CmpIOp>(
loc, mlir::arith::CmpIPredicate::ne, ptrToInt, c0);
- auto ifOp = rewriter.create<fir::IfOp>(loc, std::nullopt, isAllocated,
+ auto ifOp = rewriter.create<fir::IfOp>(loc, mlir::TypeRange{}, isAllocated,
/*withElseRegion=*/false);
rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
mlir::Value cast = fir::factory::createConvert(
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index 57eae1ff052a2..6e45aae4246d0 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -456,7 +456,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
rewriter.setInsertionPointToEnd(thisBlock);
if (destOps.has_value())
rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, destOps.value(),
- newBlock, std::nullopt);
+ newBlock, mlir::ValueRange{});
else
rewriter.create<mlir::cf::CondBranchOp>(loc, cmp, dest, newBlock);
rewriter.setInsertionPointToEnd(newBlock);
diff --git a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
index 506c8e66dbdfa..ad8464b495888 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyFIROperations.cpp
@@ -261,7 +261,7 @@ class DoConcurrentConversion
innermostUnorderdLoop = rewriter.create<fir::DoLoopOp>(
doConcurentOp.getLoc(), lb, ub, st,
/*unordred=*/true, /*finalCountValue=*/false,
- /*iterArgs=*/std::nullopt, loop.getReduceVars(),
+ /*iterArgs=*/mlir::ValueRange{}, loop.getReduceVars(),
loop.getReduceAttrsAttr());
ivArgs.push_back(innermostUnorderdLoop.getInductionVar());
rewriter.setInsertionPointToStart(innermostUnorderdLoop.getBody());
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index ee89ea38a3fc7..e3bb181f65398 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -21,4 +21,36 @@ contains
! CHECK: %[[Z:.*]] = fir.coordinate_of %[[DT]]#0, z : (!fir.ref<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
! CHECK: cuf.set_allocator_idx %[[Z]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+ subroutine sub2()
+ type(ty_device), pointer :: d1
+ end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub2()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub2Ed1"} -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMm1Fsub2Ed1"} : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
+ subroutine sub3()
+ type(ty_device), allocatable :: d1
+ end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub3()
+! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub3Ed1"} -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMm1Fsub3Ed1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
+! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
end module
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index 98270adaa7c73..59808779aa6ef 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -48,7 +48,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
mlir::Value addr = builder->create<fir::AllocaOp>(loc, eleType);
auto name = mlir::StringAttr::get(&context, "x");
auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
- /*shape=*/mlir::Value{}, /*typeParams=*/std::nullopt,
+ /*shape=*/mlir::Value{}, /*typeParams=*/mlir::ValueRange{},
/*dummy_scope=*/nullptr, name,
/*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
/*data_attr=*/cuf::DataAttributeAttr{});
@@ -102,11 +102,11 @@ TEST_F(FortranVariableTest, SimpleArray) {
extents.size(), fir::SequenceType::getUnknownExtent());
mlir::Type seqTy = fir::SequenceType::get(typeShape, eleType);
mlir::Value addr = builder->create<fir::AllocaOp>(
- loc, seqTy, /*pinned=*/false, /*typeParams=*/std::nullopt, extents);
+ loc, seqTy, /*pinned=*/false, /*typeParams=*/mlir::ValueRange{}, extents);
mlir::Value shape = createShape(extents);
auto name = mlir::StringAttr::get(&context, "x");
auto declare = builder->create<fir::DeclareOp>(loc, addr.getType(), addr,
- shape, /*typeParams*/ std::nullopt, /*dummy_scope=*/nullptr, name,
+ shape, /*typeParams=*/mlir::ValueRange{}, /*dummy_scope=*/nullptr, name,
/*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
/*data_attr=*/cuf::DataAttributeAttr{});
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index e210992c5111a..3fb62788c1168 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -571,6 +571,8 @@ function(add_integration_test test_name)
target_compile_options(${fq_build_target_name} PRIVATE
${compile_options} ${INTEGRATION_TEST_COMPILE_OPTIONS})
+ set(compiler_runtime "")
+
if(LIBC_TARGET_ARCHITECTURE_IS_AMDGPU)
target_link_options(${fq_build_target_name} PRIVATE
${LIBC_COMPILE_OPTIONS_DEFAULT} ${INTEGRATION_TEST_COMPILE_OPTIONS}
@@ -599,17 +601,19 @@ function(add_integration_test test_name)
set(link_options
-nolibc
-nostartfiles
- -static
+ -nostdlib
${LIBC_LINK_OPTIONS_DEFAULT}
${LIBC_TEST_LINK_OPTIONS_DEFAULT}
)
target_link_options(${fq_build_target_name} PRIVATE ${link_options})
+ list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
endif()
target_link_libraries(
${fq_build_target_name}
- ${fq_target_name}.__libc__
libc.startup.${LIBC_TARGET_OS}.crt1
libc.test.IntegrationTest.test
+ ${fq_target_name}.__libc__
+ ${compiler_runtime}
)
add_dependencies(${fq_build_target_name}
libc.test.IntegrationTest.test
@@ -770,6 +774,7 @@ function(add_libc_hermetic test_name)
${HERMETIC_TEST_COMPILE_OPTIONS})
set(link_libraries "")
+ set(compiler_runtime "")
foreach(lib IN LISTS HERMETIC_TEST_LINK_LIBRARIES)
if(TARGET ${lib}.hermetic)
list(APPEND link_libraries ${lib}.hermetic)
@@ -807,12 +812,12 @@ function(add_libc_hermetic test_name)
set(link_options
-nolibc
-nostartfiles
- -static
+ -nostdlib
${LIBC_LINK_OPTIONS_DEFAULT}
${LIBC_TEST_LINK_OPTIONS_DEFAULT}
)
target_link_options(${fq_build_target_name} PRIVATE ${link_options})
- list(APPEND link_libraries ${LIBGCC_S_LOCATION})
+ list(APPEND compiler_runtime ${LIBGCC_S_LOCATION})
endif()
target_link_libraries(
${fq_build_target_name}
@@ -820,7 +825,9 @@ function(add_libc_hermetic test_name)
libc.startup.${LIBC_TARGET_OS}.crt1
${link_libraries}
LibcHermeticTestSupport.hermetic
- ${fq_target_name}.__libc__)
+ ${fq_target_name}.__libc__
+ ${compiler_runtime}
+ )
add_dependencies(${fq_build_target_name}
LibcTest.hermetic
libc.test.UnitTest.ErrnoSetterMatcher
diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in
index fc01aaf2d8746..b68c0c8258366 100644
--- a/libcxx/include/__config_site.in
+++ b/libcxx/include/__config_site.in
@@ -30,7 +30,6 @@
#cmakedefine01 _LIBCPP_HAS_LOCALIZATION
#cmakedefine01 _LIBCPP_HAS_UNICODE
#cmakedefine01 _LIBCPP_HAS_WIDE_CHARACTERS
-#cmakedefine _LIBCPP_HAS_NO_STD_MODULES
#cmakedefine01 _LIBCPP_HAS_TIME_ZONE_DATABASE
#cmakedefine01 _LIBCPP_INSTRUMENTED_WITH_ASAN
diff --git a/libcxx/test/libcxx/fuzzing/random.pass.cpp b/libcxx/test/libcxx/fuzzing/random.pass.cpp
index cb074bd60fdc8..f0256a01f29ae 100644
--- a/libcxx/test/libcxx/fuzzing/random.pass.cpp
+++ b/libcxx/test/libcxx/fuzzing/random.pass.cpp
@@ -6,9 +6,10 @@
//
//===----------------------------------------------------------------------===//
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
// UNSUPPORTED: c++03, c++11
diff --git a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
index 1ba0063c1dada..f9f81d22ff80e 100644
--- a/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
+++ b/libcxx/test/std/depr/depr.c.headers/math_h.pass.cpp
@@ -6,9 +6,10 @@
//
//===----------------------------------------------------------------------===//
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
// <math.h>
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index 48c2918802fc3..8d261e9fcbdb2 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -6,9 +6,10 @@
//
//===----------------------------------------------------------------------===//
-// This test fails because Clang no longer enables -fdelayed-template-parsing
-// by default on Windows with C++20 (#69431).
-// XFAIL: msvc && (clang-18 || clang-19 || clang-20 || clang-21)
+// This doesn't work on Windows because in the MSVC UCRT headers the math.h is
+// actually intended to implement the full C++ spec requirements. For details
+// see https://github.com/llvm/llvm-project/issues/70225#issuecomment-1992528828
+// XFAIL: msvc
// <cmath>
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index f464b1d4518a4..06152046d13d4 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -76,10 +76,10 @@ DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
if (!d)
return;
auto *sec = dyn_cast_or_null<InputSection>(d->section);
- // Skip empty, discarded, ICF folded sections. Skipping ICF folded sections
- // reduces duplicate detection work in BPSectionOrderer.
+ // Skip empty, discarded, ICF folded sections, .bss. Skipping ICF folded
+ // sections reduces duplicate detection work in BPSectionOrderer.
if (!sec || sec->size == 0 || !sec->isLive() || sec->repl != sec ||
- !orderer.secToSym.try_emplace(sec, d).second)
+ !sec->content().data() || !orderer.secToSym.try_emplace(sec, d).second)
return;
rootSymbolToSectionIdxs[CachedHashStringRef(
lld::utils::getRootSymbol(sym.getName()))]
diff --git a/lld/test/ELF/bp-section-orderer.s b/lld/test/ELF/bp-section-orderer.s
index 4df2e8d43022e..438d7c2da0f76 100644
--- a/lld/test/ELF/bp-section-orderer.s
+++ b/lld/test/ELF/bp-section-orderer.s
@@ -26,28 +26,28 @@
# RUN: ld.lld -o out.s a.o --irpgo-profile=a.profdata --bp-startup-sort=function
# RUN: llvm-nm -jn out.s | tr '\n' , | FileCheck %s --check-prefix=STARTUP
-# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,{{$}}
+# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,g1,{{$}}
# RUN: ld.lld -o out.os a.o --irpgo-profile=a.profdata --bp-startup-sort=function --symbol-ordering-file a.txt
# RUN: llvm-nm -jn out.os | tr '\n' , | FileCheck %s --check-prefix=ORDER-STARTUP
-# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,{{$}}
+# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,g1,{{$}}
# RUN: ld.lld -o out.cf a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-FUNC
# RUN: ld.lld -o out.cf.icf a.o --verbose-bp-section-orderer --bp-compression-sort=function --icf=all --gc-sections 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-ICF-FUNC
# RUN: llvm-nm -jn out.cf | tr '\n' , | FileCheck %s --check-prefix=CFUNC
-# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,{{$}}
+# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,g1,{{$}}
# RUN: ld.lld -o out.cd a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-DATA
# RUN: llvm-nm -jn out.cd | tr '\n' , | FileCheck %s --check-prefix=CDATA
-# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
# RUN: ld.lld -o out.cb a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
# RUN: llvm-nm -jn out.cb | tr '\n' , | FileCheck %s --check-prefix=CBOTH
-# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,{{$}}
+# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,g1,{{$}}
# RUN: ld.lld -o out.cbs a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
# RUN: llvm-nm -jn out.cbs | tr '\n' , | FileCheck %s --check-prefix=CBOTH-STARTUP
-# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,{{$}}
+# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
# BP-COMPRESSION-FUNC: Ordered 9 sections ([[#]] bytes) using balanced partitioning
# BP-COMPRESSION-ICF-FUNC: Ordered 8 sections ([[#]] bytes) using balanced partitioning
@@ -108,6 +108,7 @@ d3
d2
#--- a.c
+int g1;
const char s5[] = "engineering";
const char s4[] = "computer program";
const char s3[] = "hardware engineer";
@@ -377,6 +378,14 @@ d1:
.word 6 // 0x6
.size d1, 16
+ .type g1, at object // @g1
+ .section .bss.g1,"aw", at nobits
+ .globl g1
+ .p2align 2, 0x0
+g1:
+ .word 0 // 0x0
+ .size g1, 4
+
.section ".note.GNU-stack","", at progbits
.addrsig
.addrsig_sym F
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index e445fa8833022..6f812b91a8b1d 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -700,7 +700,7 @@ void IRExecutionUnit::CollectCandidateCPlusPlusNames(
class LoadAddressResolver {
public:
- LoadAddressResolver(Target *target, bool &symbol_was_missing_weak)
+ LoadAddressResolver(Target &target, bool &symbol_was_missing_weak)
: m_target(target), m_symbol_was_missing_weak(symbol_was_missing_weak) {}
std::optional<lldb::addr_t> Resolve(SymbolContextList &sc_list) {
@@ -722,11 +722,11 @@ class LoadAddressResolver {
// First try the symbol.
if (candidate_sc.symbol) {
- load_address = candidate_sc.symbol->ResolveCallableAddress(*m_target);
+ load_address = candidate_sc.symbol->ResolveCallableAddress(m_target);
if (load_address == LLDB_INVALID_ADDRESS) {
Address addr = candidate_sc.symbol->GetAddress();
- load_address = m_target->GetProcessSP()
- ? addr.GetLoadAddress(m_target)
+ load_address = m_target.GetProcessSP()
+ ? addr.GetLoadAddress(&m_target)
: addr.GetFileAddress();
}
}
@@ -734,8 +734,8 @@ class LoadAddressResolver {
// If that didn't work, try the function.
if (load_address == LLDB_INVALID_ADDRESS && candidate_sc.function) {
Address addr = candidate_sc.function->GetAddress();
- load_address = m_target->GetProcessSP() ? addr.GetLoadAddress(m_target)
- : addr.GetFileAddress();
+ load_address = m_target.GetProcessSP() ? addr.GetLoadAddress(&m_target)
+ : addr.GetFileAddress();
}
// We found a load address.
@@ -766,7 +766,7 @@ class LoadAddressResolver {
}
private:
- Target *m_target;
+ Target &m_target;
bool &m_symbol_was_missing_weak;
lldb::addr_t m_best_internal_load_address = LLDB_INVALID_ADDRESS;
};
@@ -790,7 +790,7 @@ IRExecutionUnit::FindInSymbols(const std::vector<ConstString> &names,
for (size_t i = 0; i < m_preferred_modules.GetSize(); ++i)
non_local_images.Remove(m_preferred_modules.GetModuleAtIndex(i));
- LoadAddressResolver resolver(target, symbol_was_missing_weak);
+ LoadAddressResolver resolver(*target, symbol_was_missing_weak);
ModuleFunctionSearchOptions function_options;
function_options.include_symbols = true;
diff --git a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
index e1d7e42bdd1a9..cd60227572be4 100644
--- a/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
+++ b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
@@ -2,7 +2,6 @@
Test lldb data formatter subsystem.
"""
-
import lldb
from lldbsuite.test.decorators import *
from lldbsuite.test.lldbtest import *
@@ -120,6 +119,8 @@ def cleanup():
'@"2 elements"',
],
)
+
+ self.runCmd("settings set target.max-children-depth 6")
self.expect(
"frame variable mutabledict --ptr-depth 3",
substrs=[
diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
index 91de55f4ade4a..43018eacf709b 100644
--- a/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
+++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/UdtLayoutTest.script
@@ -1,3 +1,4 @@
+settings set target.max-children-depth 10
breakpoint set --file UdtLayoutTest.cpp --line 60
run
target variable
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index de5f66ce1584c..87d2a9ac3bf88 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -263,6 +263,20 @@ amaclean at nvidia.com (email), [AlexMaclean](https://github.com/AlexMaclean) (GitH
#### PowerPC backend
+Amy Kwan (esp. release issues) \
+Amy.Kwan1 at ibm.com (email), [amy-kwan](https://github.com/amy-kwan) (GitHub) \
+Lei Huang \
+lei at ca.ibm.com (email), [lei137](https://github.com/lei137) (GitHub) \
+Sean Fertile (esp. ABI/ELF/XCOFF) \
+sfertile at ca.ibm.com (email), [mandlebug](https://github.com/mandlebug) (GitHub) \
+Zhijian Lin \
+zhijian at ca.ibm.com (email), [diggerlin](https://github.com/diggerlin) (GitHub) \
+Maryam Moghadas \
+maryammo at ca.ibm.com (email), [maryammo](https://github.com/maryammo) (GitHub) \
+Roland Froese \
+froese at ca.ibm.com (email), [RolandF77](https://github.com/RolandF77) (GitHub) \
+llvmonpower \
+powerllvm at ca.ibm.com (email), [llvmonpower](https://github.com/llvmonpower) (GitHub)
#### RISCV backend
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index c614a6d7ace9e..732227b98ab9e 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -30,7 +30,7 @@ because the naming and other conventions are dictated by the C++ standard.
There are some conventions that are not uniformly followed in the code base
(e.g. the naming convention). This is because they are relatively new, and a
-lot of code was written before they were put in place. Our long term goal is
+lot of code was written before they were put in place. Our long-term goal is
for the entire codebase to follow the convention, but we explicitly *do not*
want patches that do large-scale reformatting of existing code. On the other
hand, it is reasonable to rename the methods of a class if you're about to
@@ -50,7 +50,7 @@ code imported into the tree. Generally, our preference is for standards
conforming, modern, and portable C++ code as the implementation language of
choice.
-For automation, build-systems and utility scripts Python is preferred and
+For automation, build-systems, and utility scripts, Python is preferred and
is widely used in the LLVM repository already.
C++ Standard Versions
@@ -92,7 +92,7 @@ LLVM support libraries (for example, `ADT
<https://github.com/llvm/llvm-project/tree/main/llvm/include/llvm/ADT>`_)
implement specialized data structures or functionality missing in the standard
library. Such libraries are usually implemented in the ``llvm`` namespace and
-follow the expected standard interface, when there is one.
+follow the expected standard interface when there is one.
When both C++ and the LLVM support libraries provide similar functionality, and
there isn't a specific reason to favor the C++ implementation, it is generally
@@ -325,8 +325,8 @@ implementation file. In any case, implementation files can include additional
comments (not necessarily in Doxygen markup) to explain implementation details
as needed.
-Don't duplicate function or class name at the beginning of the comment.
-For humans it is obvious which function or class is being documented;
+Don't duplicate the function or class name at the beginning of the comment.
+For humans, it is obvious which function or class is being documented;
automatic documentation processing tools are smart enough to bind the comment
to the correct declaration.
@@ -369,7 +369,7 @@ lower-case letter, and finish the last sentence without a period, if it would
end in one otherwise. Sentences which end with different punctuation, such as
"did you forget ';'?", should still do so.
-For example this is a good error message:
+For example, this is a good error message:
.. code-block:: none
@@ -443,7 +443,7 @@ Write your code to fit within 80 columns.
There must be some limit to the width of the code in
order to allow developers to have multiple files side-by-side in
windows on a modest display. If you are going to pick a width limit, it is
-somewhat arbitrary but you might as well pick something standard. Going with 90
+somewhat arbitrary, but you might as well pick something standard. Going with 90
columns (for example) instead of 80 columns wouldn't add any significant value
and would be detrimental to printing out code. Also many other projects have
standardized on 80 columns, so some people have already configured their editors
@@ -520,7 +520,7 @@ within each other and within function calls in order to build up aggregates
The historically common formatting of braced initialization of aggregate
variables does not mix cleanly with deep nesting, general expression contexts,
function arguments, and lambdas. We suggest new code use a simple rule for
-formatting braced initialization lists: act as-if the braces were parentheses
+formatting braced initialization lists: act as if the braces were parentheses
in a function call. The formatting rules exactly match those already well
understood for formatting nested function calls. Examples:
@@ -607,11 +607,11 @@ Static constructors and destructors (e.g., global variables whose types have a
constructor or destructor) should not be added to the code base, and should be
removed wherever possible.
-Globals in different source files are initialized in `arbitrary order
+Globals in different source files are initialized in an `arbitrary order
<https://yosefk.com/c++fqa/ctors.html#fqa-10.12>`_, making the code more
difficult to reason about.
-Static constructors have negative impact on launch time of programs that use
+Static constructors have a negative impact on the launch time of programs that use
LLVM as a library. We would really like for there to be zero cost for linking
in an additional LLVM target or other library into an application, but static
constructors undermine this goal.
@@ -698,7 +698,7 @@ If you use a braced initializer list when initializing a variable, use an equals
Use ``auto`` Type Deduction to Make Code More Readable
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Some are advocating a policy of "almost always ``auto``" in C++11, however LLVM
+Some are advocating a policy of "almost always ``auto``" in C++11; however, LLVM
uses a more moderate stance. Use ``auto`` if and only if it makes the code more
readable or easier to maintain. Don't "almost always" use ``auto``, but do use
``auto`` with initializers like ``cast<Foo>(...)`` or other places where the
@@ -783,14 +783,14 @@ guards, and might not include their prerequisites. Name such files with the
In general, a header should be implemented by one or more ``.cpp`` files. Each
of these ``.cpp`` files should include the header that defines their interface
-first. This ensures that all of the dependences of the header have been
+first. This ensures that all of the dependencies of the header have been
properly added to the header itself, and are not implicit. System headers
should be included after user headers for a translation unit.
Library Layering
^^^^^^^^^^^^^^^^
-A directory of header files (for example ``include/llvm/Foo``) defines a
+A directory of header files (for example, ``include/llvm/Foo``) defines a
library (``Foo``). One library (both
its headers and implementation) should only use things from the libraries
listed in its dependencies.
@@ -822,7 +822,7 @@ especially in header files.
But wait! Sometimes you need to have the definition of a class to use it, or to
inherit from it. In these cases go ahead and ``#include`` that header file. Be
-aware however that there are many cases where you don't need to have the full
+aware, however, that there are many cases where you don't need to have the full
definition of a class. If you are using a pointer or reference to a class, you
don't need the header file. If you are simply returning a class instance from a
prototyped function or method, you don't need it. In fact, for most cases, you
@@ -970,7 +970,7 @@ loops. A silly example is something like this:
When you have very, very small loops, this sort of structure is fine. But if it
exceeds more than 10-15 lines, it becomes difficult for people to read and
understand at a glance. The problem with this sort of code is that it gets very
-nested very quickly. Meaning that the reader of the code has to keep a lot of
+nested very quickly. This means that the reader of the code has to keep a lot of
context in their brain to remember what is going immediately on in the loop,
because they don't know if/when the ``if`` conditions will have ``else``\s etc.
It is strongly preferred to structure the loop like this:
@@ -988,7 +988,7 @@ It is strongly preferred to structure the loop like this:
...
}
-This has all the benefits of using early exits for functions: it reduces nesting
+This has all the benefits of using early exits for functions: it reduces the nesting
of the loop, it makes it easier to describe why the conditions are true, and it
makes it obvious to the reader that there is no ``else`` coming up that they
have to push context into their brain for. If a loop is large, this can be a
@@ -1149,12 +1149,12 @@ In general, names should be in camel case (e.g. ``TextFileReader`` and
nouns and start with an upper-case letter (e.g. ``TextFileReader``).
* **Variable names** should be nouns (as they represent state). The name should
- be camel case, and start with an upper case letter (e.g. ``Leader`` or
+ be camel case, and start with an upper-case letter (e.g. ``Leader`` or
``Boats``).
* **Function names** should be verb phrases (as they represent actions), and
command-like function should be imperative. The name should be camel case,
- and start with a lower case letter (e.g. ``openFile()`` or ``isFoo()``).
+ and start with a lower-case letter (e.g. ``openFile()`` or ``isFoo()``).
* **Enum declarations** (e.g. ``enum Foo {...}``) are types, so they should
follow the naming conventions for types. A common use for enums is as a
@@ -1207,7 +1207,7 @@ Assert Liberally
^^^^^^^^^^^^^^^^
Use the "``assert``" macro to its fullest. Check all of your preconditions and
-assumptions, you never know when a bug (not necessarily even yours) might be
+assumptions. You never know when a bug (not necessarily even yours) might be
caught early by an assertion, which reduces debugging time dramatically. The
"``<cassert>``" header file is probably already included by the header files you
are using, so it doesn't cost anything to use it.
@@ -1302,7 +1302,7 @@ preferred to write the code like this:
assert(NewToSet && "The value shouldn't be in the set yet");
In C code where ``[[maybe_unused]]`` is not supported, use ``void`` cast to
-suppress unused variable warning as follows:
+suppress an unused variable warning as follows:
.. code-block:: c
@@ -1546,7 +1546,7 @@ whenever possible.
The semantics of postincrement include making a copy of the value being
incremented, returning it, and then preincrementing the "work value". For
primitive types, this isn't a big deal. But for iterators, it can be a huge
-issue (for example, some iterators contains stack and set objects in them...
+issue (for example, some iterators contain stack and set objects in them...
copying an iterator could invoke the copy ctor's of these as well). In general,
get in the habit of always using preincrement, and you won't have a problem.
@@ -1663,7 +1663,7 @@ Don't Use Braces on Simple Single-Statement Bodies of if/else/loop Statements
When writing the body of an ``if``, ``else``, or for/while loop statement, we
prefer to omit the braces to avoid unnecessary line noise. However, braces
-should be used in cases where the omission of braces harm the readability and
+should be used in cases where the omission of braces harms the readability and
maintainability of the code.
We consider that readability is harmed when omitting the brace in the presence
@@ -1763,7 +1763,7 @@ would help to avoid running into a "dangling else" situation.
handleAttrOnDecl(D, A, i);
}
- // Use braces on the outer block because of a nested `if`; otherwise the
+ // Use braces on the outer block because of a nested `if`; otherwise, the
// compiler would warn: `add explicit braces to avoid dangling else`
if (auto *D = dyn_cast<FunctionDecl>(D)) {
if (shouldProcess(D))
diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index aef91909dd17b..794075201d646 100644
--- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -39,8 +39,8 @@ class MachineBlockFrequencyInfo {
public:
LLVM_ABI MachineBlockFrequencyInfo(); // Legacy pass manager only.
LLVM_ABI explicit MachineBlockFrequencyInfo(
- MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
- MachineLoopInfo &MLI);
+ const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+ const MachineLoopInfo &MLI);
LLVM_ABI MachineBlockFrequencyInfo(MachineBlockFrequencyInfo &&);
LLVM_ABI ~MachineBlockFrequencyInfo();
diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h
index 77cee875f16e7..f8241a3cdf160 100644
--- a/llvm/include/llvm/IR/DebugInfo.h
+++ b/llvm/include/llvm/IR/DebugInfo.h
@@ -115,8 +115,7 @@ class DebugInfoFinder {
LLVM_ABI void processVariable(DILocalVariable *DVI);
/// Process debug info location.
LLVM_ABI void processLocation(const Module &M, const DILocation *Loc);
- /// Process a DbgRecord (e.g, treat a DbgVariableRecord like a
- /// DbgVariableIntrinsic).
+ /// Process a DbgRecord.
LLVM_ABI void processDbgRecord(const Module &M, const DbgRecord &DR);
/// Process subprogram.
@@ -290,8 +289,6 @@ struct VarRecord {
DILocalVariable *Var;
DILocation *DL;
- VarRecord(DbgVariableIntrinsic *DVI)
- : Var(DVI->getVariable()), DL(getDebugValueLoc(DVI)) {}
VarRecord(DbgVariableRecord *DVR)
: Var(DVR->getVariable()), DL(getDebugValueLoc(DVR)) {}
VarRecord(DILocalVariable *Var, DILocation *DL) : Var(Var), DL(DL) {}
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 9345f95015301..f1f0c18949c35 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -66,7 +66,6 @@ namespace dwarf {
enum Tag : uint16_t;
}
-class DbgVariableIntrinsic;
class DbgVariableRecord;
LLVM_ABI extern cl::opt<bool> EnableFSDiscriminator;
@@ -4613,7 +4612,6 @@ class DebugVariable {
LLVM_ABI static const FragmentInfo DefaultFragment;
public:
- LLVM_ABI DebugVariable(const DbgVariableIntrinsic *DII);
LLVM_ABI DebugVariable(const DbgVariableRecord *DVR);
DebugVariable(const DILocalVariable *Var,
@@ -4681,7 +4679,6 @@ template <> struct DenseMapInfo<DebugVariable> {
/// information).
class DebugVariableAggregate : public DebugVariable {
public:
- LLVM_ABI DebugVariableAggregate(const DbgVariableIntrinsic *DVI);
DebugVariableAggregate(const DebugVariable &V)
: DebugVariable(V.getVariable(), std::nullopt, V.getInlinedAt()) {}
};
diff --git a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
index 4e5da81a7e885..9500b1f160ea9 100644
--- a/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/llvm/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -29,7 +29,6 @@ template <typename T> class SSAUpdaterTraits;
class Type;
class Use;
class Value;
-class DbgValueInst;
/// Helper class for SSA formation on a set of values defined in
/// multiple blocks.
@@ -122,8 +121,6 @@ class SSAUpdater {
/// the instruction. Anything outside of its block will have its
/// value set to the new SSA value if available, and undef if not.
void UpdateDebugValues(Instruction *I);
- void UpdateDebugValues(Instruction *I,
- SmallVectorImpl<DbgValueInst *> &DbgValues);
void UpdateDebugValues(Instruction *I,
SmallVectorImpl<DbgVariableRecord *> &DbgValues);
@@ -136,7 +133,6 @@ class SSAUpdater {
private:
Value *GetValueAtEndOfBlockInternal(BasicBlock *BB);
- void UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue);
void UpdateDebugValue(Instruction *I, DbgVariableRecord *DbgValue);
};
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index d9d41f1d72e35..dc8184394f74d 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -445,7 +445,6 @@ class CodeGenPrepare {
bool optimizeSwitchInst(SwitchInst *SI);
bool optimizeExtractElementInst(Instruction *Inst);
bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);
- bool fixupDbgValue(Instruction *I);
bool fixupDbgVariableRecord(DbgVariableRecord &I);
bool fixupDbgVariableRecordsOnInst(Instruction &I);
bool placeDbgValues(Function &F);
@@ -2762,9 +2761,6 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
case Intrinsic::fshl:
case Intrinsic::fshr:
return optimizeFunnelShift(II);
- case Intrinsic::dbg_assign:
- case Intrinsic::dbg_value:
- return fixupDbgValue(II);
case Intrinsic::masked_gather:
return optimizeGatherScatterInst(II, II->getArgOperand(0));
case Intrinsic::masked_scatter:
@@ -3554,8 +3550,6 @@ class TypePromotionTransaction {
/// Keep track of the original uses (pair Instruction, Index).
SmallVector<InstructionAndIdx, 4> OriginalUses;
/// Keep track of the debug users.
- SmallVector<DbgValueInst *, 1> DbgValues;
- /// And non-instruction debug-users too.
SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
/// Keep track of the new value so that we can undo it by replacing
@@ -3577,7 +3571,9 @@ class TypePromotionTransaction {
}
// Record the debug uses separately. They are not in the instruction's
// use list, but they are replaced by RAUW.
+ SmallVector<DbgValueInst *> DbgValues;
findDbgValues(DbgValues, Inst, &DbgVariableRecords);
+ assert(DbgValues.empty());
// Now, we can replace the uses.
Inst->replaceAllUsesWith(New);
@@ -3591,11 +3587,7 @@ class TypePromotionTransaction {
// RAUW has replaced all original uses with references to the new value,
// including the debug uses. Since we are undoing the replacements,
// the original debug uses must also be reinstated to maintain the
- // correctness and utility of debug value instructions.
- for (auto *DVI : DbgValues)
- DVI->replaceVariableLocationOp(New, Inst);
- // Similar story with DbgVariableRecords, the non-instruction
- // representation of dbg.values.
+ // correctness and utility of debug value records.
for (DbgVariableRecord *DVR : DbgVariableRecords)
DVR->replaceVariableLocationOp(New, Inst);
}
@@ -8933,32 +8925,6 @@ bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {
return MadeChange;
}
-// Some CGP optimizations may move or alter what's computed in a block. Check
-// whether a dbg.value intrinsic could be pointed at a more appropriate operand.
-bool CodeGenPrepare::fixupDbgValue(Instruction *I) {
- assert(isa<DbgValueInst>(I));
- DbgValueInst &DVI = *cast<DbgValueInst>(I);
-
- // Does this dbg.value refer to a sunk address calculation?
- bool AnyChange = false;
- SmallDenseSet<Value *> LocationOps(DVI.location_ops().begin(),
- DVI.location_ops().end());
- for (Value *Location : LocationOps) {
- WeakTrackingVH SunkAddrVH = SunkAddrs[Location];
- Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;
- if (SunkAddr) {
- // Point dbg.value at locally computed address, which should give the best
- // opportunity to be accurately lowered. This update may change the type
- // of pointer being referred to; however this makes no difference to
- // debugging information, and we can't generate bitcasts that may affect
- // codegen.
- DVI.replaceVariableLocationOp(Location, SunkAddr);
- AnyChange = true;
- }
- }
- return AnyChange;
-}
-
bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &I) {
bool AnyChange = false;
for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
@@ -8993,14 +8959,6 @@ bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {
return AnyChange;
}
-static void DbgInserterHelper(DbgValueInst *DVI, BasicBlock::iterator VI) {
- DVI->removeFromParent();
- if (isa<PHINode>(VI))
- DVI->insertBefore(VI->getParent()->getFirstInsertionPt());
- else
- DVI->insertAfter(VI);
-}
-
static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI) {
DVR->removeFromParent();
BasicBlock *VIBB = VI->getParent();
@@ -9065,15 +9023,8 @@ bool CodeGenPrepare::placeDbgValues(Function &F) {
for (BasicBlock &BB : F) {
for (Instruction &Insn : llvm::make_early_inc_range(BB)) {
- // Process dbg.value intrinsics.
- DbgValueInst *DVI = dyn_cast<DbgValueInst>(&Insn);
- if (DVI) {
- DbgProcessor(DVI, DVI);
- continue;
- }
-
- // If this isn't a dbg.value, process any attached DbgVariableRecord
- // records attached to this instruction.
+ // Process any DbgVariableRecord records attached to this
+ // instruction.
for (DbgVariableRecord &DVR : llvm::make_early_inc_range(
filterDbgVars(Insn.getDbgRecordRange()))) {
if (DVR.Type != DbgVariableRecord::LocationType::Value)
diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 9daacfd399787..e7fa0824fd98a 100644
--- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -202,8 +202,8 @@ MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
MachineBlockFrequencyInfo &&) = default;
MachineBlockFrequencyInfo::MachineBlockFrequencyInfo(
- MachineFunction &F, MachineBranchProbabilityInfo &MBPI,
- MachineLoopInfo &MLI) {
+ const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI,
+ const MachineLoopInfo &MLI) {
calculate(F, MBPI, MLI);
}
diff --git a/llvm/lib/CodeGen/MachineDebugify.cpp b/llvm/lib/CodeGen/MachineDebugify.cpp
index 9b9cebc74054d..1a20fe586e951 100644
--- a/llvm/lib/CodeGen/MachineDebugify.cpp
+++ b/llvm/lib/CodeGen/MachineDebugify.cpp
@@ -63,24 +63,9 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
// which cover a wide range of lines can help stress the debug info passes:
// if we can't do that, fall back to using the local variable which precedes
// all the others.
- Function *DbgValF = M.getFunction("llvm.dbg.value");
- DbgValueInst *EarliestDVI = nullptr;
DbgVariableRecord *EarliestDVR = nullptr;
DenseMap<unsigned, DILocalVariable *> Line2Var;
DIExpression *Expr = nullptr;
- if (DbgValF) {
- for (const Use &U : DbgValF->uses()) {
- auto *DVI = dyn_cast<DbgValueInst>(U.getUser());
- if (!DVI || DVI->getFunction() != &F)
- continue;
- unsigned Line = DVI->getDebugLoc().getLine();
- assert(Line != 0 && "debugify should not insert line 0 locations");
- Line2Var[Line] = DVI->getVariable();
- if (!EarliestDVI || Line < EarliestDVI->getDebugLoc().getLine())
- EarliestDVI = DVI;
- Expr = DVI->getExpression();
- }
- }
for (BasicBlock &BB : F) {
for (Instruction &I : BB) {
for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
@@ -125,8 +110,7 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
unsigned Line = MI.getDebugLoc().getLine();
auto It = Line2Var.find(Line);
if (It == Line2Var.end()) {
- Line = EarliestDVI ? EarliestDVI->getDebugLoc().getLine()
- : EarliestDVR->getDebugLoc().getLine();
+ Line = EarliestDVR->getDebugLoc().getLine();
It = Line2Var.find(Line);
assert(It != Line2Var.end());
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 40464e91f9efc..b7e41427f2c27 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7641,7 +7641,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
if (SDValue(GN0, 0).hasOneUse() &&
isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
- TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+ TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
@@ -15745,7 +15745,7 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
// fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
- TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+ TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
@@ -18727,6 +18727,12 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
return FTrunc;
+ // fold (sint_to_fp (trunc nsw x)) -> (sint_to_fp x)
+ if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoSignedWrap() &&
+ TLI.isTypeDesirableForOp(ISD::SINT_TO_FP,
+ N0.getOperand(0).getValueType()))
+ return DAG.getNode(ISD::SINT_TO_FP, DL, VT, N0.getOperand(0));
+
return SDValue();
}
@@ -18764,6 +18770,12 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
return FTrunc;
+ // fold (uint_to_fp (trunc nuw x)) -> (uint_to_fp x)
+ if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoUnsignedWrap() &&
+ TLI.isTypeDesirableForOp(ISD::UINT_TO_FP,
+ N0.getOperand(0).getValueType()))
+ return DAG.getNode(ISD::UINT_TO_FP, DL, VT, N0.getOperand(0));
+
return SDValue();
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 56c8bb441ddf8..5453828177c72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6751,7 +6751,9 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
return SDValue();
int64_t Offset = C2->getSExtValue();
switch (Opcode) {
- case ISD::ADD: break;
+ case ISD::ADD:
+ case ISD::PTRADD:
+ break;
case ISD::SUB: Offset = -uint64_t(Offset); break;
default: return SDValue();
}
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 84a56058de834..8fb33c30e5cac 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -2288,39 +2288,36 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
// Collect a map of {backing storage : dbg.declares} (currently "backing
// storage" is limited to Allocas). We'll use this to find dbg.declares to
// delete after running `trackAssignments`.
- DenseMap<const AllocaInst *, SmallPtrSet<DbgDeclareInst *, 2>> DbgDeclares;
DenseMap<const AllocaInst *, SmallPtrSet<DbgVariableRecord *, 2>> DVRDeclares;
// Create another similar map of {storage : variables} that we'll pass to
// trackAssignments.
StorageToVarsMap Vars;
- auto ProcessDeclare = [&](auto *Declare, auto &DeclareList) {
+ auto ProcessDeclare = [&](DbgVariableRecord &Declare) {
// FIXME: trackAssignments doesn't let you specify any modifiers to the
// variable (e.g. fragment) or location (e.g. offset), so we have to
// leave dbg.declares with non-empty expressions in place.
- if (Declare->getExpression()->getNumElements() != 0)
+ if (Declare.getExpression()->getNumElements() != 0)
return;
- if (!Declare->getAddress())
+ if (!Declare.getAddress())
return;
if (AllocaInst *Alloca =
- dyn_cast<AllocaInst>(Declare->getAddress()->stripPointerCasts())) {
+ dyn_cast<AllocaInst>(Declare.getAddress()->stripPointerCasts())) {
// FIXME: Skip VLAs for now (let these variables use dbg.declares).
if (!Alloca->isStaticAlloca())
return;
// Similarly, skip scalable vectors (use dbg.declares instead).
if (auto Sz = Alloca->getAllocationSize(*DL); Sz && Sz->isScalable())
return;
- DeclareList[Alloca].insert(Declare);
- Vars[Alloca].insert(VarRecord(Declare));
+ DVRDeclares[Alloca].insert(&Declare);
+ Vars[Alloca].insert(VarRecord(&Declare));
}
};
for (auto &BB : F) {
for (auto &I : BB) {
for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
if (DVR.isDbgDeclare())
- ProcessDeclare(&DVR, DVRDeclares);
+ ProcessDeclare(DVR);
}
- if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(&I))
- ProcessDeclare(DDI, DbgDeclares);
}
}
@@ -2336,8 +2333,8 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
trackAssignments(F.begin(), F.end(), Vars, *DL);
// Delete dbg.declares for variables now tracked with assignment tracking.
- auto DeleteSubsumedDeclare = [&](const auto &Markers, auto &Declares) {
- (void)Markers;
+ for (auto &[Insts, Declares] : DVRDeclares) {
+ auto Markers = at::getDVRAssignmentMarkers(Insts);
for (auto *Declare : Declares) {
// Assert that the alloca that Declare uses is now linked to a dbg.assign
// describing the same variable (i.e. check that this dbg.declare has
@@ -2356,10 +2353,6 @@ bool AssignmentTrackingPass::runOnFunction(Function &F) {
Changed = true;
}
};
- for (auto &P : DbgDeclares)
- DeleteSubsumedDeclare(at::getAssignmentMarkers(P.first), P.second);
- for (auto &P : DVRDeclares)
- DeleteSubsumedDeclare(at::getDVRAssignmentMarkers(P.first), P.second);
return Changed;
}
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 2270923bd3719..f16963dce56e1 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -49,20 +49,11 @@ uint32_t DIType::getAlignInBits() const {
const DIExpression::FragmentInfo DebugVariable::DefaultFragment = {
std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::min()};
-DebugVariable::DebugVariable(const DbgVariableIntrinsic *DII)
- : Variable(DII->getVariable()),
- Fragment(DII->getExpression()->getFragmentInfo()),
- InlinedAt(DII->getDebugLoc().getInlinedAt()) {}
-
DebugVariable::DebugVariable(const DbgVariableRecord *DVR)
: Variable(DVR->getVariable()),
Fragment(DVR->getExpression()->getFragmentInfo()),
InlinedAt(DVR->getDebugLoc().getInlinedAt()) {}
-DebugVariableAggregate::DebugVariableAggregate(const DbgVariableIntrinsic *DVI)
- : DebugVariable(DVI->getVariable(), std::nullopt,
- DVI->getDebugLoc()->getInlinedAt()) {}
-
DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
ArrayRef<Metadata *> MDs, bool ImplicitCode)
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 79eeb08cddeef..2579fa37935f0 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -54,11 +54,6 @@ static const char *PreservedSymbols[] = {
"__stack_chk_guard",
};
-static bool isPreservedGlobalVarName(StringRef Name) {
- return StringRef(PreservedSymbols[0]) == Name ||
- StringRef(PreservedSymbols[1]) == Name;
-}
-
namespace {
const char *getExpectedProducerName() {
@@ -86,16 +81,12 @@ struct Builder {
// The StringTableBuilder does not create a copy of any strings added to it,
// so this provides somewhere to store any strings that we create.
Builder(SmallVector<char, 0> &Symtab, StringTableBuilder &StrtabBuilder,
- BumpPtrAllocator &Alloc, const Triple &TT)
- : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc), TT(TT),
- Libcalls(TT) {}
+ BumpPtrAllocator &Alloc)
+ : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc) {}
DenseMap<const Comdat *, int> ComdatMap;
Mangler Mang;
- const Triple &TT;
-
- // FIXME: This shouldn't be here.
- RTLIB::RuntimeLibcallsInfo Libcalls;
+ Triple TT;
std::vector<storage::Comdat> Comdats;
std::vector<storage::Module> Mods;
@@ -107,10 +98,6 @@ struct Builder {
std::vector<storage::Str> DependentLibraries;
- bool isPreservedLibFuncName(StringRef Name) {
- return Libcalls.getSupportedLibcallImpl(Name) != RTLIB::Unsupported;
- }
-
void setStr(storage::Str &S, StringRef Value) {
S.Offset = StrtabBuilder.add(Value);
S.Size = Value.size();
@@ -226,6 +213,18 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
return P.first->second;
}
+static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
+ DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
+ std::end(PreservedSymbols));
+ // FIXME: Do we need to pass in ABI fields from TargetOptions?
+ RTLIB::RuntimeLibcallsInfo Libcalls(TT);
+ for (RTLIB::LibcallImpl Impl : Libcalls.getLibcallImpls()) {
+ if (Impl != RTLIB::Unsupported)
+ PreservedSymbolSet.insert(Libcalls.getLibcallImplName(Impl));
+ }
+ return PreservedSymbolSet;
+}
+
Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
const SmallPtrSet<GlobalValue *, 4> &Used,
ModuleSymbolTable::Symbol Msym) {
@@ -279,11 +278,13 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
return Error::success();
}
- StringRef GVName = GV->getName();
- setStr(Sym.IRName, GVName);
+ setStr(Sym.IRName, GV->getName());
+
+ static const DenseSet<StringRef> PreservedSymbolsSet =
+ buildPreservedSymbolsSet(GV->getParent()->getTargetTriple());
+ bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
- if (Used.count(GV) || isPreservedLibFuncName(GVName) ||
- isPreservedGlobalVarName(GVName))
+ if (Used.count(GV) || IsPreservedSymbol)
Sym.Flags |= 1 << storage::Symbol::FB_used;
if (GV->isThreadLocal())
Sym.Flags |= 1 << storage::Symbol::FB_tls;
@@ -350,6 +351,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
setStr(Hdr.Producer, kExpectedProducerName);
setStr(Hdr.TargetTriple, IRMods[0]->getTargetTriple().str());
setStr(Hdr.SourceFileName, IRMods[0]->getSourceFileName());
+ TT = IRMods[0]->getTargetTriple();
for (auto *M : IRMods)
if (Error Err = addModule(M))
@@ -375,8 +377,7 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
Error irsymtab::build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
StringTableBuilder &StrtabBuilder,
BumpPtrAllocator &Alloc) {
- const Triple &TT = Mods[0]->getTargetTriple();
- return Builder(Symtab, StrtabBuilder, Alloc, TT).build(Mods);
+ return Builder(Symtab, StrtabBuilder, Alloc).build(Mods);
}
// Upgrade a vector of bitcode modules created by an old version of LLVM by
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d04e6c45e2103..f026726c3f484 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6439,7 +6439,9 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
}
}
- return true;
+ EVT PreExtScalarVT = ExtVal->getOperand(0).getValueType().getScalarType();
+ return PreExtScalarVT == MVT::i8 || PreExtScalarVT == MVT::i16 ||
+ PreExtScalarVT == MVT::i32 || PreExtScalarVT == MVT::i64;
}
unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 23f106a9c1d4d..007b481f84960 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -153,6 +153,9 @@ struct AMDGPULowerBufferFatPointersPass
const TargetMachine &TM;
};
+void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &);
+extern char &AMDGPUPrepareAGPRAllocLegacyID;
+
void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &);
extern char &AMDGPUReserveWWMRegsLegacyID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 250547acb1ee7..b6c6d927d0e89 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -114,6 +114,7 @@ MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUse
MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
MACHINE_FUNCTION_PASS("amdgpu-preload-kern-arg-prolog", AMDGPUPreloadKernArgPrologPass())
+MACHINE_FUNCTION_PASS("amdgpu-prepare-agpr-alloc", AMDGPUPrepareAGPRAllocPass())
MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass())
MACHINE_FUNCTION_PASS("amdgpu-wait-sgpr-hazards", AMDGPUWaitSGPRHazardsPass())
MACHINE_FUNCTION_PASS("gcn-create-vopd", GCNCreateVOPDPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
new file mode 100644
index 0000000000000..3b06e9b00ac69
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
@@ -0,0 +1,108 @@
+//===-- AMDGPUPrepareAGPRAlloc.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Make simple transformations to relax register constraints for cases which can
+// allocate to AGPRs or VGPRs. Replace materialize of inline immediates into
+// AGPR or VGPR with a pseudo with an AV_* class register constraint. This
+// allows later passes to inflate the register class if necessary. The register
+// allocator does not know to replace instructions to relax constraints.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUPrepareAGPRAlloc.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-prepare-agpr-alloc"
+
+namespace {
+
+class AMDGPUPrepareAGPRAllocImpl {
+private:
+ const SIInstrInfo &TII;
+ MachineRegisterInfo &MRI;
+
+public:
+ AMDGPUPrepareAGPRAllocImpl(const GCNSubtarget &ST, MachineRegisterInfo &MRI)
+ : TII(*ST.getInstrInfo()), MRI(MRI) {}
+ bool run(MachineFunction &MF);
+};
+
+class AMDGPUPrepareAGPRAllocLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUPrepareAGPRAllocLegacy() : MachineFunctionPass(ID) {
+ initializeAMDGPUPrepareAGPRAllocLegacyPass(
+ *PassRegistry::getPassRegistry());
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ StringRef getPassName() const override { return "AMDGPU Prepare AGPR Alloc"; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+ "AMDGPU Prepare AGPR Alloc", false, false)
+INITIALIZE_PASS_END(AMDGPUPrepareAGPRAllocLegacy, DEBUG_TYPE,
+ "AMDGPU Prepare AGPR Alloc", false, false)
+
+char AMDGPUPrepareAGPRAllocLegacy::ID = 0;
+
+char &llvm::AMDGPUPrepareAGPRAllocLegacyID = AMDGPUPrepareAGPRAllocLegacy::ID;
+
+bool AMDGPUPrepareAGPRAllocLegacy::runOnMachineFunction(MachineFunction &MF) {
+ if (skipFunction(MF.getFunction()))
+ return false;
+
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ return AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+}
+
+PreservedAnalyses
+AMDGPUPrepareAGPRAllocPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ AMDGPUPrepareAGPRAllocImpl(ST, MF.getRegInfo()).run(MF);
+ return PreservedAnalyses::all();
+}
+
+bool AMDGPUPrepareAGPRAllocImpl::run(MachineFunction &MF) {
+ if (MRI.isReserved(AMDGPU::AGPR0))
+ return false;
+
+ const MCInstrDesc &AVImmPseudo = TII.get(AMDGPU::AV_MOV_B32_IMM_PSEUDO);
+
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ if ((MI.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
+ TII.isInlineConstant(MI, 1)) ||
+ (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
+ MI.getOperand(1).isImm())) {
+ MI.setDesc(AVImmPseudo);
+ Changed = true;
+ }
+ }
+ }
+
+ return Changed;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
new file mode 100644
index 0000000000000..dc598c98f241b
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.h
@@ -0,0 +1,23 @@
+//===- AMDGPUPrepareAGPRAlloc.h ---------------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class AMDGPUPrepareAGPRAllocPass
+ : public PassInfoMixin<AMDGPUPrepareAGPRAllocPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUPREPAREAGPRALLOC_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
index 46027b8890234..8101c68986241 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
@@ -167,77 +167,39 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
Info.UsesVCC =
MRI.isPhysRegUsed(AMDGPU::VCC_LO) || MRI.isPhysRegUsed(AMDGPU::VCC_HI);
+ Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass,
+ /*IncludeCalls=*/false);
+ if (ST.hasMAIInsts())
+ Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass,
+ /*IncludeCalls=*/false);
// If there are no calls, MachineRegisterInfo can tell us the used register
// count easily.
// A tail call isn't considered a call for MachineFrameInfo's purposes.
if (!FrameInfo.hasCalls() && !FrameInfo.hasTailCall()) {
- Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass);
- Info.NumExplicitSGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::SGPR_32RegClass);
- if (ST.hasMAIInsts())
- Info.NumAGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::AGPR_32RegClass);
+ Info.NumVGPR = TRI.getNumUsedPhysRegs(MRI, AMDGPU::VGPR_32RegClass,
+ /*IncludeCalls=*/false);
return Info;
}
int32_t MaxVGPR = -1;
- int32_t MaxAGPR = -1;
- int32_t MaxSGPR = -1;
Info.CalleeSegmentSize = 0;
for (const MachineBasicBlock &MBB : MF) {
for (const MachineInstr &MI : MBB) {
- // TODO: Check regmasks? Do they occur anywhere except calls?
- for (const MachineOperand &MO : MI.operands()) {
- unsigned Width = 0;
- bool IsSGPR = false;
- bool IsAGPR = false;
+ for (unsigned I = 0; I < MI.getNumOperands(); ++I) {
+ const MachineOperand &MO = MI.getOperand(I);
if (!MO.isReg())
continue;
Register Reg = MO.getReg();
switch (Reg) {
- case AMDGPU::EXEC:
- case AMDGPU::EXEC_LO:
- case AMDGPU::EXEC_HI:
- case AMDGPU::SCC:
- case AMDGPU::M0:
- case AMDGPU::M0_LO16:
- case AMDGPU::M0_HI16:
- case AMDGPU::SRC_SHARED_BASE_LO:
- case AMDGPU::SRC_SHARED_BASE:
- case AMDGPU::SRC_SHARED_LIMIT_LO:
- case AMDGPU::SRC_SHARED_LIMIT:
- case AMDGPU::SRC_PRIVATE_BASE_LO:
- case AMDGPU::SRC_PRIVATE_BASE:
- case AMDGPU::SRC_PRIVATE_LIMIT_LO:
- case AMDGPU::SRC_PRIVATE_LIMIT:
- case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
- case AMDGPU::SGPR_NULL:
- case AMDGPU::SGPR_NULL64:
- case AMDGPU::MODE:
- continue;
-
case AMDGPU::NoRegister:
assert(MI.isDebugInstr() &&
"Instruction uses invalid noreg register");
continue;
- case AMDGPU::VCC:
- case AMDGPU::VCC_LO:
- case AMDGPU::VCC_HI:
- case AMDGPU::VCC_LO_LO16:
- case AMDGPU::VCC_LO_HI16:
- case AMDGPU::VCC_HI_LO16:
- case AMDGPU::VCC_HI_HI16:
- Info.UsesVCC = true;
- continue;
-
- case AMDGPU::FLAT_SCR:
- case AMDGPU::FLAT_SCR_LO:
- case AMDGPU::FLAT_SCR_HI:
- continue;
-
case AMDGPU::XNACK_MASK:
case AMDGPU::XNACK_MASK_LO:
case AMDGPU::XNACK_MASK_HI:
@@ -267,170 +229,22 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
break;
}
- if (AMDGPU::SGPR_32RegClass.contains(Reg) ||
- AMDGPU::SGPR_LO16RegClass.contains(Reg) ||
- AMDGPU::SGPR_HI16RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 1;
- } else if (AMDGPU::VGPR_32RegClass.contains(Reg) ||
- AMDGPU::VGPR_16RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 1;
- } else if (AMDGPU::AGPR_32RegClass.contains(Reg) ||
- AMDGPU::AGPR_LO16RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 1;
- } else if (AMDGPU::SGPR_64RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 2;
- } else if (AMDGPU::VReg_64RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 2;
- } else if (AMDGPU::AReg_64RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 2;
- } else if (AMDGPU::VReg_96RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 3;
- } else if (AMDGPU::SReg_96RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 3;
- } else if (AMDGPU::AReg_96RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 3;
- } else if (AMDGPU::SGPR_128RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 4;
- } else if (AMDGPU::VReg_128RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 4;
- } else if (AMDGPU::AReg_128RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 4;
- } else if (AMDGPU::VReg_160RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 5;
- } else if (AMDGPU::SReg_160RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 5;
- } else if (AMDGPU::AReg_160RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 5;
- } else if (AMDGPU::VReg_192RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 6;
- } else if (AMDGPU::SReg_192RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 6;
- } else if (AMDGPU::AReg_192RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 6;
- } else if (AMDGPU::VReg_224RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 7;
- } else if (AMDGPU::SReg_224RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 7;
- } else if (AMDGPU::AReg_224RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 7;
- } else if (AMDGPU::SReg_256RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 8;
- } else if (AMDGPU::VReg_256RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 8;
- } else if (AMDGPU::AReg_256RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 8;
- } else if (AMDGPU::VReg_288RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 9;
- } else if (AMDGPU::SReg_288RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 9;
- } else if (AMDGPU::AReg_288RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 9;
- } else if (AMDGPU::VReg_320RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 10;
- } else if (AMDGPU::SReg_320RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 10;
- } else if (AMDGPU::AReg_320RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 10;
- } else if (AMDGPU::VReg_352RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 11;
- } else if (AMDGPU::SReg_352RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 11;
- } else if (AMDGPU::AReg_352RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 11;
- } else if (AMDGPU::VReg_384RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 12;
- } else if (AMDGPU::SReg_384RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 12;
- } else if (AMDGPU::AReg_384RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 12;
- } else if (AMDGPU::SReg_512RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 16;
- } else if (AMDGPU::VReg_512RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 16;
- } else if (AMDGPU::AReg_512RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 16;
- } else if (AMDGPU::SReg_1024RegClass.contains(Reg)) {
- IsSGPR = true;
- Width = 32;
- } else if (AMDGPU::VReg_1024RegClass.contains(Reg)) {
- IsSGPR = false;
- Width = 32;
- } else if (AMDGPU::AReg_1024RegClass.contains(Reg)) {
- IsSGPR = false;
- IsAGPR = true;
- Width = 32;
- } else {
- // We only expect TTMP registers or registers that do not belong to
- // any RC.
- assert((AMDGPU::TTMP_32RegClass.contains(Reg) ||
- AMDGPU::TTMP_64RegClass.contains(Reg) ||
- AMDGPU::TTMP_128RegClass.contains(Reg) ||
- AMDGPU::TTMP_256RegClass.contains(Reg) ||
- AMDGPU::TTMP_512RegClass.contains(Reg) ||
- !TRI.getPhysRegBaseClass(Reg)) &&
- "Unknown register class");
- }
+ const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg);
+ assert((!RC || TRI.isVGPRClass(RC) || TRI.isSGPRClass(RC) ||
+ TRI.isAGPRClass(RC) || AMDGPU::TTMP_32RegClass.contains(Reg) ||
+ AMDGPU::TTMP_64RegClass.contains(Reg) ||
+ AMDGPU::TTMP_128RegClass.contains(Reg) ||
+ AMDGPU::TTMP_256RegClass.contains(Reg) ||
+ AMDGPU::TTMP_512RegClass.contains(Reg)) &&
+ "Unknown register class");
+
+ if (!RC || !TRI.isVGPRClass(RC))
+ continue;
+
+ unsigned Width = divideCeil(TRI.getRegSizeInBits(*RC), 32);
unsigned HWReg = TRI.getHWRegIndex(Reg);
int MaxUsed = HWReg + Width - 1;
- if (IsSGPR) {
- MaxSGPR = MaxUsed > MaxSGPR ? MaxUsed : MaxSGPR;
- } else if (IsAGPR) {
- MaxAGPR = MaxUsed > MaxAGPR ? MaxUsed : MaxAGPR;
- } else {
- MaxVGPR = MaxUsed > MaxVGPR ? MaxUsed : MaxVGPR;
- }
+ MaxVGPR = std::max(MaxUsed, MaxVGPR);
}
if (MI.isCall()) {
@@ -492,9 +306,7 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
}
}
- Info.NumExplicitSGPR = MaxSGPR + 1;
Info.NumVGPR = MaxVGPR + 1;
- Info.NumAGPR = MaxAGPR + 1;
return Info;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 31a80e00edd3b..c865082a1dcea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -25,6 +25,7 @@
#include "AMDGPUMacroFusion.h"
#include "AMDGPUPerfHintAnalysis.h"
#include "AMDGPUPreloadKernArgProlog.h"
+#include "AMDGPUPrepareAGPRAlloc.h"
#include "AMDGPURemoveIncompatibleFunctions.h"
#include "AMDGPUReserveWWMRegs.h"
#include "AMDGPUResourceUsageAnalysis.h"
@@ -499,6 +500,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeGlobalISel(*PR);
initializeAMDGPUAsmPrinterPass(*PR);
initializeAMDGPUDAGToDAGISelLegacyPass(*PR);
+ initializeAMDGPUPrepareAGPRAllocLegacyPass(*PR);
initializeGCNDPPCombineLegacyPass(*PR);
initializeSILowerI1CopiesLegacyPass(*PR);
initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
@@ -1196,6 +1198,7 @@ class GCNPassConfig final : public AMDGPUPassConfig {
bool addRegBankSelect() override;
void addPreGlobalInstructionSelect() override;
bool addGlobalInstructionSelect() override;
+ void addPreRegAlloc() override;
void addFastRegAlloc() override;
void addOptimizedRegAlloc() override;
@@ -1539,6 +1542,11 @@ void GCNPassConfig::addFastRegAlloc() {
TargetPassConfig::addFastRegAlloc();
}
+void GCNPassConfig::addPreRegAlloc() {
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(&AMDGPUPrepareAGPRAllocLegacyID);
+}
+
void GCNPassConfig::addOptimizedRegAlloc() {
if (EnableDCEInRA)
insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID);
@@ -2235,6 +2243,11 @@ void AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(
Base::addOptimizedRegAlloc(addPass);
}
+void AMDGPUCodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const {
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(AMDGPUPrepareAGPRAllocPass());
+}
+
Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(
AddMachinePass &addPass) const {
// TODO: Check --regalloc-npm option
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 3b2f39c14a9bc..e0f1296ddded8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -181,7 +181,9 @@ class AMDGPUCodeGenPassBuilder
void addMachineSSAOptimization(AddMachinePass &) const;
void addPostRegAlloc(AddMachinePass &) const;
void addPreEmitPass(AddMachinePass &) const;
+ void addPreEmitRegAlloc(AddMachinePass &) const;
Error addRegAssignmentOptimized(AddMachinePass &) const;
+ void addPreRegAlloc(AddMachinePass &) const;
void addOptimizedRegAlloc(AddMachinePass &) const;
void addPreSched2(AddMachinePass &) const;
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index e3519f192137c..42edec0d01493 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -74,6 +74,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPULowerKernelArguments.cpp
AMDGPULowerKernelAttributes.cpp
AMDGPULowerModuleLDSPass.cpp
+ AMDGPUPrepareAGPRAlloc.cpp
AMDGPUSwLowerLDS.cpp
AMDGPUMachineFunction.cpp
AMDGPUMachineModuleInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 400795c29b0e4..0c76ff2ec5ea7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -33,6 +33,7 @@
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
@@ -46,6 +47,7 @@
#include <optional>
using namespace llvm;
+using namespace llvm::SDPatternMatch;
#define DEBUG_TYPE "si-lower"
@@ -14561,7 +14563,7 @@ static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
// instead of a tree.
SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
DAGCombinerInfo &DCI) const {
- assert(N->getOpcode() == ISD::ADD);
+ assert(N->isAnyAdd());
SelectionDAG &DAG = DCI.DAG;
EVT VT = N->getValueType(0);
@@ -14594,7 +14596,7 @@ SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
for (SDNode *User : LHS->users()) {
// There is a use that does not feed into addition, so the multiply can't
// be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
- if (User->getOpcode() != ISD::ADD)
+ if (!User->isAnyAdd())
return SDValue();
// We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
@@ -14706,8 +14708,11 @@ SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
SDValue Hi = getHiHalf64(LHS, DAG);
SDValue ConstHi32 = DAG.getConstant(Hi_32(Val), SL, MVT::i32);
+ unsigned Opcode = N->getOpcode();
+ if (Opcode == ISD::PTRADD)
+ Opcode = ISD::ADD;
SDValue AddHi =
- DAG.getNode(N->getOpcode(), SL, MVT::i32, Hi, ConstHi32, N->getFlags());
+ DAG.getNode(Opcode, SL, MVT::i32, Hi, ConstHi32, N->getFlags());
SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Lo, AddHi);
@@ -15181,42 +15186,123 @@ SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);
+ EVT VT = N->getValueType(0);
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
- if (N1.getOpcode() == ISD::ADD) {
- // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
- // y is not, and (add y, z) is used only once.
- // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
- // z is not, and (add y, z) is used only once.
- // The goal is to move constant offsets to the outermost ptradd, to create
- // more opportunities to fold offsets into memory instructions.
- // Together with the generic combines in DAGCombiner.cpp, this also
- // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
- //
- // This transform is here instead of in the general DAGCombiner as it can
- // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
- // AArch64's CPA.
- SDValue X = N0;
- SDValue Y = N1.getOperand(0);
- SDValue Z = N1.getOperand(1);
- if (N1.hasOneUse()) {
- bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
- bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
- if (ZIsConstant != YIsConstant) {
- // If both additions in the original were NUW, the new ones are as well.
- SDNodeFlags Flags =
- (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
- if (YIsConstant)
- std::swap(Y, Z);
+ // The following folds transform PTRADDs into regular arithmetic in cases
+ // where the PTRADD wouldn't be folded as an immediate offset into memory
+ // instructions anyway. They are target-specific in that other targets might
+ // prefer to not lose information about the pointer arithmetic.
+
+ // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
+ // Adapted from DAGCombiner::visitADDLikeCommutative.
+ SDValue V, K;
+ if (sd_match(N1, m_Shl(m_Neg(m_Value(V)), m_Value(K)))) {
+ SDNodeFlags ShlFlags = N1->getFlags();
+ // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
+ // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
+ // preserved.
+ SDNodeFlags NewShlFlags =
+ ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
+ ? SDNodeFlags::NoSignedWrap
+ : SDNodeFlags();
+ SDValue Inner = DAG.getNode(ISD::SHL, DL, VT, V, K, NewShlFlags);
+ DCI.AddToWorklist(Inner.getNode());
+ return DAG.getNode(ISD::SUB, DL, VT, N0, Inner);
+ }
+
+ // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
+ // performAddCombine.
+ if (N1.getOpcode() == ISD::MUL) {
+ if (Subtarget->hasMad64_32()) {
+ if (SDValue Folded = tryFoldToMad64_32(N, DCI))
+ return Folded;
+ }
+ }
- SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, Flags);
+ // If the 32 low bits of the constant are all zero, there is nothing to fold
+ // into an immediate offset, so it's better to eliminate the unnecessary
+ // addition for the lower 32 bits than to preserve the PTRADD.
+ // Analogous to a fold in performAddCombine.
+ if (VT == MVT::i64) {
+ if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
+ return Folded;
+ }
+
+ if (N0.getOpcode() == ISD::PTRADD && N1.getOpcode() == ISD::Constant) {
+ // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
+ // global address GA and constant c, such that c can be folded into GA.
+ SDValue GAValue = N0.getOperand(0);
+ if (const GlobalAddressSDNode *GA =
+ dyn_cast<GlobalAddressSDNode>(GAValue)) {
+ if (DCI.isBeforeLegalizeOps() && isOffsetFoldingLegal(GA)) {
+ // If both additions in the original were NUW, reassociation preserves
+ // that.
+ SDNodeFlags Flags =
+ (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+ SDValue Inner = DAG.getMemBasePlusOffset(GAValue, N1, DL, Flags);
DCI.AddToWorklist(Inner.getNode());
- return DAG.getMemBasePlusOffset(Inner, Z, DL, Flags);
+ return DAG.getMemBasePlusOffset(Inner, N0.getOperand(1), DL, Flags);
}
}
}
+ if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
+ return SDValue();
+
+ // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
+ // y is not, and (add y, z) is used only once.
+ // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
+ // z is not, and (add y, z) is used only once.
+ // The goal is to move constant offsets to the outermost ptradd, to create
+ // more opportunities to fold offsets into memory instructions.
+ // Together with the generic combines in DAGCombiner.cpp, this also
+ // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
+ //
+ // This transform is here instead of in the general DAGCombiner as it can
+ // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
+ // AArch64's CPA.
+ SDValue X = N0;
+ SDValue Y = N1.getOperand(0);
+ SDValue Z = N1.getOperand(1);
+ bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
+ bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
+
+ // If both additions in the original were NUW, reassociation preserves that.
+ SDNodeFlags ReassocFlags =
+ (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+
+ if (ZIsConstant != YIsConstant) {
+ if (YIsConstant)
+ std::swap(Y, Z);
+ SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+ DCI.AddToWorklist(Inner.getNode());
+ return DAG.getMemBasePlusOffset(Inner, Z, DL, ReassocFlags);
+ }
+
+ // If one of Y and Z is constant, they have been handled above. If both were
+ // constant, the addition would have been folded in SelectionDAG::getNode
+ // already. This ensures that the generic DAG combines won't undo the
+ // following reassociation.
+ assert(!YIsConstant && !ZIsConstant);
+
+ if (!X->isDivergent() && Y->isDivergent() != Z->isDivergent()) {
+ // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
+ // y are uniform and z isn't.
+ // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
+ // z are uniform and y isn't.
+ // The goal is to push uniform operands up in the computation, so that they
+ // can be handled with scalar operations. We can't use reassociateScalarOps
+ // for this since it requires two identical commutative operations to
+ // reassociate.
+ if (Y->isDivergent())
+ std::swap(Y, Z);
+ SDValue UniformInner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
+ DCI.AddToWorklist(UniformInner.getNode());
+ return DAG.getMemBasePlusOffset(UniformInner, Z, DL, ReassocFlags);
+ }
+
return SDValue();
}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2764ed3d3f0b1..5e92921f3ea21 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1113,7 +1113,6 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
// that will not require an additional 4-bytes; this function assumes that it
// will.
bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
- assert(!MO.isReg() && "isInlineConstant called on register operand!");
if (!MO.isImm())
return false;
return isInlineConstant(MO.getImm(), OperandType);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 9173041a7bccd..fa2b8db6ba55a 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -4052,11 +4052,11 @@ SIRegisterInfo::getSubRegAlignmentNumBits(const TargetRegisterClass *RC,
return 0;
}
-unsigned
-SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
- const TargetRegisterClass &RC) const {
+unsigned SIRegisterInfo::getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
+ const TargetRegisterClass &RC,
+ bool IncludeCalls) const {
for (MCPhysReg Reg : reverse(RC.getRegisters()))
- if (MRI.isPhysRegUsed(Reg))
+ if (MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/!IncludeCalls))
return getHWRegIndex(Reg) + 1;
return 0;
}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 06a7a17b0246b..0008e5f8cf3b4 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -486,9 +486,11 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
unsigned SubReg) const;
// \returns a number of registers of a given \p RC used in a function.
- // Does not go inside function calls.
+ // Does not go inside function calls. If \p IncludeCalls is true, it will
+ // include registers that may be clobbered by calls.
unsigned getNumUsedPhysRegs(const MachineRegisterInfo &MRI,
- const TargetRegisterClass &RC) const;
+ const TargetRegisterClass &RC,
+ bool IncludeCalls = true) const;
std::optional<uint8_t> getVRegFlagValue(StringRef Name) const override {
return Name == "WWM_REG" ? AMDGPU::VirtRegFlag::WWM_REG
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 1bbbb610305e9..3ee90857b34b8 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -1148,6 +1148,7 @@ defm V_MOV_B64 : VOP1_Real_FULL <GFX1250Gen, 0x1d>;
defm V_TANH_F32 : VOP1_Real_FULL<GFX1250Gen, 0x01e>;
defm V_TANH_F16 : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x01f>;
defm V_TANH_BF16 : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x04a>;
+defm V_PRNG_B32 : VOP1_Real_FULL<GFX1250Gen, 0x04b>;
defm V_CVT_F32_BF16 : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x072, "v_cvt_f32_bf16", "V_CVT_F32_BF16_gfx1250">;
defm V_CVT_PK_F16_FP8 : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x075>;
defm V_CVT_PK_F16_BF8 : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x076>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
index ac5e7f3891c72..1493bf4cba695 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchFrameLowering.cpp
@@ -158,7 +158,12 @@ void LoongArchFrameLowering::processFunctionBeforeFrameFinalized(
// estimateStackSize has been observed to under-estimate the final stack
// size, so give ourselves wiggle-room by checking for stack size
// representable an 11-bit signed field rather than 12-bits.
- if (!isInt<11>(MFI.estimateStackSize(MF)))
+ // For [x]vstelm.{b/h/w/d} memory instructions with 8 imm offset, 7-bit
+ // signed field is fine.
+ unsigned EstimateStackSize = MFI.estimateStackSize(MF);
+ if (!isInt<11>(EstimateStackSize) ||
+ (MF.getSubtarget<LoongArchSubtarget>().hasExtLSX() &&
+ !isInt<7>(EstimateStackSize)))
ScavSlotsNum = std::max(ScavSlotsNum, 1u);
// For CFR spill.
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 6c8e3da80b932..23b4554349003 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -95,6 +95,11 @@ static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
/* -21, -22, -23, -24 are reserved */
};
+/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
+static bool needsDwarfCFI(const MachineFunction &MF) {
+ return MF.needsFrameMoves();
+}
+
// For now we use x3, a.k.a gp, as pointer to shadow call stack.
// User should not use x3 in their asm.
static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -141,6 +146,9 @@ static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
.addImm(-SlotSize)
.setMIFlag(MachineInstr::FrameSetup);
+ if (!needsDwarfCFI(MF))
+ return;
+
// Emit a CFI instruction that causes SlotSize to be subtracted from the value
// of the shadow stack pointer when unwinding past this frame.
char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
@@ -199,8 +207,10 @@ static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
.addReg(SCSPReg)
.addImm(-SlotSize)
.setMIFlag(MachineInstr::FrameDestroy);
- // Restore the SCS pointer
- CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+ if (needsDwarfCFI(MF)) {
+ // Restore the SCS pointer
+ CFIInstBuilder(MBB, MI, MachineInstr::FrameDestroy).buildRestore(SCSPReg);
+ }
}
// Insert instruction to swap mscratchsw with sp
@@ -935,6 +945,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
getUnmanagedCSI(MF, CSI).size());
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
+ bool NeedsDwarfCFI = needsDwarfCFI(MF);
// If libcalls are used to spill and restore callee-saved registers, the frame
// has two sections; the opaque section managed by the libcalls, and the
@@ -962,10 +973,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
RVFI->setLibCallStackSize(LibCallFrameSize);
- CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
- for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
- CFIBuilder.buildOffset(CS.getReg(),
- MFI.getObjectOffset(CS.getFrameIdx()));
+ if (NeedsDwarfCFI) {
+ CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
+ for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+ CFIBuilder.buildOffset(CS.getReg(),
+ MFI.getObjectOffset(CS.getFrameIdx()));
+ }
}
// FIXME (note copied from Lanai): This appears to be overallocating. Needs
@@ -996,14 +1009,17 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
// could only be the next instruction.
++PossiblePush;
- // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
- // could be. The PUSH will also get its own CFI metadata for its own
- // modifications, which should come after the PUSH.
- CFIInstBuilder PushCFIBuilder(MBB, PossiblePush, MachineInstr::FrameSetup);
- PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
- for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
- PushCFIBuilder.buildOffset(CS.getReg(),
- MFI.getObjectOffset(CS.getFrameIdx()));
+ if (NeedsDwarfCFI) {
+ // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
+ // could be. The PUSH will also get its own CFI metadata for its own
+ // modifications, which should come after the PUSH.
+ CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
+ MachineInstr::FrameSetup);
+ PushCFIBuilder.buildDefCFAOffset(QCIInterruptPushAmount);
+ for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
+ PushCFIBuilder.buildOffset(CS.getReg(),
+ MFI.getObjectOffset(CS.getFrameIdx()));
+ }
}
if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
@@ -1017,10 +1033,12 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
PossiblePush->getOperand(1).setImm(StackAdj);
StackSize -= StackAdj;
- CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
- for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
- CFIBuilder.buildOffset(CS.getReg(),
- MFI.getObjectOffset(CS.getFrameIdx()));
+ if (NeedsDwarfCFI) {
+ CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
+ for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+ CFIBuilder.buildOffset(CS.getReg(),
+ MFI.getObjectOffset(CS.getFrameIdx()));
+ }
}
// Allocate space on the stack if necessary.
@@ -1031,7 +1049,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
bool DynAllocation =
MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
if (StackSize != 0)
- allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true,
+ allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, NeedsDwarfCFI,
NeedProbe, ProbeSize, DynAllocation,
MachineInstr::FrameSetup);
@@ -1049,8 +1067,10 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
// Iterate over list of callee-saved registers and emit .cfi_offset
// directives.
- for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
- CFIBuilder.buildOffset(CS.getReg(), MFI.getObjectOffset(CS.getFrameIdx()));
+ if (NeedsDwarfCFI)
+ for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+ CFIBuilder.buildOffset(CS.getReg(),
+ MFI.getObjectOffset(CS.getFrameIdx()));
// Generate new FP.
if (hasFP(MF)) {
@@ -1069,7 +1089,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
MachineInstr::FrameSetup, getStackAlign());
}
- CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
+ if (NeedsDwarfCFI)
+ CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
}
uint64_t SecondSPAdjustAmount = 0;
@@ -1080,15 +1101,16 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
"SecondSPAdjustAmount should be greater than zero");
allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
- getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe,
- ProbeSize, DynAllocation, MachineInstr::FrameSetup);
+ getStackSizeWithRVVPadding(MF), NeedsDwarfCFI && !hasFP(MF),
+ NeedProbe, ProbeSize, DynAllocation,
+ MachineInstr::FrameSetup);
}
if (RVVStackSize) {
if (NeedProbe) {
allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, RVVStackSize,
- MachineInstr::FrameSetup, !hasFP(MF),
- DynAllocation);
+ MachineInstr::FrameSetup,
+ NeedsDwarfCFI && !hasFP(MF), DynAllocation);
} else {
// We must keep the stack pointer aligned through any intermediate
// updates.
@@ -1097,14 +1119,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
MachineInstr::FrameSetup, getStackAlign());
}
- if (!hasFP(MF)) {
+ if (NeedsDwarfCFI && !hasFP(MF)) {
// Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
CFIBuilder.insertCFIInst(createDefCFAExpression(
*RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
}
std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
- emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
+ if (NeedsDwarfCFI)
+ emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
}
if (hasFP(MF)) {
@@ -1171,8 +1194,9 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
MachineInstr::FrameDestroy, getStackAlign());
StackSize = 0;
- CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
- .buildDefCFAOffset(CFAOffset);
+ if (needsDwarfCFI(MF))
+ CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
+ .buildDefCFAOffset(CFAOffset);
}
void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
@@ -1212,6 +1236,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
MachineInstr::FrameDestroy);
+ bool NeedsDwarfCFI = needsDwarfCFI(MF);
uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
@@ -1232,10 +1257,11 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
StackOffset::getScalable(RVVStackSize),
MachineInstr::FrameDestroy, getStackAlign());
- if (!hasFP(MF))
- CFIBuilder.buildDefCFA(SPReg, RealStackSize);
-
- emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+ if (NeedsDwarfCFI) {
+ if (!hasFP(MF))
+ CFIBuilder.buildDefCFA(SPReg, RealStackSize);
+ emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
+ }
}
if (FirstSPAdjustAmount) {
@@ -1251,7 +1277,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
StackOffset::getFixed(SecondSPAdjustAmount),
MachineInstr::FrameDestroy, getStackAlign());
- if (!hasFP(MF))
+ if (NeedsDwarfCFI && !hasFP(MF))
CFIBuilder.buildDefCFAOffset(FirstSPAdjustAmount);
}
@@ -1272,7 +1298,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
getStackAlign());
}
- if (hasFP(MF))
+ if (NeedsDwarfCFI && hasFP(MF))
CFIBuilder.buildDefCFA(SPReg, RealStackSize);
// Skip to after the restores of scalar callee-saved registers
@@ -1295,8 +1321,9 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
}
// Recover callee-saved registers.
- for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
- CFIBuilder.buildRestore(CS.getReg());
+ if (NeedsDwarfCFI)
+ for (const CalleeSavedInfo &CS : getUnmanagedCSI(MF, CSI))
+ CFIBuilder.buildRestore(CS.getReg());
if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
// Use available stack adjustment in pop instruction to deallocate stack
@@ -1315,15 +1342,17 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
auto NextI = next_nodbg(MBBI, MBB.end());
if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
++MBBI;
- CFIBuilder.setInsertPoint(MBBI);
+ if (NeedsDwarfCFI) {
+ CFIBuilder.setInsertPoint(MBBI);
- for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
- CFIBuilder.buildRestore(CS.getReg());
+ for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
+ CFIBuilder.buildRestore(CS.getReg());
- // Update CFA Offset. If this is a QCI interrupt function, there will be a
- // leftover offset which is deallocated by `QC.C.MILEAVERET`, otherwise
- // getQCIInterruptStackSize() will be 0.
- CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+ // Update CFA Offset. If this is a QCI interrupt function, there will
+ // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
+ // otherwise getQCIInterruptStackSize() will be 0.
+ CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
+ }
}
}
@@ -1812,7 +1841,8 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
// allocateStack.
bool DynAllocation =
MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
- allocateStack(MBB, MI, MF, -Amount, -Amount, !hasFP(MF),
+ allocateStack(MBB, MI, MF, -Amount, -Amount,
+ needsDwarfCFI(MF) && !hasFP(MF),
/*NeedProbe=*/true, ProbeSize, DynAllocation,
MachineInstr::NoFlags);
} else {
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index 82e8ce4e0bd7c..5d5a705893242 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -389,7 +389,7 @@ static bool CC_X86_32_I128_FP128(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
if (!ArgFlags.isInConsecutiveRegsLast())
return true;
- assert(PendingMembers.size() == 4 && "Should have two parts");
+ assert(PendingMembers.size() == 4 && "Should have four parts");
int64_t Offset = State.AllocateStack(16, Align(16));
PendingMembers[0].convertToMem(Offset);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 57fbc71fa22ee..9cd35e35d4bc9 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -176,10 +176,10 @@ constexpr FeatureBitset FeaturesArrowlakeS =
FeaturesArrowlake | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
FeatureSM4;
constexpr FeatureBitset FeaturesPantherlake =
- FeaturesArrowlakeS ^ FeatureWIDEKL | FeaturePREFETCHI;
+ (FeaturesArrowlakeS ^ FeatureWIDEKL) | FeaturePREFETCHI;
constexpr FeatureBitset FeaturesClearwaterforest =
- FeaturesSierraforest ^ FeatureWIDEKL | FeatureAVXVNNIINT16 | FeatureSHA512 |
- FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
+ (FeaturesSierraforest ^ FeatureWIDEKL) | FeatureAVXVNNIINT16 |
+ FeatureSHA512 | FeatureSM3 | FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
// Geode Processor.
constexpr FeatureBitset FeaturesGeode =
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index fbeb7218ba9a3..a65d0fb54c212 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1103,14 +1103,13 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
FrameTy->getElementType(FrameData.getFieldIndex(E.first)), GEP,
SpillAlignment, E.first->getName() + Twine(".reload"));
- TinyPtrVector<DbgDeclareInst *> DIs = findDbgDeclares(Def);
TinyPtrVector<DbgVariableRecord *> DVRs = findDVRDeclares(Def);
// Try best to find dbg.declare. If the spill is a temp, there may not
// be a direct dbg.declare. Walk up the load chain to find one from an
// alias.
if (F->getSubprogram()) {
auto *CurDef = Def;
- while (DIs.empty() && DVRs.empty() && isa<LoadInst>(CurDef)) {
+ while (DVRs.empty() && isa<LoadInst>(CurDef)) {
auto *LdInst = cast<LoadInst>(CurDef);
// Only consider ptr to ptr same type load.
if (LdInst->getPointerOperandType() != LdInst->getType())
@@ -1118,12 +1117,11 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
CurDef = LdInst->getPointerOperand();
if (!isa<AllocaInst, LoadInst>(CurDef))
break;
- DIs = findDbgDeclares(CurDef);
DVRs = findDVRDeclares(CurDef);
}
}
- auto SalvageOne = [&](auto *DDI) {
+ auto SalvageOne = [&](DbgVariableRecord *DDI) {
// This dbg.declare is preserved for all coro-split function
// fragments. It will be unreachable in the main function, and
// processed by coro::salvageDebugInfo() by the Cloner.
@@ -1137,7 +1135,6 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
// will be deleted in all coro-split functions.
coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
};
- for_each(DIs, SalvageOne);
for_each(DVRs, SalvageOne);
}
@@ -1225,8 +1222,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
SmallVector<DbgVariableIntrinsic *, 4> DIs;
SmallVector<DbgVariableRecord *> DbgVariableRecords;
findDbgUsers(DIs, Alloca, &DbgVariableRecords);
- for (auto *DVI : DIs)
- DVI->replaceUsesOfWith(Alloca, G);
+ assert(DIs.empty() && "Should never see debug-intrinsics");
for (auto *DVR : DbgVariableRecords)
DVR->replaceVariableLocationOp(Alloca, G);
@@ -1920,48 +1916,6 @@ salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
return {{*Storage, *Expr}};
}
-void coro::salvageDebugInfo(
- SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
- DbgVariableIntrinsic &DVI, bool UseEntryValue) {
-
- Function *F = DVI.getFunction();
- // Follow the pointer arithmetic all the way to the incoming
- // function argument and convert into a DIExpression.
- bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
- Value *OriginalStorage = DVI.getVariableLocationOp(0);
-
- auto SalvagedInfo =
- ::salvageDebugInfoImpl(ArgToAllocaMap, UseEntryValue, F, OriginalStorage,
- DVI.getExpression(), SkipOutermostLoad);
- if (!SalvagedInfo)
- return;
-
- Value *Storage = &SalvagedInfo->first;
- DIExpression *Expr = &SalvagedInfo->second;
-
- DVI.replaceVariableLocationOp(OriginalStorage, Storage);
- DVI.setExpression(Expr);
- // We only hoist dbg.declare today since it doesn't make sense to hoist
- // dbg.value since it does not have the same function wide guarantees that
- // dbg.declare does.
- if (isa<DbgDeclareInst>(DVI)) {
- std::optional<BasicBlock::iterator> InsertPt;
- if (auto *I = dyn_cast<Instruction>(Storage)) {
- InsertPt = I->getInsertionPointAfterDef();
- // Update DILocation only if variable was not inlined.
- DebugLoc ILoc = I->getDebugLoc();
- DebugLoc DVILoc = DVI.getDebugLoc();
- if (ILoc && DVILoc &&
- DVILoc->getScope()->getSubprogram() ==
- ILoc->getScope()->getSubprogram())
- DVI.setDebugLoc(I->getDebugLoc());
- } else if (isa<Argument>(Storage))
- InsertPt = F->getEntryBlock().begin();
- if (InsertPt)
- DVI.moveBefore(*(*InsertPt)->getParent(), *InsertPt);
- }
-}
-
void coro::salvageDebugInfo(
SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
DbgVariableRecord &DVR, bool UseEntryValue) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index b53c5a48eb10b..52f4ffe292dae 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -34,14 +34,11 @@ void suppressCoroAllocs(CoroIdInst *CoroId);
void suppressCoroAllocs(LLVMContext &Context,
ArrayRef<CoroAllocInst *> CoroAllocs);
-/// Attempts to rewrite the location operand of debug intrinsics in terms of
+/// Attempts to rewrite the location operand of debug records in terms of
/// the coroutine frame pointer, folding pointer offsets into the DIExpression
/// of the intrinsic.
/// If the frame pointer is an Argument, store it into an alloca to enhance the
/// debugability.
-void salvageDebugInfo(
- SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
- DbgVariableIntrinsic &DVI, bool IsEntryPoint);
void salvageDebugInfo(
SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
DbgVariableRecord &DVR, bool UseEntryValue);
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 5a8a41f0dc432..64b33e46404f0 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -618,19 +618,15 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
}
}
-/// Returns all DbgVariableIntrinsic in F.
-static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
- SmallVector<DbgVariableRecord *>>
-collectDbgVariableIntrinsics(Function &F) {
- SmallVector<DbgVariableIntrinsic *, 8> Intrinsics;
+/// Returns all debug records in F.
+static SmallVector<DbgVariableRecord *>
+collectDbgVariableRecords(Function &F) {
SmallVector<DbgVariableRecord *> DbgVariableRecords;
for (auto &I : instructions(F)) {
for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
DbgVariableRecords.push_back(&DVR);
- if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
- Intrinsics.push_back(DVI);
}
- return {Intrinsics, DbgVariableRecords};
+ return DbgVariableRecords;
}
void coro::BaseCloner::replaceSwiftErrorOps() {
@@ -638,13 +634,11 @@ void coro::BaseCloner::replaceSwiftErrorOps() {
}
void coro::BaseCloner::salvageDebugInfo() {
- auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
+ auto DbgVariableRecords = collectDbgVariableRecords(*NewF);
SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
// Only 64-bit ABIs have a register we can refer to with the entry value.
bool UseEntryValue = OrigF.getParent()->getTargetTriple().isArch64Bit();
- for (DbgVariableIntrinsic *DVI : Worklist)
- coro::salvageDebugInfo(ArgToAllocaMap, *DVI, UseEntryValue);
for (DbgVariableRecord *DVR : DbgVariableRecords)
coro::salvageDebugInfo(ArgToAllocaMap, *DVR, UseEntryValue);
@@ -655,7 +649,7 @@ void coro::BaseCloner::salvageDebugInfo() {
return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
&DomTree);
};
- auto RemoveOne = [&](auto *DVI) {
+ auto RemoveOne = [&](DbgVariableRecord *DVI) {
if (IsUnreachableBlock(DVI->getParent()))
DVI->eraseFromParent();
else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
@@ -669,7 +663,6 @@ void coro::BaseCloner::salvageDebugInfo() {
DVI->eraseFromParent();
}
};
- for_each(Worklist, RemoveOne);
for_each(DbgVariableRecords, RemoveOne);
}
@@ -2022,9 +2015,7 @@ static void doSplitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
// original function. The Cloner has already salvaged debug info in the new
// coroutine funclets.
SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
- auto [DbgInsts, DbgVariableRecords] = collectDbgVariableIntrinsics(F);
- for (auto *DDI : DbgInsts)
- coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
+ auto DbgVariableRecords = collectDbgVariableRecords(F);
for (DbgVariableRecord *DVR : DbgVariableRecords)
coro::salvageDebugInfo(ArgToAllocaMap, *DVR, false /*UseEntryValue*/);
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 8017db1cfe146..5fd5f7d9dad09 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -514,7 +514,7 @@ void collectSpillsAndAllocasFromInsts(
void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
const SuspendCrossingInfo &Checker) {
// We don't want the layout of coroutine frame to be affected
- // by debug information. So we only choose to salvage DbgValueInst for
+ // by debug information. So we only choose to salvage dbg.values for
// whose value is already in the frame.
// We would handle the dbg.values for allocas specially
for (auto &Iter : Spills) {
@@ -522,9 +522,7 @@ void collectSpillsFromDbgInfo(SpillInfo &Spills, Function &F,
SmallVector<DbgValueInst *, 16> DVIs;
SmallVector<DbgVariableRecord *, 16> DVRs;
findDbgValues(DVIs, V, &DVRs);
- for (DbgValueInst *DVI : DVIs)
- if (Checker.isDefinitionAcrossSuspend(*V, DVI))
- Spills[V].push_back(DVI);
+ assert(DVIs.empty());
// Add the instructions which carry debug info that is in the frame.
for (DbgVariableRecord *DVR : DVRs)
if (Checker.isDefinitionAcrossSuspend(*V, DVR->Marker->MarkedInstr))
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index d4555e9435f1d..f5525deb0172f 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -572,7 +572,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
// Work out whether a dbg.value intrinsic or an equivalent DbgVariableRecord
// is a parameter to be preserved.
- auto ExamineDbgValue = [](auto *DbgVal, auto &Container) {
+ auto ExamineDbgValue = [&PDVRRelated](DbgVariableRecord *DbgVal) {
LLVM_DEBUG(dbgs() << " Deciding: ");
LLVM_DEBUG(DbgVal->print(dbgs()));
LLVM_DEBUG(dbgs() << "\n");
@@ -581,7 +581,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
LLVM_DEBUG(dbgs() << " Include (parameter): ");
LLVM_DEBUG(DbgVal->print(dbgs()));
LLVM_DEBUG(dbgs() << "\n");
- Container.insert(DbgVal);
+ PDVRRelated.insert(DbgVal);
} else {
LLVM_DEBUG(dbgs() << " Delete (!parameter): ");
LLVM_DEBUG(DbgVal->print(dbgs()));
@@ -589,7 +589,8 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
}
};
- auto ExamineDbgDeclare = [&PDIRelated](auto *DbgDecl, auto &Container) {
+ auto ExamineDbgDeclare = [&PDIRelated,
+ &PDVRRelated](DbgVariableRecord *DbgDecl) {
LLVM_DEBUG(dbgs() << " Deciding: ");
LLVM_DEBUG(DbgDecl->print(dbgs()));
LLVM_DEBUG(dbgs() << "\n");
@@ -616,7 +617,7 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
LLVM_DEBUG(dbgs() << " Include: ");
LLVM_DEBUG(DbgDecl->print(dbgs()));
LLVM_DEBUG(dbgs() << "\n");
- Container.insert(DbgDecl);
+ PDVRRelated.insert(DbgDecl);
} else {
LLVM_DEBUG(dbgs() << " Delete (!parameter): ");
LLVM_DEBUG(SI->print(dbgs()));
@@ -647,18 +648,14 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
// they connected to parameters?
for (DbgVariableRecord &DVR : filterDbgVars(BI->getDbgRecordRange())) {
if (DVR.isDbgValue() || DVR.isDbgAssign()) {
- ExamineDbgValue(&DVR, PDVRRelated);
+ ExamineDbgValue(&DVR);
} else {
assert(DVR.isDbgDeclare());
- ExamineDbgDeclare(&DVR, PDVRRelated);
+ ExamineDbgDeclare(&DVR);
}
}
- if (auto *DVI = dyn_cast<DbgValueInst>(&*BI)) {
- ExamineDbgValue(DVI, PDIRelated);
- } else if (auto *DDI = dyn_cast<DbgDeclareInst>(&*BI)) {
- ExamineDbgDeclare(DDI, PDIRelated);
- } else if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
+ if (BI->isTerminator() && &*BI == GEntryBlock->getTerminator()) {
LLVM_DEBUG(dbgs() << " Will Include Terminator: ");
LLVM_DEBUG(BI->print(dbgs()));
LLVM_DEBUG(dbgs() << "\n");
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 9d7c025ccff86..f7fbf0815df03 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -825,9 +825,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
bool tryToSinkInstruction(Instruction *I, BasicBlock *DestBlock);
- void tryToSinkInstructionDbgValues(
- Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
- BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers);
void tryToSinkInstructionDbgVariableRecords(
Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
BasicBlock *DestBlock, SmallVectorImpl<DbgVariableRecord *> &DPUsers);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 684b9a1f90161..503611a4fc32c 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1420,21 +1420,16 @@ void InstCombinerImpl::freelyInvertAllUsersOf(Value *I, Value *IgnoredUser) {
SmallVector<DbgValueInst *, 4> DbgValues;
SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
+ assert(DbgValues.empty());
- auto InvertDbgValueUse = [&](auto *DbgVal) {
+ for (DbgVariableRecord *DbgVal : DbgVariableRecords) {
SmallVector<uint64_t, 1> Ops = {dwarf::DW_OP_not};
for (unsigned Idx = 0, End = DbgVal->getNumVariableLocationOps();
Idx != End; ++Idx)
if (DbgVal->getVariableLocationOp(Idx) == I)
DbgVal->setExpression(
DIExpression::appendOpsToArg(DbgVal->getExpression(), Ops, Idx));
- };
-
- for (DbgValueInst *DVI : DbgValues)
- InvertDbgValueUse(DVI);
-
- for (DbgVariableRecord *DVR : DbgVariableRecords)
- InvertDbgValueUse(DVR);
+ }
}
/// Given a 'sub' instruction, return the RHS of the instruction if the LHS is a
@@ -3575,6 +3570,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
std::unique_ptr<DIBuilder> DIB;
if (isa<AllocaInst>(MI)) {
findDbgUsers(DVIs, &MI, &DVRs);
+ assert(DVIs.empty());
DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
}
@@ -5253,8 +5249,7 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
SmallVector<DbgVariableIntrinsic *, 2> DbgUsers;
SmallVector<DbgVariableRecord *, 2> DbgVariableRecords;
findDbgUsers(DbgUsers, I, &DbgVariableRecords);
- if (!DbgUsers.empty())
- tryToSinkInstructionDbgValues(I, InsertPos, SrcBlock, DestBlock, DbgUsers);
+ assert(DbgUsers.empty());
if (!DbgVariableRecords.empty())
tryToSinkInstructionDbgVariableRecords(I, InsertPos, SrcBlock, DestBlock,
DbgVariableRecords);
@@ -5271,71 +5266,12 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I,
return true;
}
-void InstCombinerImpl::tryToSinkInstructionDbgValues(
- Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
- BasicBlock *DestBlock, SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers) {
- // For all debug values in the destination block, the sunk instruction
- // will still be available, so they do not need to be dropped.
- SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSalvage;
- for (auto &DbgUser : DbgUsers)
- if (DbgUser->getParent() != DestBlock)
- DbgUsersToSalvage.push_back(DbgUser);
-
- // Process the sinking DbgUsersToSalvage in reverse order, as we only want
- // to clone the last appearing debug intrinsic for each given variable.
- SmallVector<DbgVariableIntrinsic *, 2> DbgUsersToSink;
- for (DbgVariableIntrinsic *DVI : DbgUsersToSalvage)
- if (DVI->getParent() == SrcBlock)
- DbgUsersToSink.push_back(DVI);
- llvm::sort(DbgUsersToSink,
- [](auto *A, auto *B) { return B->comesBefore(A); });
-
- SmallVector<DbgVariableIntrinsic *, 2> DIIClones;
- SmallSet<DebugVariable, 4> SunkVariables;
- for (auto *User : DbgUsersToSink) {
- // A dbg.declare instruction should not be cloned, since there can only be
- // one per variable fragment. It should be left in the original place
- // because the sunk instruction is not an alloca (otherwise we could not be
- // here).
- if (isa<DbgDeclareInst>(User))
- continue;
-
- DebugVariable DbgUserVariable =
- DebugVariable(User->getVariable(), User->getExpression(),
- User->getDebugLoc()->getInlinedAt());
-
- if (!SunkVariables.insert(DbgUserVariable).second)
- continue;
-
- // Leave dbg.assign intrinsics in their original positions and there should
- // be no need to insert a clone.
- if (isa<DbgAssignIntrinsic>(User))
- continue;
-
- DIIClones.emplace_back(cast<DbgVariableIntrinsic>(User->clone()));
- if (isa<DbgDeclareInst>(User) && isa<CastInst>(I))
- DIIClones.back()->replaceVariableLocationOp(I, I->getOperand(0));
- LLVM_DEBUG(dbgs() << "CLONE: " << *DIIClones.back() << '\n');
- }
-
- // Perform salvaging without the clones, then sink the clones.
- if (!DIIClones.empty()) {
- salvageDebugInfoForDbgValues(*I, DbgUsersToSalvage, {});
- // The clones are in reverse order of original appearance, reverse again to
- // maintain the original order.
- for (auto &DIIClone : llvm::reverse(DIIClones)) {
- DIIClone->insertBefore(InsertPos);
- LLVM_DEBUG(dbgs() << "SINK: " << *DIIClone << '\n');
- }
- }
-}
-
void InstCombinerImpl::tryToSinkInstructionDbgVariableRecords(
Instruction *I, BasicBlock::iterator InsertPos, BasicBlock *SrcBlock,
BasicBlock *DestBlock,
SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
- // Implementation of tryToSinkInstructionDbgValues, but for the
- // DbgVariableRecord of variable assignments rather than dbg.values.
+ // For all debug values in the destination block, the sunk instruction
+ // will still be available, so they do not need to be dropped.
// Fetch all DbgVariableRecords not already in the destination.
SmallVector<DbgVariableRecord *, 2> DbgVariableRecordsToSalvage;
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 2786d81773ed9..df3160233c510 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1489,6 +1489,7 @@ static bool checkAndReplaceCondition(
SmallVector<DbgVariableIntrinsic *> DbgUsers;
SmallVector<DbgVariableRecord *> DVRUsers;
findDbgUsers(DbgUsers, Cmp, &DVRUsers);
+ assert(DbgUsers.empty());
for (auto *DVR : DVRUsers) {
auto *DTN = DT.getNode(DVR->getParent());
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 85dd9a1bf7161..0f63ed0166cf4 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -2079,6 +2079,7 @@ struct DSEState {
AllocFnKind AllocKind =
Attrs.getFnAttr(Attribute::AllocKind).getAllocKind() |
AllocFnKind::Zeroed;
+ AllocKind &= ~AllocFnKind::Uninitialized;
Attrs =
Attrs.addFnAttribute(Ctx, Attribute::getWithAllocKind(Ctx, AllocKind))
.removeFnAttribute(Ctx, "alloc-variant-zeroed");
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index b5dbef13289ac..4d1f44076db7e 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1979,15 +1979,13 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
// Find debug values outside of the block
findDbgValues(DbgValues, &I, &DbgVariableRecords);
- llvm::erase_if(DbgValues, [&](const DbgValueInst *DbgVal) {
- return DbgVal->getParent() == BB;
- });
+ assert(DbgValues.empty());
llvm::erase_if(DbgVariableRecords, [&](const DbgVariableRecord *DbgVarRec) {
return DbgVarRec->getParent() == BB;
});
// If there are no uses outside the block, we're done with this instruction.
- if (UsesToRename.empty() && DbgValues.empty() && DbgVariableRecords.empty())
+ if (UsesToRename.empty() && DbgVariableRecords.empty())
continue;
LLVM_DEBUG(dbgs() << "JT: Renaming non-local uses of: " << I << "\n");
@@ -2000,8 +1998,7 @@ void JumpThreadingPass::updateSSA(BasicBlock *BB, BasicBlock *NewBB,
while (!UsesToRename.empty())
SSAUpdate.RewriteUse(*UsesToRename.pop_back_val());
- if (!DbgValues.empty() || !DbgVariableRecords.empty()) {
- SSAUpdate.UpdateDebugValues(&I, DbgValues);
+ if (!DbgVariableRecords.empty()) {
SSAUpdate.UpdateDebugValues(&I, DbgVariableRecords);
DbgValues.clear();
DbgVariableRecords.clear();
@@ -2032,32 +2029,7 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
// copy of the block 'NewBB'. If there are PHI nodes in the source basic
// block, evaluate them to account for entry from PredBB.
- // Retargets llvm.dbg.value to any renamed variables.
- auto RetargetDbgValueIfPossible = [&](Instruction *NewInst) -> bool {
- auto DbgInstruction = dyn_cast<DbgValueInst>(NewInst);
- if (!DbgInstruction)
- return false;
-
- SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
- for (auto DbgOperand : DbgInstruction->location_ops()) {
- auto DbgOperandInstruction = dyn_cast<Instruction>(DbgOperand);
- if (!DbgOperandInstruction)
- continue;
-
- auto I = ValueMapping.find(DbgOperandInstruction);
- if (I != ValueMapping.end()) {
- OperandsToRemap.insert(
- std::pair<Value *, Value *>(DbgOperand, I->second));
- }
- }
-
- for (auto &[OldOp, MappedOp] : OperandsToRemap)
- DbgInstruction->replaceVariableLocationOp(OldOp, MappedOp);
- return true;
- };
-
- // Duplicate implementation of the above dbg.value code, using
- // DbgVariableRecords instead.
+ // Retargets dbg.value to any renamed variables.
auto RetargetDbgVariableRecordIfPossible = [&](DbgVariableRecord *DVR) {
SmallSet<std::pair<Value *, Value *>, 16> OperandsToRemap;
for (auto *Op : DVR->location_ops()) {
@@ -2116,9 +2088,6 @@ void JumpThreadingPass::cloneInstructions(ValueToValueMapTy &ValueMapping,
if (const DebugLoc &DL = New->getDebugLoc())
mapAtomInstance(DL, ValueMapping);
- if (RetargetDbgValueIfPossible(New))
- continue;
-
// Remap operands to patch up intra-block references.
for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
if (Instruction *Inst = dyn_cast<Instruction>(New->getOperand(i))) {
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index a0f9f3c4a35a5..70e9eee5339a7 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -78,6 +78,7 @@ enum class RuleTy {
PerLoopCacheAnalysis,
PerInstrOrderCost,
ForVectorization,
+ Ignore
};
} // end anonymous namespace
@@ -106,14 +107,20 @@ static cl::list<RuleTy> Profitabilities(
clEnumValN(RuleTy::PerInstrOrderCost, "instorder",
"Prioritize the IVs order of each instruction"),
clEnumValN(RuleTy::ForVectorization, "vectorize",
- "Prioritize vectorization")));
+ "Prioritize vectorization"),
+ clEnumValN(RuleTy::Ignore, "ignore",
+ "Ignore profitability, force interchange (does not "
+ "work with other options)")));
#ifndef NDEBUG
-static bool noDuplicateRules(ArrayRef<RuleTy> Rules) {
+static bool noDuplicateRulesAndIgnore(ArrayRef<RuleTy> Rules) {
SmallSet<RuleTy, 4> Set;
- for (RuleTy Rule : Rules)
+ for (RuleTy Rule : Rules) {
if (!Set.insert(Rule).second)
return false;
+ if (Rule == RuleTy::Ignore)
+ return false;
+ }
return true;
}
@@ -1357,6 +1364,13 @@ std::optional<bool> LoopInterchangeProfitability::isProfitableForVectorization(
bool LoopInterchangeProfitability::isProfitable(
const Loop *InnerLoop, const Loop *OuterLoop, unsigned InnerLoopId,
unsigned OuterLoopId, CharMatrix &DepMatrix, CacheCostManager &CCM) {
+
+ // Return true if interchange is forced and the cost-model ignored.
+ if (Profitabilities.size() == 1 && Profitabilities[0] == RuleTy::Ignore)
+ return true;
+ assert(noDuplicateRulesAndIgnore(Profitabilities) &&
+ "Duplicate rules and option 'ignore' are not allowed");
+
// isProfitable() is structured to avoid endless loop interchange. If the
// highest priority rule (isProfitablePerLoopCacheAnalysis by default) could
// decide the profitability then, profitability check will stop and return the
@@ -1365,7 +1379,6 @@ bool LoopInterchangeProfitability::isProfitable(
// second highest priority rule (isProfitablePerInstrOrderCost by default).
// Likewise, if it failed to analysis the profitability then only, the last
// rule (isProfitableForVectorization by default) will decide.
- assert(noDuplicateRules(Profitabilities) && "Detect duplicate rules");
std::optional<bool> shouldInterchange;
for (RuleTy RT : Profitabilities) {
switch (RT) {
@@ -1382,6 +1395,9 @@ bool LoopInterchangeProfitability::isProfitable(
shouldInterchange =
isProfitableForVectorization(InnerLoopId, OuterLoopId, DepMatrix);
break;
+ case RuleTy::Ignore:
+ llvm_unreachable("Option 'ignore' is not supported with other options");
+ break;
}
// If this rule could determine the profitability, don't call subsequent
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index dc8fa4379752f..9e318b04c2c99 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -3790,6 +3790,11 @@ LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
continue;
}
+ // Do not consider uses inside lifetime intrinsics. These are not
+ // actually materialized.
+ if (UserInst->isLifetimeStartOrEnd())
+ continue;
+
std::pair<size_t, Immediate> P =
getUse(S, LSRUse::Basic, MemAccessTy());
size_t LUIdx = P.first;
@@ -6630,13 +6635,10 @@ struct SCEVDbgValueBuilder {
/// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
/// and DIExpression.
struct DVIRecoveryRec {
- DVIRecoveryRec(DbgValueInst *DbgValue)
- : DbgRef(DbgValue), Expr(DbgValue->getExpression()),
- HadLocationArgList(false) {}
DVIRecoveryRec(DbgVariableRecord *DVR)
: DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
- PointerUnion<DbgValueInst *, DbgVariableRecord *> DbgRef;
+ DbgVariableRecord *DbgRef;
DIExpression *Expr;
bool HadLocationArgList;
SmallVector<WeakVH, 2> LocationOps;
@@ -6695,44 +6697,38 @@ static void updateDVIWithLocations(T &DbgVal,
}
/// Write the new expression and new location ops for the dbg.value. If possible
-/// reduce the szie of the dbg.value intrinsic by omitting DIArglist. This
+/// reduce the szie of the dbg.value by omitting DIArglist. This
/// can be omitted if:
/// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
/// 2. The DW_OP_LLVM_arg is the first operand in the expression.
-static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec,
- SmallVectorImpl<Value *> &NewLocationOps,
- SmallVectorImpl<uint64_t> &NewExpr) {
- auto UpdateDbgValueInstImpl = [&](auto *DbgVal) {
- unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
- if (NumLLVMArgs == 0) {
- // Location assumed to be on the stack.
- updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
- } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
- // There is only a single DW_OP_llvm_arg at the start of the expression,
- // so it can be omitted along with DIArglist.
- assert(NewExpr[1] == 0 &&
- "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
- llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
- updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
- } else {
- // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
- updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
- }
+static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
+ SmallVectorImpl<Value *> &NewLocationOps,
+ SmallVectorImpl<uint64_t> &NewExpr) {
+ DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+ unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
+ if (NumLLVMArgs == 0) {
+ // Location assumed to be on the stack.
+ updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
+ } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
+ // There is only a single DW_OP_llvm_arg at the start of the expression,
+ // so it can be omitted along with DIArglist.
+ assert(NewExpr[1] == 0 &&
+ "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
+ llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2));
+ updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
+ } else {
+ // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
+ updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
+ }
- // If the DIExpression was previously empty then add the stack terminator.
- // Non-empty expressions have only had elements inserted into them and so
- // the terminator should already be present e.g. stack_value or fragment.
- DIExpression *SalvageExpr = DbgVal->getExpression();
- if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
- SalvageExpr =
- DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
- DbgVal->setExpression(SalvageExpr);
- }
- };
- if (isa<DbgValueInst *>(DVIRec.DbgRef))
- UpdateDbgValueInstImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
- else
- UpdateDbgValueInstImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+ // If the DIExpression was previously empty then add the stack terminator.
+ // Non-empty expressions have only had elements inserted into them and so
+ // the terminator should already be present e.g. stack_value or fragment.
+ DIExpression *SalvageExpr = DbgVal->getExpression();
+ if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
+ SalvageExpr = DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
+ DbgVal->setExpression(SalvageExpr);
+ }
}
/// Cached location ops may be erased during LSR, in which case a poison is
@@ -6746,39 +6742,34 @@ static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
/// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
- auto RestorePreTransformStateImpl = [&](auto *DbgVal) {
- LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
- << "scev-salvage: post-LSR: " << *DbgVal << '\n');
- assert(DVIRec.Expr && "Expected an expression");
- DbgVal->setExpression(DVIRec.Expr);
-
- // Even a single location-op may be inside a DIArgList and referenced with
- // DW_OP_LLVM_arg, which is valid only with a DIArgList.
- if (!DVIRec.HadLocationArgList) {
- assert(DVIRec.LocationOps.size() == 1 &&
- "Unexpected number of location ops.");
- // LSR's unsuccessful salvage attempt may have added DIArgList, which in
- // this case was not present before, so force the location back to a
- // single uncontained Value.
- Value *CachedValue =
- getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
- DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
- } else {
- SmallVector<ValueAsMetadata *, 3> MetadataLocs;
- for (WeakVH VH : DVIRec.LocationOps) {
- Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
- MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
- }
- auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
- DbgVal->setRawLocation(
- llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+ DbgVariableRecord *DbgVal = DVIRec.DbgRef;
+ LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
+ << "scev-salvage: post-LSR: " << *DbgVal << '\n');
+ assert(DVIRec.Expr && "Expected an expression");
+ DbgVal->setExpression(DVIRec.Expr);
+
+ // Even a single location-op may be inside a DIArgList and referenced with
+ // DW_OP_LLVM_arg, which is valid only with a DIArgList.
+ if (!DVIRec.HadLocationArgList) {
+ assert(DVIRec.LocationOps.size() == 1 &&
+ "Unexpected number of location ops.");
+ // LSR's unsuccessful salvage attempt may have added DIArgList, which in
+ // this case was not present before, so force the location back to a
+ // single uncontained Value.
+ Value *CachedValue =
+ getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
+ DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
+ } else {
+ SmallVector<ValueAsMetadata *, 3> MetadataLocs;
+ for (WeakVH VH : DVIRec.LocationOps) {
+ Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
+ MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
}
- LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
- };
- if (isa<DbgValueInst *>(DVIRec.DbgRef))
- RestorePreTransformStateImpl(cast<DbgValueInst *>(DVIRec.DbgRef));
- else
- RestorePreTransformStateImpl(cast<DbgVariableRecord *>(DVIRec.DbgRef));
+ auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
+ DbgVal->setRawLocation(
+ llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
+ }
+ LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
}
static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
@@ -6786,9 +6777,7 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
const SCEV *SCEVInductionVar,
SCEVDbgValueBuilder IterCountExpr) {
- if (isa<DbgValueInst *>(DVIRec.DbgRef)
- ? !cast<DbgValueInst *>(DVIRec.DbgRef)->isKillLocation()
- : !cast<DbgVariableRecord *>(DVIRec.DbgRef)->isKillLocation())
+ if (!DVIRec.DbgRef->isKillLocation())
return false;
// LSR may have caused several changes to the dbg.value in the failed salvage
@@ -6882,13 +6871,8 @@ static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
DbgBuilder->appendToVectors(NewExpr, NewLocationOps);
}
- UpdateDbgValueInst(DVIRec, NewLocationOps, NewExpr);
- if (isa<DbgValueInst *>(DVIRec.DbgRef))
- LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
- << *cast<DbgValueInst *>(DVIRec.DbgRef) << "\n");
- else
- LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: "
- << *cast<DbgVariableRecord *>(DVIRec.DbgRef) << "\n");
+ UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
+ LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
return true;
}
@@ -6934,21 +6918,23 @@ static void DbgRewriteSalvageableDVIs(
/// cacheing and salvaging.
static void DbgGatherSalvagableDVI(
Loop *L, ScalarEvolution &SE,
- SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs,
- SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) {
+ SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs) {
for (const auto &B : L->getBlocks()) {
for (auto &I : *B) {
- auto ProcessDbgValue = [&](auto *DbgVal) -> bool {
+ for (DbgVariableRecord &DbgVal : filterDbgVars(I.getDbgRecordRange())) {
+ if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
+ continue;
+
// Ensure that if any location op is undef that the dbg.vlue is not
// cached.
- if (DbgVal->isKillLocation())
- return false;
+ if (DbgVal.isKillLocation())
+ continue;
// Check that the location op SCEVs are suitable for translation to
// DIExpression.
const auto &HasTranslatableLocationOps =
- [&](const auto *DbgValToTranslate) -> bool {
- for (const auto LocOp : DbgValToTranslate->location_ops()) {
+ [&](const DbgVariableRecord &DbgValToTranslate) -> bool {
+ for (const auto LocOp : DbgValToTranslate.location_ops()) {
if (!LocOp)
return false;
@@ -6963,31 +6949,21 @@ static void DbgGatherSalvagableDVI(
};
if (!HasTranslatableLocationOps(DbgVal))
- return false;
+ continue;
std::unique_ptr<DVIRecoveryRec> NewRec =
- std::make_unique<DVIRecoveryRec>(DbgVal);
+ std::make_unique<DVIRecoveryRec>(&DbgVal);
// Each location Op may need a SCEVDbgValueBuilder in order to recover
// it. Pre-allocating a vector will enable quick lookups of the builder
// later during the salvage.
- NewRec->RecoveryExprs.resize(DbgVal->getNumVariableLocationOps());
- for (const auto LocOp : DbgVal->location_ops()) {
+ NewRec->RecoveryExprs.resize(DbgVal.getNumVariableLocationOps());
+ for (const auto LocOp : DbgVal.location_ops()) {
NewRec->SCEVs.push_back(SE.getSCEV(LocOp));
NewRec->LocationOps.push_back(LocOp);
- NewRec->HadLocationArgList = DbgVal->hasArgList();
+ NewRec->HadLocationArgList = DbgVal.hasArgList();
}
SalvageableDVISCEVs.push_back(std::move(NewRec));
- return true;
- };
- for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
- if (DVR.isDbgValue() || DVR.isDbgAssign())
- ProcessDbgValue(&DVR);
}
- auto DVI = dyn_cast<DbgValueInst>(&I);
- if (!DVI)
- continue;
- if (ProcessDbgValue(DVI))
- DVIHandles.insert(DVI);
}
}
}
@@ -7036,8 +7012,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
// Debug preservation - before we start removing anything identify which DVI
// meet the salvageable criteria and store their DIExpression and SCEVs.
SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
- SmallSet<AssertingVH<DbgValueInst>, 2> DVIHandles;
- DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles);
+ DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords);
bool Changed = false;
std::unique_ptr<MemorySSAUpdater> MSSAU;
@@ -7105,7 +7080,6 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
for (auto &Rec : SalvageableDVIRecords)
Rec->clear();
SalvageableDVIRecords.clear();
- DVIHandles.clear();
return Changed;
}
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index eacaf42e4e8ba..1d1af42153325 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1222,9 +1222,7 @@ static void eraseDebugIntrinsicsWithNonLocalRefs(Function &F) {
SmallVector<DbgVariableIntrinsic *, 4> DbgUsers;
SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
findDbgUsers(DbgUsers, &I, &DbgVariableRecords);
- for (DbgVariableIntrinsic *DVI : DbgUsers)
- if (DVI->getFunction() != &F)
- DVI->eraseFromParent();
+ assert(DbgUsers.empty());
for (DbgVariableRecord *DVR : DbgVariableRecords)
if (DVR->getFunction() != &F)
DVR->eraseFromParent();
@@ -1289,14 +1287,12 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgUsers(DbgUsers, Input, &DPUsers);
+ assert(DbgUsers.empty());
DIExpression *Expr = DIB.createExpression();
// Iterate the debud users of the Input values. If they are in the extracted
// function then update their location with the new value. If they are in
// the parent function then create a similar debug record.
- for (auto *DVI : DbgUsers)
- UpdateOrInsertDebugRecord(DVI, Input, NewVal, Expr,
- isa<DbgDeclareInst>(DVI));
for (auto *DVR : DPUsers)
UpdateOrInsertDebugRecord(DVR, Input, NewVal, Expr, DVR->isDbgDeclare());
}
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index a1f030a336c15..4210ce6da1eb2 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -808,9 +808,6 @@ bool checkDebugifyMetadata(Module &M,
// Find missing lines.
for (Instruction &I : instructions(F)) {
- if (isa<DbgValueInst>(&I))
- continue;
-
auto DL = I.getDebugLoc();
if (DL && DL.getLine() != 0) {
MissingLines.reset(DL.getLine() - 1);
@@ -839,10 +836,6 @@ bool checkDebugifyMetadata(Module &M,
for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
if (DVR.isDbgValue() || DVR.isDbgAssign())
CheckForMisSized(&DVR);
- auto *DVI = dyn_cast<DbgValueInst>(&I);
- if (!DVI)
- continue;
- CheckForMisSized(DVI);
}
}
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 6929d14bc56ea..ed3dca2f7c307 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1978,14 +1978,13 @@ static at::StorageToVarsMap collectEscapedLocals(const DataLayout &DL,
continue;
// Find all local variables associated with the backing storage.
- auto CollectAssignsForStorage = [&](auto *DbgAssign) {
+ auto CollectAssignsForStorage = [&](DbgVariableRecord *DbgAssign) {
// Skip variables from inlined functions - they are not local variables.
if (DbgAssign->getDebugLoc().getInlinedAt())
return;
LLVM_DEBUG(errs() << " > DEF : " << *DbgAssign << "\n");
EscapedLocals[Base].insert(at::VarRecord(DbgAssign));
};
- for_each(at::getAssignmentMarkers(Base), CollectAssignsForStorage);
for_each(at::getDVRAssignmentMarkers(Base), CollectAssignsForStorage);
}
return EscapedLocals;
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index b14bbeac97675..d481ad9dee181 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -613,11 +613,10 @@ bool llvm::replaceDbgUsesWithUndef(Instruction *I) {
SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgUsers(DbgUsers, I, &DPUsers);
- for (auto *DII : DbgUsers)
- DII->setKillLocation();
+ assert(DbgUsers.empty());
for (auto *DVR : DPUsers)
DVR->setKillLocation();
- return !DbgUsers.empty() || !DPUsers.empty();
+ return !DPUsers.empty();
}
/// areAllUsesEqual - Check whether the uses of a value are all the same.
@@ -1607,12 +1606,8 @@ static bool PhiHasDebugValue(DILocalVariable *DIVar,
SmallVector<DbgValueInst *, 1> DbgValues;
SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
findDbgValues(DbgValues, APN, &DbgVariableRecords);
- for (auto *DVI : DbgValues) {
- assert(is_contained(DVI->getValues(), APN));
- if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
- return true;
- }
- for (auto *DVR : DbgVariableRecords) {
+ assert(DbgValues.empty());
+ for (DbgVariableRecord *DVR : DbgVariableRecords) {
assert(is_contained(DVR->location_ops(), APN));
if ((DVR->getVariable() == DIVar) && (DVR->getExpression() == DIExpr))
return true;
@@ -1971,7 +1966,6 @@ bool llvm::replaceDbgDeclare(Value *Address, Value *NewAddress,
static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
DILocalVariable *DIVar,
DIExpression *DIExpr, Value *NewAddress,
- DbgValueInst *DVI,
DbgVariableRecord *DVR,
DIBuilder &Builder, int Offset) {
assert(DIVar && "Missing variable");
@@ -1987,14 +1981,8 @@ static void updateOneDbgValueForAlloca(const DebugLoc &Loc,
if (Offset)
DIExpr = DIExpression::prepend(DIExpr, 0, Offset);
- if (DVI) {
- DVI->setExpression(DIExpr);
- DVI->replaceVariableLocationOp(0u, NewAddress);
- } else {
- assert(DVR);
- DVR->setExpression(DIExpr);
- DVR->replaceVariableLocationOp(0u, NewAddress);
- }
+ DVR->setExpression(DIExpr);
+ DVR->replaceVariableLocationOp(0u, NewAddress);
}
void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
@@ -2002,18 +1990,13 @@ void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
SmallVector<DbgValueInst *, 1> DbgUsers;
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgValues(DbgUsers, AI, &DPUsers);
-
- // Attempt to replace dbg.values that use this alloca.
- for (auto *DVI : DbgUsers)
- updateOneDbgValueForAlloca(DVI->getDebugLoc(), DVI->getVariable(),
- DVI->getExpression(), NewAllocaAddress, DVI,
- nullptr, Builder, Offset);
+ assert(DbgUsers.empty());
// Replace any DbgVariableRecords that use this alloca.
for (DbgVariableRecord *DVR : DPUsers)
updateOneDbgValueForAlloca(DVR->getDebugLoc(), DVR->getVariable(),
- DVR->getExpression(), NewAllocaAddress, nullptr,
- DVR, Builder, Offset);
+ DVR->getExpression(), NewAllocaAddress, DVR,
+ Builder, Offset);
}
/// Where possible to salvage debug information for \p I do so.
@@ -2022,6 +2005,7 @@ void llvm::salvageDebugInfo(Instruction &I) {
SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgUsers(DbgUsers, &I, &DPUsers);
+ assert(DbgUsers.empty());
salvageDebugInfoForDbgValues(I, DbgUsers, DPUsers);
}
@@ -2070,66 +2054,9 @@ void llvm::salvageDebugInfoForDbgValues(
const unsigned MaxExpressionSize = 128;
bool Salvaged = false;
- for (auto *DII : DbgUsers) {
- if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DII)) {
- if (DAI->getAddress() == &I) {
- salvageDbgAssignAddress(DAI);
- Salvaged = true;
- }
- if (DAI->getValue() != &I)
- continue;
- }
+ // We should never see debug intrinsics nowadays.
+ assert(DbgUsers.empty());
- // Do not add DW_OP_stack_value for DbgDeclare, because they are implicitly
- // pointing out the value as a DWARF memory location description.
- bool StackValue = isa<DbgValueInst>(DII);
- auto DIILocation = DII->location_ops();
- assert(
- is_contained(DIILocation, &I) &&
- "DbgVariableIntrinsic must use salvaged instruction as its location");
- SmallVector<Value *, 4> AdditionalValues;
- // `I` may appear more than once in DII's location ops, and each use of `I`
- // must be updated in the DIExpression and potentially have additional
- // values added; thus we call salvageDebugInfoImpl for each `I` instance in
- // DIILocation.
- Value *Op0 = nullptr;
- DIExpression *SalvagedExpr = DII->getExpression();
- auto LocItr = find(DIILocation, &I);
- while (SalvagedExpr && LocItr != DIILocation.end()) {
- SmallVector<uint64_t, 16> Ops;
- unsigned LocNo = std::distance(DIILocation.begin(), LocItr);
- uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
- Op0 = salvageDebugInfoImpl(I, CurrentLocOps, Ops, AdditionalValues);
- if (!Op0)
- break;
- SalvagedExpr =
- DIExpression::appendOpsToArg(SalvagedExpr, Ops, LocNo, StackValue);
- LocItr = std::find(++LocItr, DIILocation.end(), &I);
- }
- // salvageDebugInfoImpl should fail on examining the first element of
- // DbgUsers, or none of them.
- if (!Op0)
- break;
-
- SalvagedExpr = SalvagedExpr->foldConstantMath();
- DII->replaceVariableLocationOp(&I, Op0);
- bool IsValidSalvageExpr = SalvagedExpr->getNumElements() <= MaxExpressionSize;
- if (AdditionalValues.empty() && IsValidSalvageExpr) {
- DII->setExpression(SalvagedExpr);
- } else if (isa<DbgValueInst>(DII) && IsValidSalvageExpr &&
- DII->getNumVariableLocationOps() + AdditionalValues.size() <=
- MaxDebugArgs) {
- DII->addVariableLocationOps(AdditionalValues, SalvagedExpr);
- } else {
- // Do not salvage using DIArgList for dbg.declare, as it is not currently
- // supported in those instructions. Also do not salvage if the resulting
- // DIArgList would contain an unreasonably large number of values.
- DII->setKillLocation();
- }
- LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
- Salvaged = true;
- }
- // Duplicate of above block for DbgVariableRecords.
for (auto *DVR : DPUsers) {
if (DVR->isDbgAssign()) {
if (DVR->getAddress() == &I) {
@@ -2198,9 +2125,6 @@ void llvm::salvageDebugInfoForDbgValues(
if (Salvaged)
return;
- for (auto *DII : DbgUsers)
- DII->setKillLocation();
-
for (auto *DVR : DPUsers)
DVR->setKillLocation();
}
@@ -3429,8 +3353,7 @@ void llvm::dropDebugUsers(Instruction &I) {
SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgUsers(DbgUsers, &I, &DPUsers);
- for (auto *DII : DbgUsers)
- DII->eraseFromParent();
+ assert(DbgUsers.empty());
for (auto *DVR : DPUsers)
DVR->eraseFromParent();
}
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 66d0573e83f65..06115e0741ade 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -161,29 +161,8 @@ static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader,
SmallVector<DbgValueInst *, 1> DbgValues;
SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
llvm::findDbgValues(DbgValues, OrigHeaderVal, &DbgVariableRecords);
- for (auto &DbgValue : DbgValues) {
- // The original users in the OrigHeader are already using the original
- // definitions.
- BasicBlock *UserBB = DbgValue->getParent();
- if (UserBB == OrigHeader)
- continue;
-
- // Users in the OrigPreHeader need to use the value to which the
- // original definitions are mapped and anything else can be handled by
- // the SSAUpdater. To avoid adding PHINodes, check if the value is
- // available in UserBB, if not substitute poison.
- Value *NewVal;
- if (UserBB == OrigPreheader)
- NewVal = OrigPreHeaderVal;
- else if (SSA.HasValueForBlock(UserBB))
- NewVal = SSA.GetValueInMiddleOfBlock(UserBB);
- else
- NewVal = PoisonValue::get(OrigHeaderVal->getType());
- DbgValue->replaceVariableLocationOp(OrigHeaderVal, NewVal);
- }
+ assert(DbgValues.empty());
- // RemoveDIs: duplicate implementation for non-instruction debug-info
- // storage in DbgVariableRecords.
for (DbgVariableRecord *DVR : DbgVariableRecords) {
// The original users in the OrigHeader are already using the original
// definitions.
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 154f50acf3d1d..e7623aaff105d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1103,7 +1103,6 @@ Value *llvm::createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left,
(RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum ||
RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum)) {
- // TODO: Add float minnum/maxnum support when FMF nnan is set.
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RK);
return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,
"rdx.minmax");
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index ccd7ee360e014..73b5f48796b7a 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -190,7 +190,6 @@ class AssignmentTrackingInfo {
};
struct AllocaInfo {
- using DbgUserVec = SmallVector<DbgVariableIntrinsic *, 1>;
using DPUserVec = SmallVector<DbgVariableRecord *, 1>;
SmallVector<BasicBlock *, 32> DefiningBlocks;
@@ -201,7 +200,6 @@ struct AllocaInfo {
bool OnlyUsedInOneBlock;
/// Debug users of the alloca - does not include dbg.assign intrinsics.
- DbgUserVec DbgUsers;
DPUserVec DPUsers;
/// Helper to update assignment tracking debug info.
AssignmentTrackingInfo AssignmentTracking;
@@ -212,7 +210,6 @@ struct AllocaInfo {
OnlyStore = nullptr;
OnlyBlock = nullptr;
OnlyUsedInOneBlock = true;
- DbgUsers.clear();
DPUsers.clear();
AssignmentTracking.clear();
}
@@ -246,13 +243,10 @@ struct AllocaInfo {
OnlyUsedInOneBlock = false;
}
}
- DbgUserVec AllDbgUsers;
+ SmallVector<DbgVariableIntrinsic *> AllDbgUsers;
SmallVector<DbgVariableRecord *> AllDPUsers;
findDbgUsers(AllDbgUsers, AI, &AllDPUsers);
- std::copy_if(AllDbgUsers.begin(), AllDbgUsers.end(),
- std::back_inserter(DbgUsers), [](DbgVariableIntrinsic *DII) {
- return !isa<DbgAssignIntrinsic>(DII);
- });
+ assert(AllDbgUsers.empty());
std::copy_if(AllDPUsers.begin(), AllDPUsers.end(),
std::back_inserter(DPUsers),
[](DbgVariableRecord *DVR) { return !DVR->isDbgAssign(); });
@@ -380,10 +374,9 @@ struct PromoteMem2Reg {
/// to.
DenseMap<PHINode *, unsigned> PhiToAllocaMap;
- /// For each alloca, we keep track of the dbg.declare intrinsic that
+ /// For each alloca, we keep track of the dbg.declare record that
/// describes it, if any, so that we can convert it to a dbg.value
- /// intrinsic if the alloca gets promoted.
- SmallVector<AllocaInfo::DbgUserVec, 8> AllocaDbgUsers;
+ /// record if the alloca gets promoted.
SmallVector<AllocaInfo::DPUserVec, 8> AllocaDPUsers;
/// For each alloca, keep an instance of a helper class that gives us an easy
@@ -741,14 +734,11 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
AI->eraseFromParent();
// The alloca's debuginfo can be removed as well.
- auto DbgUpdateForAlloca = [&](auto &Container) {
- for (auto *DbgItem : Container)
- if (DbgItem->isAddressOfVariable() ||
- DbgItem->getExpression()->startsWithDeref())
- DbgItem->eraseFromParent();
- };
- DbgUpdateForAlloca(Info.DbgUsers);
- DbgUpdateForAlloca(Info.DPUsers);
+ for (DbgVariableRecord *DbgItem : Info.DPUsers) {
+ if (DbgItem->isAddressOfVariable() ||
+ DbgItem->getExpression()->startsWithDeref())
+ DbgItem->eraseFromParent();
+ }
++NumLocalPromoted;
return true;
@@ -757,7 +747,6 @@ promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
void PromoteMem2Reg::run() {
Function &F = *DT.getRoot()->getParent();
- AllocaDbgUsers.resize(Allocas.size());
AllocaATInfo.resize(Allocas.size());
AllocaDPUsers.resize(Allocas.size());
@@ -816,9 +805,7 @@ void PromoteMem2Reg::run() {
if (BBNumPreds.empty())
BBNumPreds.resize(F.getMaxBlockNumber());
- // Remember the dbg.declare intrinsic describing this alloca, if any.
- if (!Info.DbgUsers.empty())
- AllocaDbgUsers[AllocaNum] = Info.DbgUsers;
+ // Remember the dbg.declare record describing this alloca, if any.
if (!Info.AssignmentTracking.empty())
AllocaATInfo[AllocaNum] = Info.AssignmentTracking;
if (!Info.DPUsers.empty())
@@ -894,16 +881,12 @@ void PromoteMem2Reg::run() {
}
// Remove alloca's dbg.declare intrinsics from the function.
- auto RemoveDbgDeclares = [&](auto &Container) {
- for (auto &DbgUsers : Container) {
- for (auto *DbgItem : DbgUsers)
- if (DbgItem->isAddressOfVariable() ||
- DbgItem->getExpression()->startsWithDeref())
- DbgItem->eraseFromParent();
- }
- };
- RemoveDbgDeclares(AllocaDbgUsers);
- RemoveDbgDeclares(AllocaDPUsers);
+ for (auto &DbgUsers : AllocaDPUsers) {
+ for (DbgVariableRecord *DbgItem : DbgUsers)
+ if (DbgItem->isAddressOfVariable() ||
+ DbgItem->getExpression()->startsWithDeref())
+ DbgItem->eraseFromParent();
+ }
// Loop over all of the PHI nodes and see if there are any that we can get
// rid of because they merge all of the same incoming values. This can
diff --git a/llvm/lib/Transforms/Utils/SSAUpdater.cpp b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
index 5db7fc956c497..561c898ec55d8 100644
--- a/llvm/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/llvm/lib/Transforms/Utils/SSAUpdater.cpp
@@ -200,11 +200,7 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
SmallVector<DbgValueInst *, 4> DbgValues;
SmallVector<DbgVariableRecord *, 4> DbgVariableRecords;
llvm::findDbgValues(DbgValues, I, &DbgVariableRecords);
- for (auto &DbgValue : DbgValues) {
- if (DbgValue->getParent() == I->getParent())
- continue;
- UpdateDebugValue(I, DbgValue);
- }
+ assert(DbgValues.empty());
for (auto &DVR : DbgVariableRecords) {
if (DVR->getParent() == I->getParent())
continue;
@@ -212,13 +208,6 @@ void SSAUpdater::UpdateDebugValues(Instruction *I) {
}
}
-void SSAUpdater::UpdateDebugValues(Instruction *I,
- SmallVectorImpl<DbgValueInst *> &DbgValues) {
- for (auto &DbgValue : DbgValues) {
- UpdateDebugValue(I, DbgValue);
- }
-}
-
void SSAUpdater::UpdateDebugValues(
Instruction *I, SmallVectorImpl<DbgVariableRecord *> &DbgVariableRecords) {
for (auto &DVR : DbgVariableRecords) {
@@ -226,15 +215,6 @@ void SSAUpdater::UpdateDebugValues(
}
}
-void SSAUpdater::UpdateDebugValue(Instruction *I, DbgValueInst *DbgValue) {
- BasicBlock *UserBB = DbgValue->getParent();
- if (HasValueForBlock(UserBB)) {
- Value *NewVal = GetValueAtEndOfBlock(UserBB);
- DbgValue->replaceVariableLocationOp(I, NewVal);
- } else
- DbgValue->setKillLocation();
-}
-
void SSAUpdater::UpdateDebugValue(Instruction *I, DbgVariableRecord *DVR) {
BasicBlock *UserBB = DVR->getParent();
if (HasValueForBlock(UserBB)) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index db0902c589d49..6e420632d83e5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -947,9 +947,8 @@ class LoopVectorizationCostModel {
/// user options, for the given register kind.
bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
- /// \return True if maximizing vector bandwidth is enabled by the target or
- /// user options, for the given vector factor.
- bool useMaxBandwidth(ElementCount VF);
+ /// \return True if register pressure should be calculated for the given VF.
+ bool shouldCalculateRegPressureForVF(ElementCount VF);
/// \return The size (in bits) of the smallest and widest types in the code
/// that needs to be vectorized. We ignore values that remain scalar such as
@@ -1736,6 +1735,9 @@ class LoopVectorizationCostModel {
/// Whether this loop should be optimized for size based on function attribute
/// or profile information.
bool OptForSize;
+
+ /// The highest VF possible for this loop, without using MaxBandwidth.
+ FixedScalableVFPair MaxPermissibleVFWithoutMaxBW;
};
} // end namespace llvm
@@ -3832,10 +3834,16 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
return FixedScalableVFPair::getNone();
}
-bool LoopVectorizationCostModel::useMaxBandwidth(ElementCount VF) {
- return useMaxBandwidth(VF.isScalable()
- ? TargetTransformInfo::RGK_ScalableVector
- : TargetTransformInfo::RGK_FixedWidthVector);
+bool LoopVectorizationCostModel::shouldCalculateRegPressureForVF(
+ ElementCount VF) {
+ if (!useMaxBandwidth(VF.isScalable()
+ ? TargetTransformInfo::RGK_ScalableVector
+ : TargetTransformInfo::RGK_FixedWidthVector))
+ return false;
+ // Only calculate register pressure for VFs enabled by MaxBandwidth.
+ return ElementCount::isKnownGT(
+ VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
+ : MaxPermissibleVFWithoutMaxBW.FixedVF);
}
bool LoopVectorizationCostModel::useMaxBandwidth(
@@ -3911,6 +3919,12 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
: TargetTransformInfo::RGK_FixedWidthVector;
ElementCount MaxVF = MaxVectorElementCount;
+
+ if (MaxVF.isScalable())
+ MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
+ else
+ MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
+
if (useMaxBandwidth(RegKind)) {
auto MaxVectorElementCountMaxBW = ElementCount::get(
llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
@@ -4264,9 +4278,11 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
if (VF.isScalar())
continue;
- /// Don't consider the VF if it exceeds the number of registers for the
- /// target.
- if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI))
+ /// If the register pressure needs to be considered for VF,
+ /// don't consider the VF as valid if it exceeds the number
+ /// of registers for the target.
+ if (CM.shouldCalculateRegPressureForVF(VF) &&
+ RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
continue;
InstructionCost C = CM.expectedCost(VF);
@@ -7048,7 +7064,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
InstructionCost Cost = cost(*P, VF);
VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
- if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) {
+ if (CM.shouldCalculateRegPressureForVF(VF) &&
+ RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
<< VF << " because it uses too many registers\n");
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 66657b98b094b..3499e650ae853 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -405,9 +405,12 @@ static unsigned getVFScaleFactor(VPRecipeBase *R) {
return 1;
}
-bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI) const {
- return any_of(MaxLocalUsers, [&TTI](auto &LU) {
- return LU.second > TTI.getNumberOfRegisters(LU.first);
+bool VPRegisterUsage::exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+ unsigned OverrideMaxNumRegs) const {
+ return any_of(MaxLocalUsers, [&TTI, &OverrideMaxNumRegs](auto &LU) {
+ return LU.second > (OverrideMaxNumRegs > 0
+ ? OverrideMaxNumRegs
+ : TTI.getNumberOfRegisters(LU.first));
});
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index 7bcf9dba8c311..cd86d27cf9122 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -85,8 +85,10 @@ struct VPRegisterUsage {
SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
/// Check if any of the tracked live intervals exceeds the number of
- /// available registers for the target.
- bool exceedsMaxNumRegs(const TargetTransformInfo &TTI) const;
+ /// available registers for the target. If non-zero, OverrideMaxNumRegs
+ /// is used in place of the target's number of registers.
+ bool exceedsMaxNumRegs(const TargetTransformInfo &TTI,
+ unsigned OverrideMaxNumRegs = 0) const;
};
/// Estimate the register usage for \p Plan and vectorization factors in \p VFs
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
index 4153f0be611a1..9698f1a6768fd 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldst-ext.ll
@@ -231,3 +231,27 @@ define <vscale x 8 x i64> @sload_8i8_8i64(ptr %a) {
%aext = sext <vscale x 8 x i8> %aval to <vscale x 8 x i64>
ret <vscale x 8 x i64> %aext
}
+
+; Ensure we don't try to promote a predicate load to a sign-extended load.
+define <vscale x 16 x i8> @sload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: sload_16i1_16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr p0, [x0]
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+ %load = load <vscale x 16 x i1>, ptr %addr
+ %zext = sext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %zext
+}
+
+; Ensure we don't try to promote a predicate load to a zero-extended load.
+define <vscale x 16 x i8> @zload_16i1_16i8(ptr %addr) {
+; CHECK-LABEL: zload_16i1_16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr p0, [x0]
+; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ret
+ %load = load <vscale x 16 x i1>, ptr %addr
+ %zext = zext <vscale x 16 x i1> %load to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %zext
+}
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
index 6742ae6c1d584..f6465de86fa4f 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-remat.ll
@@ -6,17 +6,17 @@
define amdgpu_kernel void @remat_constant_voids_spill(ptr addrspace(1) %p) #1 {
; GFX908-LABEL: remat_constant_voids_spill:
; GFX908: ; %bb.0:
-; GFX908-NEXT: v_accvgpr_write_b32 a1, 1
-; GFX908-NEXT: v_accvgpr_write_b32 a5, 6
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 7
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 8
-; GFX908-NEXT: v_accvgpr_write_b32 a0, 9
-; GFX908-NEXT: v_accvgpr_write_b32 a2, 2
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 3
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 4
+; GFX908-NEXT: v_accvgpr_write_b32 a0, 1
+; GFX908-NEXT: v_accvgpr_write_b32 a1, 2
+; GFX908-NEXT: v_accvgpr_write_b32 a2, 3
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 4
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ;;#ASMEND
-; GFX908-NEXT: v_accvgpr_write_b32 a1, 5
+; GFX908-NEXT: v_accvgpr_write_b32 a0, 5
+; GFX908-NEXT: v_accvgpr_write_b32 a1, 6
+; GFX908-NEXT: v_accvgpr_write_b32 a2, 7
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 8
+; GFX908-NEXT: v_accvgpr_write_b32 a4, 9
; GFX908-NEXT: ;;#ASMSTART
; GFX908-NEXT: ;;#ASMEND
; GFX908-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
new file mode 100644
index 0000000000000..69bdb1f5066f0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
@@ -0,0 +1,95 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX90A %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefixes=HAS-AGPR,GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx906 -passes=amdgpu-prepare-agpr-alloc -o - %s | FileCheck -check-prefix=NO-AGPR %s
+
+--- |
+ define void @func() {
+ ret void
+ }
+
+ ; Attribute is ignored for gfx90a
+ define void @no_agprs() "amdgpu-agpr-alloc"="0,0" {
+ ret void
+ }
+
+...
+---
+name: func
+tracksRegLiveness: true
+stack:
+ - { id: 0, size: 4 }
+body: |
+ ; HAS-AGPR-LABEL: name: func
+ ; HAS-AGPR: bb.0:
+ ; HAS-AGPR-NEXT: successors: %bb.1(0x80000000)
+ ; HAS-AGPR-NEXT: liveins: $vgpr0
+ ; HAS-AGPR-NEXT: {{ $}}
+ ; HAS-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ ; HAS-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+ ; HAS-AGPR-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; HAS-AGPR-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+ ; HAS-AGPR-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+ ; HAS-AGPR-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+ ; HAS-AGPR-NEXT: [[AV_MOV_2:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 6, implicit $exec
+ ; HAS-AGPR-NEXT: {{ $}}
+ ; HAS-AGPR-NEXT: bb.1:
+ ; HAS-AGPR-NEXT: [[AV_MOV_3:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 3, implicit $exec
+ ;
+ ; NO-AGPR-LABEL: name: func
+ ; NO-AGPR: bb.0:
+ ; NO-AGPR-NEXT: successors: %bb.1(0x80000000)
+ ; NO-AGPR-NEXT: liveins: $vgpr0
+ ; NO-AGPR-NEXT: {{ $}}
+ ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+ ; NO-AGPR-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; NO-AGPR-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+ ; NO-AGPR-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+ ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+ ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_2:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+ ; NO-AGPR-NEXT: {{ $}}
+ ; NO-AGPR-NEXT: bb.1:
+ ; NO-AGPR-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+ bb.0:
+ liveins: $vgpr0
+ %0:vgpr_32 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
+ %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %3:vgpr_32 = V_MOV_B32_e32 65, implicit $exec
+ %4:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
+ %5:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+ %6:agpr_32 = V_ACCVGPR_WRITE_B32_e64 6, implicit $exec
+
+ bb.1:
+ %7:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+
+...
+
+---
+name: no_agprs
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr0
+ ; GFX90A-LABEL: name: no_agprs
+ ; GFX90A: liveins: $vgpr0
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GFX90A-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+ ;
+ ; GFX908-LABEL: name: no_agprs
+ ; GFX908: liveins: $vgpr0
+ ; GFX908-NEXT: {{ $}}
+ ; GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:agpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+ ;
+ ; NO-AGPR-LABEL: name: no_agprs
+ ; NO-AGPR: liveins: $vgpr0
+ ; NO-AGPR-NEXT: {{ $}}
+ ; NO-AGPR-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; NO-AGPR-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+ %0:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
new file mode 100644
index 0000000000000..1a457c94778fd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bad-agpr-vgpr-regalloc-priority.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+
+---
+name: bad_ra
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: sgpr_64, preferred-register: '$sgpr4_sgpr5' }
+ - { id: 1, class: sgpr_128, preferred-register: '%2' }
+ - { id: 2, class: areg_128, preferred-register: '%1' }
+ - { id: 3, class: areg_128, preferred-register: '%4' }
+ - { id: 4, class: av_128, preferred-register: '%3' }
+ - { id: 5, class: areg_128, preferred-register: '%6' }
+ - { id: 6, class: vreg_128, preferred-register: '%5' }
+ - { id: 7, class: areg_128, preferred-register: '%4' }
+ - { id: 8, class: vgpr_32 }
+ - { id: 9, class: vgpr_32 }
+ - { id: 10, class: vgpr_32 }
+ - { id: 11, class: areg_128 }
+liveins:
+ - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
+frameInfo:
+ maxAlignment: 1
+ isCalleeSavedInfoValid: true
+machineFunctionInfo:
+ isEntryFunction: true
+ scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99'
+ stackPtrOffsetReg: '$sgpr32'
+ occupancy: 10
+ vgprForAGPRCopy: '$vgpr255'
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+
+ ; CHECK-LABEL: name: bad_ra
+ ; CHECK: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+ ; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 1065353216, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr5 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr6 = V_MOV_B32_e32 1073741824, implicit $exec
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = COPY renamable $agpr1
+ ; CHECK-NEXT: renamable $vgpr0 = COPY renamable $agpr0
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr4, $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr3 = COPY renamable $agpr1
+ ; CHECK-NEXT: renamable $vgpr2 = COPY killed renamable $agpr0
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
+ ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 killed $vgpr4, killed $vgpr6, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr5, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ early-clobber renamable $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64), align 4, addrspace 4)
+ renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM renamable $sgpr6_sgpr7, 0, 0 :: ("amdgpu-noclobber" load (s128), addrspace 1)
+ %8:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+ %9:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ %10:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+ %2:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
+ %3:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %2, 0, 0, 0, implicit $mode, implicit $exec
+ undef %4.sub1:av_128 = COPY %3.sub1
+ %4.sub0:av_128 = COPY %3.sub0
+ %11:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %3, 0, 0, 0, implicit $mode, implicit $exec
+ %4.sub3:av_128 = COPY %11.sub1
+ %4.sub2:av_128 = COPY %11.sub0
+ %7:areg_128 = COPY %4
+ %5:areg_128 = V_MFMA_F32_4X4X1F32_e64 %8, %10, %7, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vreg_128 = COPY %5
+ GLOBAL_STORE_DWORDX4_SADDR %9, %6, killed renamable $sgpr6_sgpr7, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index ae90cfb631e8d..7eb7d72e6cb97 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr17, 8, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_CSELECT_B64 -1, 0, implicit killed $scc
; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr18_sgpr19, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = DS_READ_B32_gfx9 renamable $vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc
@@ -56,8 +56,8 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr30 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $vgpr15 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr17 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr15 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr17 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: S_CBRANCH_VCCZ %bb.57, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.4.bb15:
@@ -112,14 +112,14 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: successors: %bb.7(0x80000000)
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr15, $vgpr17, $vgpr30, $vgpr31, $vgpr52, $vgpr53, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr19 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr18 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr21 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr20 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr23 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr22 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr25 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr24 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr19 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr18 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr21 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr22 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr25 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.7.Flow19:
; GFX90A-NEXT: successors: %bb.62(0x40000000), %bb.8(0x40000000)
@@ -671,7 +671,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr56_sgpr57
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr30, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.54, implicit $vcc
; GFX90A-NEXT: {{ $}}
@@ -759,7 +759,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $vgpr18, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47:0x000000000000000F, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x0000000000000003, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr53 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr10 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr14_vgpr15 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $vgpr10 = COPY renamable $sgpr21, implicit $exec
; GFX90A-NEXT: renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr10, 0, 0, implicit $exec :: (load (s64) from %ir.7, addrspace 3)
@@ -801,12 +801,12 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr42_vgpr43 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr40_vgpr41 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr46_vgpr47 = IMPLICIT_DEF
- ; GFX90A-NEXT: renamable $vgpr14 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr52 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr16 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr53 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr13 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr12 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr14 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr52 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr53 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr13 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr12 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 0
; GFX90A-NEXT: S_BRANCH %bb.7
; GFX90A-NEXT: {{ $}}
@@ -814,7 +814,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: successors: %bb.3(0x80000000)
; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr33, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr46_sgpr47:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
; GFX90A-NEXT: renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
@@ -913,7 +913,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr25, implicit $exec
; GFX90A-NEXT: renamable $vgpr3 = V_OR_B32_e32 killed $vgpr11, killed $vgpr19, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr3, killed $vgpr2, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U32_sdwa 0, killed $vgpr53, 0, $vgpr3, 0, 0, 6, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr10 = V_OR_B32_e32 killed $vgpr52, killed $vgpr13, implicit $exec
@@ -955,7 +955,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr10 = COPY renamable $sgpr27, implicit $exec
; GFX90A-NEXT: renamable $vgpr2, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr2, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr3, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr10, killed $vgpr3, killed $vcc, 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr27 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr27 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr49 = COPY renamable $vgpr27, implicit $exec
; GFX90A-NEXT: renamable $vgpr35 = COPY renamable $vgpr27, implicit $exec
; GFX90A-NEXT: renamable $vgpr39 = COPY renamable $vgpr27, implicit $exec
@@ -989,7 +989,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr10 = V_OR_B32_e32 $vgpr50, killed $vgpr16, implicit $exec
; GFX90A-NEXT: renamable $vgpr54 = V_OR_B32_e32 killed $vgpr10, killed $vgpr14, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr55 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr55 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr55, renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_MOV_B64 0
; GFX90A-NEXT: S_BRANCH %bb.69
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 50fa7ac2a19aa..4f81d3599d1de 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -10,9 +10,9 @@
; GCN-O0: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(atomic-expand,verify,gc-lowering,lower-constant-intrinsics,unreachableblockelim,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(lower-switch,lower-invoke,unreachableblockelim,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa,require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,localstackalloc))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,phi-node-elimination,two-address-instruction,regallocfast,si-fix-vgpr-copies,remove-redundant-debug-values,fixup-statepoint-caller-saved,prolog-epilog,post-ra-pseudos,si-post-ra-bundler,fentry-insert,xray-instrumentation,patchable-function,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
-; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O2: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,early-cse<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,early-cse<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
-; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
+; GCN-O3: require<MachineModuleAnalysis>,require<profile-summary>,require<collector-metadata>,pre-isel-intrinsic-lowering,function(expand-large-div-rem,expand-fp),amdgpu-remove-incompatible-functions,amdgpu-printf-runtime-binding,amdgpu-lower-ctor-dtor,function(amdgpu-image-intrinsic-opt),expand-variadics,amdgpu-always-inline,always-inline,amdgpu-export-kernel-runtime-handles,amdgpu-sw-lower-lds,amdgpu-lower-module-lds,function(amdgpu-atomic-optimizer,atomic-expand,amdgpu-promote-alloca,separate-const-offset-from-gep<>,slsr,gvn<>,nary-reassociate,early-cse<>,amdgpu-codegenprepare,verify,loop-mssa(loop-reduce),mergeicmps,expand-memcmp,gc-lowering,lower-constant-intrinsics,unreachableblockelim,consthoist,replace-with-veclib,partially-inline-libcalls,ee-instrument<post-inline>,scalarize-masked-mem-intrin,expand-reductions,gvn<>),amdgpu-preload-kernel-arguments,function(amdgpu-lower-kernel-arguments),amdgpu-lower-buffer-fat-pointers,cgscc(function(codegenprepare,load-store-vectorizer,lower-switch,lower-invoke,unreachableblockelim,flatten-cfg,sink,amdgpu-late-codegenprepare,amdgpu-unify-divergent-exit-nodes,fix-irreducible,unify-loop-exits,StructurizeCFGPass,amdgpu-annotate-uniform,si-annotate-control-flow,amdgpu-rewrite-undef-for-phi,lcssa)),amdgpu-perf-hint,cgscc(function(require<uniformity>,callbr-prepare,safe-stack,stack-protector,verify)),cgscc(function(machine-function(amdgpu-isel,si-fix-sgpr-copies,si-i1-copies,finalize-isel,early-tailduplication,opt-phis,stack-coloring,localstackalloc,dead-mi-elimination,early-machinelicm,machine-cse,machine-sink,peephole-opt,dead-mi-elimination,si-fold-operands,gcn-dpp-combine,si-load-store-opt,si-peephole-sdwa,early-machinelicm,machine-cse,si-fold-operands,dead-mi-elimination,si-shrink-instructions))),require<reg-usage>,cgscc(function(machine-function(reg-usage-propagation,amdgpu-prepare-agpr-alloc,detect-dead-lanes,dead-mi-elimination,init-undef,process-imp-defs,unreachable-mbb-elimination,require<live-vars>,si-opt-vgpr-liverange,require<machine-loops>,phi-node-elimination,si-lower-control-flow,two-address-instruction,register-coalescer,rename-independent-subregs,amdgpu-rewrite-partial-reg-uses,machine-scheduler,amdgpu-pre-ra-optimizations,si-wqm,si-optimize-exec-masking-pre-ra,si-form-memory-clauses,amdgpu-pre-ra-long-branch-reg,greedy<sgpr>,virt-reg-rewriter<no-clear-vregs>,stack-slot-coloring,si-lower-sgpr-spills,si-pre-allocate-wwm-regs,greedy<wwm>,si-lower-wwm-copies,virt-reg-rewriter<no-clear-vregs>,amdgpu-reserve-wwm-regs,greedy<vgpr>,amdgpu-nsa-reassign,virt-reg-rewriter,amdgpu-mark-last-scratch-load,machine-cp,machinelicm,si-fix-vgpr-copies,si-optimize-exec-masking,remove-redundant-debug-values,fixup-statepoint-caller-saved,postra-machine-sink,shrink-wrap,prolog-epilog,branch-folder,tailduplication,machine-latecleanup,machine-cp,post-ra-pseudos,si-shrink-instructions,si-post-ra-bundler,postmisched,block-placement,fentry-insert,xray-instrumentation,patchable-function,gcn-create-vopd,si-memory-legalizer,si-insert-waitcnts,si-late-branch-lowering,si-pre-emit-peephole,post-RA-hazard-rec,amdgpu-wait-sgpr-hazards,amdgpu-insert-delay-alu,branch-relaxation,reg-usage-collector,remove-loads-into-fake-uses,live-debug-values,machine-sanmd,stack-frame-layout,verify),free-machine-function))
define void @empty() {
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index af3241e95e91d..2a5c65278f7dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -329,6 +329,7 @@
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: SI Shrink Instructions
; GCN-O1-NEXT: Register Usage Information Propagation
+; GCN-O1-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-NEXT: Detect Dead Lanes
; GCN-O1-NEXT: Remove dead machine instructions
; GCN-O1-NEXT: Init Undef Pass
@@ -640,6 +641,7 @@
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: SI Shrink Instructions
; GCN-O1-OPTS-NEXT: Register Usage Information Propagation
+; GCN-O1-OPTS-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O1-OPTS-NEXT: Detect Dead Lanes
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
; GCN-O1-OPTS-NEXT: Init Undef Pass
@@ -956,6 +958,7 @@
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: SI Shrink Instructions
; GCN-O2-NEXT: Register Usage Information Propagation
+; GCN-O2-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O2-NEXT: Detect Dead Lanes
; GCN-O2-NEXT: Remove dead machine instructions
; GCN-O2-NEXT: Init Undef Pass
@@ -1286,6 +1289,7 @@
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: SI Shrink Instructions
; GCN-O3-NEXT: Register Usage Information Propagation
+; GCN-O3-NEXT: AMDGPU Prepare AGPR Alloc
; GCN-O3-NEXT: Detect Dead Lanes
; GCN-O3-NEXT: Remove dead machine instructions
; GCN-O3-NEXT: Init Undef Pass
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
index 67ae05eb6f0b8..561eaca3b77df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll
@@ -4365,8 +4365,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
; NOLIT-SRCC: ; %bb.0: ; %bb
; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
-; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0
+; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 0
; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 0
@@ -4465,8 +4465,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) #
; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm:
; LIT-SRCC: ; %bb.0: ; %bb
; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0
-; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0
+; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 0
; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
index 2faf375a97a86..465414c5471ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
@@ -1,5 +1,7 @@
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
declare i32 @llvm.amdgcn.prng.b32(i32) #0
@@ -29,4 +31,4 @@ define amdgpu_kernel void @prng_b32_constant_100(ptr addrspace(1) %out) #1 {
attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
\ No newline at end of file
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index 3844d6054e130..cf244f0b1f884 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -6,16 +6,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX942-LABEL: matmul_kernel:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_accvgpr_write_b32 a2, v0
+; GFX942-NEXT: v_accvgpr_write_b32 a2, 0
; GFX942-NEXT: s_mov_b32 s2, 0
; GFX942-NEXT: v_accvgpr_write_b32 a1, 0
+; GFX942-NEXT: s_mov_b32 s3, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_lg_u32 s0, 0
; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX942-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
; GFX942-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX942-NEXT: s_mov_b32 s3, 0
; GFX942-NEXT: s_branch .LBB0_2
; GFX942-NEXT: .LBB0_1: ; %bb2
; GFX942-NEXT: ; in Loop: Header=BB0_2 Depth=1
@@ -43,16 +42,15 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX908-LABEL: matmul_kernel:
; GFX908: ; %bb.0: ; %entry
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX908-NEXT: v_mov_b32_e32 v0, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
; GFX908-NEXT: s_mov_b32 s2, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT: s_mov_b32 s3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_cmp_lg_u32 s0, 0
; GFX908-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX908-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
; GFX908-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX908-NEXT: s_mov_b32 s3, 0
; GFX908-NEXT: s_branch .LBB0_2
; GFX908-NEXT: .LBB0_1: ; %bb2
; GFX908-NEXT: ; in Loop: Header=BB0_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
index ee5481617cf59..01506d0af1913 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.mir
@@ -80,16 +80,16 @@ body: |
; COALESCE-NEXT: S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
; COALESCE-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
; COALESCE-NEXT: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
- ; COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+ ; COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; COALESCE-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
; COALESCE-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
- ; COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+ ; COALESCE-NEXT: undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; COALESCE-NEXT: {{ $}}
; COALESCE-NEXT: bb.1:
; COALESCE-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; COALESCE-NEXT: {{ $}}
- ; COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+ ; COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
; COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
; COALESCE-NEXT: $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
; COALESCE-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -103,10 +103,10 @@ body: |
; COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
; COALESCE-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
- ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
- ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
- ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
- ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+ ; COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+ ; COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+ ; COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
; COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
; COALESCE-NEXT: {{ $}}
; COALESCE-NEXT: bb.3:
@@ -134,16 +134,16 @@ body: |
; GFX908-COALESCE-NEXT: S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
; GFX908-COALESCE-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_64 = S_MOV_B32 0
; GFX908-COALESCE-NEXT: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit killed $scc
- ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+ ; GFX908-COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX908-COALESCE-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_CSELECT_B64_]], implicit $exec
; GFX908-COALESCE-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
- ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+ ; GFX908-COALESCE-NEXT: undef [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX908-COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX908-COALESCE-NEXT: {{ $}}
; GFX908-COALESCE-NEXT: bb.1:
; GFX908-COALESCE-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; GFX908-COALESCE-NEXT: {{ $}}
- ; GFX908-COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+ ; GFX908-COALESCE-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
; GFX908-COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 -1
; GFX908-COALESCE-NEXT: $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
; GFX908-COALESCE-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
@@ -157,10 +157,10 @@ body: |
; GFX908-COALESCE-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_ASHR_I32_]], [[S_OR_B32_]], implicit-def dead $scc
; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_64 = COPY [[S_MOV_B32_]].sub0
; GFX908-COALESCE-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]]
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub1
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = COPY [[COPY1]]
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub1
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY2]], [[COPY2]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
; GFX908-COALESCE-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 0
; GFX908-COALESCE-NEXT: {{ $}}
; GFX908-COALESCE-NEXT: bb.3:
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
index 49c0aaf9fb390..a9207de317ea1 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-read.mir
@@ -67,7 +67,7 @@ body: |
; COALESCE-NEXT: bb.1:
; COALESCE-NEXT: successors: %bb.3(0x80000000)
; COALESCE-NEXT: {{ $}}
- ; COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+ ; COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; COALESCE-NEXT: S_BRANCH %bb.3
; COALESCE-NEXT: {{ $}}
; COALESCE-NEXT: bb.2:
@@ -78,13 +78,13 @@ body: |
; COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], 0, 0, 0, 0, implicit $mode, implicit $exec
; COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
- ; COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
; COALESCE-NEXT: {{ $}}
; COALESCE-NEXT: bb.3:
- ; COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+ ; COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
; COALESCE-NEXT: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
; COALESCE-NEXT: undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
- ; COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
@@ -105,28 +105,28 @@ body: |
; GFX908-COALESCE-NEXT: bb.1:
; GFX908-COALESCE-NEXT: successors: %bb.3(0x80000000)
; GFX908-COALESCE-NEXT: {{ $}}
- ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
+ ; GFX908-COALESCE-NEXT: undef [[AV_MOV_:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX908-COALESCE-NEXT: S_BRANCH %bb.3
; GFX908-COALESCE-NEXT: {{ $}}
; GFX908-COALESCE-NEXT: bb.2:
; GFX908-COALESCE-NEXT: successors: %bb.3(0x80000000)
; GFX908-COALESCE-NEXT: {{ $}}
- ; GFX908-COALESCE-NEXT: undef [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub0:areg_128_align2 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[V_ACCVGPR_WRITE_B32_e64_1]].sub0
+ ; GFX908-COALESCE-NEXT: undef [[AV_MOV_1:%[0-9]+]].sub0:areg_128_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub1:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub2:areg_128_align2 = COPY [[AV_MOV_1]].sub0
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_1:%[0-9]+]].sub3:areg_128_align2 = COPY [[AV_MOV_1]].sub0
; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; GFX908-COALESCE-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0_sub1
- ; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_ACCVGPR_WRITE_B32_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[AV_MOV_1]], 0, 0, 0, implicit $mode, implicit $exec
; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_1:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; GFX908-COALESCE-NEXT: [[V_MFMA_F32_16X16X16F16_e64_2:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
- ; GFX908-COALESCE-NEXT: [[V_ACCVGPR_WRITE_B32_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GFX908-COALESCE-NEXT: [[AV_MOV_:%[0-9]+]]:areg_128_align2 = V_MFMA_F32_16X16X16F16_e64 [[COPY1]], [[COPY1]], [[V_MFMA_F32_16X16X16F16_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
; GFX908-COALESCE-NEXT: {{ $}}
; GFX908-COALESCE-NEXT: bb.3:
- ; GFX908-COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[V_ACCVGPR_WRITE_B32_e64_]].sub0
+ ; GFX908-COALESCE-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
; GFX908-COALESCE-NEXT: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[COPY2]], implicit $mode, implicit $exec
; GFX908-COALESCE-NEXT: undef [[V_PACK_B32_F16_e64_:%[0-9]+]].sub0:vreg_64_align2 = nofpexcept V_PACK_B32_F16_e64 0, [[V_CVT_F16_F32_e32_]], 0, 0, 0, 0, implicit $mode, implicit $exec
- ; GFX908-COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX908-COALESCE-NEXT: [[V_PACK_B32_F16_e64_:%[0-9]+]].sub1:vreg_64_align2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
; GFX908-COALESCE-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
diff --git a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
index 663fd98b46bf7..ce96766116089 100644
--- a/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/partial-regcopy-and-spill-missed-at-regalloc.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; REGALLOC-GFX908-NEXT: [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
- ; REGALLOC-GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
- ; REGALLOC-GFX908-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
- ; REGALLOC-GFX908-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; REGALLOC-GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; REGALLOC-GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+ ; REGALLOC-GFX908-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX908-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
; REGALLOC-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
@@ -42,8 +42,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; PEI-GFX908-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; PEI-GFX908-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
- ; PEI-GFX908-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; PEI-GFX908-NEXT: renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+ ; PEI-GFX908-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; PEI-GFX908-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
; PEI-GFX908-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
; PEI-GFX908-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
; PEI-GFX908-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
@@ -62,9 +62,9 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %15:vreg_64_align2, %7, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; REGALLOC-GFX90A-NEXT: [[COPY:%[0-9]+]]:areg_128_align2 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
- ; REGALLOC-GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
- ; REGALLOC-GFX90A-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
- ; REGALLOC-GFX90A-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; REGALLOC-GFX90A-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; REGALLOC-GFX90A-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+ ; REGALLOC-GFX90A-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128_align2 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX2 undef %17:vreg_64_align2, %8, 0, 0, implicit $exec :: (volatile store (s64) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef %19:vreg_64_align2, [[V_MFMA_I32_4X4X4I8_e64_]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; REGALLOC-GFX90A-NEXT: S_ENDPGM 0
@@ -85,8 +85,8 @@ define amdgpu_kernel void @partial_copy(<4 x i32> %arg) #0 {
; PEI-GFX90A-NEXT: GLOBAL_STORE_DWORDX4 undef renamable $vgpr0_vgpr1, killed renamable $agpr0_agpr1_agpr2_agpr3, 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
; PEI-GFX90A-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec
- ; PEI-GFX90A-NEXT: renamable $vgpr0 = V_MOV_B32_e32 1, implicit $exec
- ; PEI-GFX90A-NEXT: renamable $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+ ; PEI-GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; PEI-GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
; PEI-GFX90A-NEXT: renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_I32_4X4X4I8_e64 killed $vgpr0, killed $vgpr1, killed $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec
; PEI-GFX90A-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load (s32) from %stack.0, addrspace 5)
; PEI-GFX90A-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr4, implicit $exec, implicit $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 1ec94162951a6..d48bfe0bb7f21 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -145,49 +145,29 @@ entry:
; Test skipping the lower-32-bit addition if it is unnecessary.
define ptr @huge_offset_low_32_unused(ptr %p) {
-; GFX942_PTRADD-LABEL: huge_offset_low_32_unused:
-; GFX942_PTRADD: ; %bb.0:
-; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT: s_mov_b32 s0, 0
-; GFX942_PTRADD-NEXT: s_mov_b32 s1, 1
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
-; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: huge_offset_low_32_unused:
-; GFX942_LEGACY: ; %bb.0:
-; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: huge_offset_low_32_unused:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_add_u32_e32 v1, 1, v1
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds i8, ptr %p, i64 u0x100000000
ret ptr %gep
}
; Reassociate address computation if it leads to more scalar operations.
define amdgpu_kernel void @reassoc_scalar_r(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_r:
-; GFX942_PTRADD: ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT: v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[6:7]
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT: s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_r:
-; GFX942_LEGACY: ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT: v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT: s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT: s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT: s_endpgm
+; GFX942-LABEL: reassoc_scalar_r:
+; GFX942: ; %bb.0: ; %entry
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_add_u32 s2, s2, s6
+; GFX942-NEXT: s_addc_u32 s3, s3, s7
+; GFX942-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
entry:
%voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
%voffset = zext i32 %voffset32 to i64
@@ -198,30 +178,18 @@ entry:
}
define amdgpu_kernel void @reassoc_scalar_l(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
-; GFX942_PTRADD-LABEL: reassoc_scalar_l:
-; GFX942_PTRADD: ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_PTRADD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_PTRADD-NEXT: v_mov_b32_e32 v1, 0
-; GFX942_PTRADD-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], s[6:7], 0, v[0:1]
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
-; GFX942_PTRADD-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_PTRADD-NEXT: s_endpgm
-;
-; GFX942_LEGACY-LABEL: reassoc_scalar_l:
-; GFX942_LEGACY: ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX942_LEGACY-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
-; GFX942_LEGACY-NEXT: v_mov_b32_e32 v1, 0
-; GFX942_LEGACY-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT: s_add_u32 s2, s2, s6
-; GFX942_LEGACY-NEXT: s_addc_u32 s3, s3, s7
-; GFX942_LEGACY-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
-; GFX942_LEGACY-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
-; GFX942_LEGACY-NEXT: s_endpgm
+; GFX942-LABEL: reassoc_scalar_l:
+; GFX942: ; %bb.0: ; %entry
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_add_u32 s2, s2, s6
+; GFX942-NEXT: s_addc_u32 s3, s3, s7
+; GFX942-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942-NEXT: global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942-NEXT: s_endpgm
entry:
%voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
%voffset = zext i32 %voffset32 to i64
@@ -233,24 +201,14 @@ entry:
; Tests the target-specific (ptradd x, shl(0 - y, k)) -> sub(x, shl(y, k)) fold
define ptr addrspace(1) @shl_neg_offset(ptr addrspace(1) %p, i64 %noffset, i64 %shift) {
-; GFX942_PTRADD-LABEL: shl_neg_offset:
-; GFX942_PTRADD: ; %bb.0:
-; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT: v_sub_co_u32_e32 v2, vcc, 0, v2
-; GFX942_PTRADD-NEXT: s_nop 1
-; GFX942_PTRADD-NEXT: v_subb_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX942_PTRADD-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: shl_neg_offset:
-; GFX942_LEGACY: ; %bb.0:
-; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
-; GFX942_LEGACY-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
-; GFX942_LEGACY-NEXT: s_nop 1
-; GFX942_LEGACY-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31]
+; GFX942-LABEL: shl_neg_offset:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX942-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-NEXT: s_setpc_b64 s[30:31]
%offset = sub i64 0, %noffset
%x = shl i64 %offset, %shift
%gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 %x
@@ -268,10 +226,9 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
; GFX942_PTRADD: ; %bb.0:
; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942_PTRADD-NEXT: s_getpc_b64 s[0:1]
-; GFX942_PTRADD-NEXT: s_add_u32 s0, s0, v0 at rel32@lo+4
-; GFX942_PTRADD-NEXT: s_addc_u32 s1, s1, v0 at rel32@hi+12
+; GFX942_PTRADD-NEXT: s_add_u32 s0, s0, v0 at rel32@lo+14
+; GFX942_PTRADD-NEXT: s_addc_u32 s1, s1, v0 at rel32@hi+22
; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, 10
; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31]
;
; GFX942_LEGACY-LABEL: complextype_global_gep:
@@ -291,30 +248,63 @@ define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
; Tests the tryFoldToMad64_32 PTRADD combine.
define amdgpu_kernel void @fold_mad64(ptr addrspace(1) %p) {
-; GFX942_PTRADD-LABEL: fold_mad64:
-; GFX942_PTRADD: ; %bb.0:
-; GFX942_PTRADD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_PTRADD-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_PTRADD-NEXT: v_mul_hi_u32_u24_e32 v1, 12, v0
-; GFX942_PTRADD-NEXT: v_mul_u32_u24_e32 v0, 12, v0
-; GFX942_PTRADD-NEXT: v_mov_b32_e32 v2, 1.0
-; GFX942_PTRADD-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
-; GFX942_PTRADD-NEXT: global_store_dword v[0:1], v2, off
-; GFX942_PTRADD-NEXT: s_endpgm
-;
-; GFX942_LEGACY-LABEL: fold_mad64:
-; GFX942_LEGACY: ; %bb.0:
-; GFX942_LEGACY-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_LEGACY-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942_LEGACY-NEXT: v_mov_b32_e32 v2, 1.0
-; GFX942_LEGACY-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
-; GFX942_LEGACY-NEXT: global_store_dword v[0:1], v2, off
-; GFX942_LEGACY-NEXT: s_endpgm
+; GFX942-LABEL: fold_mad64:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT: v_mov_b32_e32 v2, 1.0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
+; GFX942-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-NEXT: s_endpgm
%voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
%voffset = zext i32 %voffset32 to i64
%p1 = getelementptr inbounds %S, ptr addrspace(1) %p, i64 %voffset, i32 0
store float 1.0, ptr addrspace(1) %p1
ret void
}
+
+; Use non-zero shift amounts in v_lshl_add_u64.
+define ptr @select_v_lshl_add_u64(ptr %base, i64 %voffset) {
+; GFX942_PTRADD-LABEL: select_v_lshl_add_u64:
+; GFX942_PTRADD: ; %bb.0:
+; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT: v_lshlrev_b64 v[2:3], 3, v[2:3]
+; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: select_v_lshl_add_u64:
+; GFX942_LEGACY: ; %bb.0:
+; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 3, v[0:1]
+; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr inbounds i64, ptr %base, i64 %voffset
+ ret ptr %gep
+}
+
+; Fold mul and add into v_mad, even if amdgpu-codegenprepare-mul24 turned the
+; mul into a mul24.
+define ptr @fold_mul24_into_mad(ptr %base, i64 %a, i64 %b) {
+; GFX942_PTRADD-LABEL: fold_mul24_into_mad:
+; GFX942_PTRADD: ; %bb.0:
+; GFX942_PTRADD-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT: v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_PTRADD-NEXT: v_and_b32_e32 v4, 0xfffff, v4
+; GFX942_PTRADD-NEXT: v_mul_hi_u32_u24_e32 v3, v2, v4
+; GFX942_PTRADD-NEXT: v_mul_u32_u24_e32 v2, v2, v4
+; GFX942_PTRADD-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942_PTRADD-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: fold_mul24_into_mad:
+; GFX942_LEGACY: ; %bb.0:
+; GFX942_LEGACY-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT: v_and_b32_e32 v2, 0xfffff, v2
+; GFX942_LEGACY-NEXT: v_and_b32_e32 v3, 0xfffff, v4
+; GFX942_LEGACY-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v3, v[0:1]
+; GFX942_LEGACY-NEXT: s_setpc_b64 s[30:31]
+ %a_masked = and i64 %a, u0xfffff
+ %b_masked = and i64 %b, u0xfffff
+ %mul = mul i64 %a_masked, %b_masked
+ %gep = getelementptr inbounds i8, ptr %base, i64 %mul
+ ret ptr %gep
+}
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
index bd255e88b9512..648b59f69ea79 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vector-superclass.ll
@@ -9,9 +9,9 @@ define amdgpu_kernel void @test_spill_av_class(<4 x i32> %arg) #0 {
; GCN-NEXT: {{ $}}
; GCN-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; GCN-NEXT: [[COPY:%[0-9]+]]:areg_128 = COPY killed renamable $sgpr0_sgpr1_sgpr2_sgpr3
- ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
- ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
- ; GCN-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GCN-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1, implicit $exec
+ ; GCN-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 2, implicit $exec
+ ; GCN-NEXT: [[V_MFMA_I32_4X4X4I8_e64_:%[0-9]+]]:areg_128 = V_MFMA_I32_4X4X4I8_e64 [[AV_MOV_]], [[AV_MOV_1]], [[COPY]], 0, 0, 0, implicit $mode, implicit $exec
; GCN-NEXT: INLINEASM &"; def $0", 1 /* sideeffect attdialect */, 2228234 /* regdef:VGPR_32 */, def undef %14.sub0
; GCN-NEXT: [[COPY1:%[0-9]+]]:vreg_128 = COPY [[V_MFMA_I32_4X4X4I8_e64_]]
; GCN-NEXT: GLOBAL_STORE_DWORDX4 undef %24:vreg_64, [[COPY1]], 0, 0, implicit $exec :: (volatile store (s128) into `ptr addrspace(1) poison`, addrspace 1)
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
index d07e2914c753a..f7653af1fa9ba 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-common.ll
@@ -122,23 +122,23 @@ define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
define i64 @caller_large_scalars() nounwind {
; CHECK-LABEL: caller_large_scalars:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -80
-; CHECK-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $zero, $sp, 24
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $zero, $sp, 40
; CHECK-NEXT: vrepli.b $vr0, 0
-; CHECK-NEXT: vst $vr0, $sp, 8
+; CHECK-NEXT: vst $vr0, $sp, 24
; CHECK-NEXT: ori $a0, $zero, 2
-; CHECK-NEXT: st.d $a0, $sp, 0
-; CHECK-NEXT: st.d $zero, $sp, 56
-; CHECK-NEXT: vst $vr0, $sp, 40
+; CHECK-NEXT: st.d $a0, $sp, 16
+; CHECK-NEXT: st.d $zero, $sp, 72
+; CHECK-NEXT: vst $vr0, $sp, 56
; CHECK-NEXT: ori $a2, $zero, 1
-; CHECK-NEXT: addi.d $a0, $sp, 32
-; CHECK-NEXT: addi.d $a1, $sp, 0
-; CHECK-NEXT: st.d $a2, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 48
+; CHECK-NEXT: addi.d $a1, $sp, 16
+; CHECK-NEXT: st.d $a2, $sp, 48
; CHECK-NEXT: pcaddu18i $ra, %call36(callee_large_scalars)
; CHECK-NEXT: jirl $ra, $ra, 0
-; CHECK-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 80
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%1 = call i64 @callee_large_scalars(i256 1, i256 2)
ret i64 %1
@@ -177,20 +177,20 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d,
define i64 @caller_large_scalars_exhausted_regs() nounwind {
; CHECK-LABEL: caller_large_scalars_exhausted_regs:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -96
-; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $a0, $sp, 16
+; CHECK-NEXT: addi.d $sp, $sp, -112
+; CHECK-NEXT: st.d $ra, $sp, 104 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: st.d $a0, $sp, 8
; CHECK-NEXT: ori $a0, $zero, 9
; CHECK-NEXT: st.d $a0, $sp, 0
-; CHECK-NEXT: st.d $zero, $sp, 40
+; CHECK-NEXT: st.d $zero, $sp, 56
; CHECK-NEXT: vrepli.b $vr0, 0
-; CHECK-NEXT: vst $vr0, $sp, 24
+; CHECK-NEXT: vst $vr0, $sp, 40
; CHECK-NEXT: ori $a0, $zero, 10
-; CHECK-NEXT: st.d $a0, $sp, 16
-; CHECK-NEXT: st.d $zero, $sp, 72
+; CHECK-NEXT: st.d $a0, $sp, 32
+; CHECK-NEXT: st.d $zero, $sp, 88
; CHECK-NEXT: ori $a0, $zero, 8
-; CHECK-NEXT: st.d $a0, $sp, 48
+; CHECK-NEXT: st.d $a0, $sp, 64
; CHECK-NEXT: ori $a0, $zero, 1
; CHECK-NEXT: ori $a1, $zero, 2
; CHECK-NEXT: ori $a2, $zero, 3
@@ -198,12 +198,12 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
; CHECK-NEXT: ori $a4, $zero, 5
; CHECK-NEXT: ori $a5, $zero, 6
; CHECK-NEXT: ori $a6, $zero, 7
-; CHECK-NEXT: addi.d $a7, $sp, 48
-; CHECK-NEXT: vst $vr0, $sp, 56
+; CHECK-NEXT: addi.d $a7, $sp, 64
+; CHECK-NEXT: vst $vr0, $sp, 72
; CHECK-NEXT: pcaddu18i $ra, %call36(callee_large_scalars_exhausted_regs)
; CHECK-NEXT: jirl $ra, $ra, 0
-; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 96
+; CHECK-NEXT: ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 112
; CHECK-NEXT: ret
%1 = call i64 @callee_large_scalars_exhausted_regs(
i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i256 8, i64 9,
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
index c88b67f13d1e7..da8c3e93f6842 100644
--- a/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-half.ll
@@ -1252,8 +1252,8 @@ define i32 @caller_half_on_stack() nounwind {
;
; LA64F-LP64S-LABEL: caller_half_on_stack:
; LA64F-LP64S: # %bb.0:
-; LA64F-LP64S-NEXT: addi.d $sp, $sp, -80
-; LA64F-LP64S-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64F-LP64S-NEXT: addi.d $sp, $sp, -96
+; LA64F-LP64S-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
; LA64F-LP64S-NEXT: lu12i.w $a0, -12
; LA64F-LP64S-NEXT: ori $a1, $a0, 3200
; LA64F-LP64S-NEXT: lu32i.d $a1, 0
@@ -1292,8 +1292,8 @@ define i32 @caller_half_on_stack() nounwind {
; LA64F-LP64S-NEXT: st.w $t0, $sp, 0
; LA64F-LP64S-NEXT: pcaddu18i $ra, %call36(callee_half_on_stack)
; LA64F-LP64S-NEXT: jirl $ra, $ra, 0
-; LA64F-LP64S-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64F-LP64S-NEXT: addi.d $sp, $sp, 80
+; LA64F-LP64S-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64F-LP64S-NEXT: addi.d $sp, $sp, 96
; LA64F-LP64S-NEXT: ret
;
; LA64F-LP64D-LABEL: caller_half_on_stack:
@@ -1336,8 +1336,8 @@ define i32 @caller_half_on_stack() nounwind {
;
; LA64D-LP64S-LABEL: caller_half_on_stack:
; LA64D-LP64S: # %bb.0:
-; LA64D-LP64S-NEXT: addi.d $sp, $sp, -80
-; LA64D-LP64S-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64D-LP64S-NEXT: addi.d $sp, $sp, -96
+; LA64D-LP64S-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
; LA64D-LP64S-NEXT: lu12i.w $a0, -12
; LA64D-LP64S-NEXT: ori $a1, $a0, 3200
; LA64D-LP64S-NEXT: lu32i.d $a1, 0
@@ -1376,8 +1376,8 @@ define i32 @caller_half_on_stack() nounwind {
; LA64D-LP64S-NEXT: st.w $t0, $sp, 0
; LA64D-LP64S-NEXT: pcaddu18i $ra, %call36(callee_half_on_stack)
; LA64D-LP64S-NEXT: jirl $ra, $ra, 0
-; LA64D-LP64S-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64D-LP64S-NEXT: addi.d $sp, $sp, 80
+; LA64D-LP64S-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64D-LP64S-NEXT: addi.d $sp, $sp, 96
; LA64D-LP64S-NEXT: ret
;
; LA64D-LP64D-LABEL: caller_half_on_stack:
diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
index 52d8dd05aaa4c..1a9de3b0ef3d1 100644
--- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
+++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
@@ -14,41 +14,41 @@
define dso_local noundef signext i32 @main() nounwind {
; CHECK-LABEL: main:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: addi.d $sp, $sp, -272
-; CHECK-NEXT: st.d $ra, $sp, 264 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $sp, $sp, -288
+; CHECK-NEXT: st.d $ra, $sp, 280 # 8-byte Folded Spill
; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI0_0)
; CHECK-NEXT: xvld $xr0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT: xvst $xr0, $sp, 96 # 32-byte Folded Spill
+; CHECK-NEXT: xvst $xr0, $sp, 112 # 32-byte Folded Spill
; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI0_1)
; CHECK-NEXT: xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
-; CHECK-NEXT: xvst $xr1, $sp, 64 # 32-byte Folded Spill
+; CHECK-NEXT: xvst $xr1, $sp, 80 # 32-byte Folded Spill
; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI0_2)
; CHECK-NEXT: xvld $xr2, $a0, %pc_lo12(.LCPI0_2)
-; CHECK-NEXT: xvst $xr2, $sp, 32 # 32-byte Folded Spill
+; CHECK-NEXT: xvst $xr2, $sp, 48 # 32-byte Folded Spill
; CHECK-NEXT: pcalau12i $a0, %pc_hi20(.LCPI0_3)
; CHECK-NEXT: xvld $xr3, $a0, %pc_lo12(.LCPI0_3)
-; CHECK-NEXT: xvst $xr3, $sp, 0 # 32-byte Folded Spill
-; CHECK-NEXT: xvst $xr0, $sp, 136
-; CHECK-NEXT: xvst $xr1, $sp, 168
-; CHECK-NEXT: xvst $xr2, $sp, 200
-; CHECK-NEXT: xvst $xr3, $sp, 232
-; CHECK-NEXT: addi.d $a0, $sp, 136
+; CHECK-NEXT: xvst $xr3, $sp, 16 # 32-byte Folded Spill
+; CHECK-NEXT: xvst $xr0, $sp, 152
+; CHECK-NEXT: xvst $xr1, $sp, 184
+; CHECK-NEXT: xvst $xr2, $sp, 216
+; CHECK-NEXT: xvst $xr3, $sp, 248
+; CHECK-NEXT: addi.d $a0, $sp, 152
; CHECK-NEXT: pcaddu18i $ra, %call36(foo)
; CHECK-NEXT: jirl $ra, $ra, 0
-; CHECK-NEXT: xvld $xr0, $sp, 96 # 32-byte Folded Reload
-; CHECK-NEXT: xvst $xr0, $sp, 136
-; CHECK-NEXT: xvld $xr0, $sp, 64 # 32-byte Folded Reload
-; CHECK-NEXT: xvst $xr0, $sp, 168
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
-; CHECK-NEXT: xvst $xr0, $sp, 200
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
-; CHECK-NEXT: xvst $xr0, $sp, 232
-; CHECK-NEXT: addi.d $a0, $sp, 136
+; CHECK-NEXT: xvld $xr0, $sp, 112 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 152
+; CHECK-NEXT: xvld $xr0, $sp, 80 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 184
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 216
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 248
+; CHECK-NEXT: addi.d $a0, $sp, 152
; CHECK-NEXT: pcaddu18i $ra, %call36(bar)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: move $a0, $zero
-; CHECK-NEXT: ld.d $ra, $sp, 264 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 272
+; CHECK-NEXT: ld.d $ra, $sp, 280 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 288
; CHECK-NEXT: ret
entry:
%s = alloca %struct.S, align 2
diff --git a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
index ccc5c703e71ed..15ac95dfc6c55 100644
--- a/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
+++ b/llvm/test/CodeGen/LoongArch/emergency-spill-slot.ll
@@ -28,12 +28,12 @@ define void @func() {
; CHECK-NEXT: ld.w $a3, $a1, 0
; CHECK-NEXT: ld.w $a2, $a1, 0
; CHECK-NEXT: ld.w $a0, $a1, 0
-; CHECK-NEXT: st.d $fp, $sp, 0
+; CHECK-NEXT: st.d $fp, $sp, 0 # 8-byte Folded Spill
; CHECK-NEXT: lu12i.w $fp, 1
; CHECK-NEXT: ori $fp, $fp, 12
; CHECK-NEXT: add.d $fp, $sp, $fp
; CHECK-NEXT: st.w $t8, $fp, 0
-; CHECK-NEXT: ld.d $fp, $sp, 0
+; CHECK-NEXT: ld.d $fp, $sp, 0 # 8-byte Folded Reload
; CHECK-NEXT: st.w $t8, $a1, 0
; CHECK-NEXT: st.w $t7, $a1, 0
; CHECK-NEXT: st.w $t6, $a1, 0
diff --git a/llvm/test/CodeGen/LoongArch/frame.ll b/llvm/test/CodeGen/LoongArch/frame.ll
index 048703029d8c6..b29d8634854f3 100644
--- a/llvm/test/CodeGen/LoongArch/frame.ll
+++ b/llvm/test/CodeGen/LoongArch/frame.ll
@@ -1,5 +1,6 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch64 -mattr=+d < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,-lsx < %s | FileCheck %s --check-prefixes=CHECK,NOLSX
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx < %s | FileCheck %s --check-prefixes=CHECK,LSX
%struct.key_t = type { i32, [16 x i8] }
@@ -7,20 +8,35 @@ declare void @llvm.memset.p0.i64(ptr, i8, i64, i1)
declare void @test1(ptr)
define i32 @test() nounwind {
-; CHECK-LABEL: test:
-; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -32
-; CHECK-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill
-; CHECK-NEXT: st.w $zero, $sp, 16
-; CHECK-NEXT: vrepli.b $vr0, 0
-; CHECK-NEXT: vst $vr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 4
-; CHECK-NEXT: pcaddu18i $ra, %call36(test1)
-; CHECK-NEXT: jirl $ra, $ra, 0
-; CHECK-NEXT: move $a0, $zero
-; CHECK-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 32
-; CHECK-NEXT: ret
+; NOLSX-LABEL: test:
+; NOLSX: # %bb.0:
+; NOLSX-NEXT: addi.d $sp, $sp, -32
+; NOLSX-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill
+; NOLSX-NEXT: st.w $zero, $sp, 16
+; NOLSX-NEXT: st.d $zero, $sp, 8
+; NOLSX-NEXT: st.d $zero, $sp, 0
+; NOLSX-NEXT: addi.d $a0, $sp, 4
+; NOLSX-NEXT: pcaddu18i $ra, %call36(test1)
+; NOLSX-NEXT: jirl $ra, $ra, 0
+; NOLSX-NEXT: move $a0, $zero
+; NOLSX-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; NOLSX-NEXT: addi.d $sp, $sp, 32
+; NOLSX-NEXT: ret
+;
+; LSX-LABEL: test:
+; LSX: # %bb.0:
+; LSX-NEXT: addi.d $sp, $sp, -32
+; LSX-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill
+; LSX-NEXT: st.w $zero, $sp, 16
+; LSX-NEXT: vrepli.b $vr0, 0
+; LSX-NEXT: vst $vr0, $sp, 0
+; LSX-NEXT: addi.d $a0, $sp, 4
+; LSX-NEXT: pcaddu18i $ra, %call36(test1)
+; LSX-NEXT: jirl $ra, $ra, 0
+; LSX-NEXT: move $a0, $zero
+; LSX-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
+; LSX-NEXT: addi.d $sp, $sp, 32
+; LSX-NEXT: ret
%key = alloca %struct.key_t, align 4
call void @llvm.memset.p0.i64(ptr %key, i8 0, i64 20, i1 false)
%1 = getelementptr inbounds %struct.key_t, ptr %key, i64 0, i32 1, i64 0
@@ -98,3 +114,62 @@ define void @test_large_frame_size_1234576() "frame-pointer"="all" {
%1 = alloca i8, i32 1234567
ret void
}
+
+;; Note: will create an emergency spill slot, if (!isInt<7>(StackSize)).
+;; Should involve only one SP-adjusting addi per adjustment.
+;; LSX 112 + 16(emergency solt) = 128
+define void @test_frame_size_112() {
+; NOLSX-LABEL: test_frame_size_112:
+; NOLSX: # %bb.0:
+; NOLSX-NEXT: addi.d $sp, $sp, -112
+; NOLSX-NEXT: .cfi_def_cfa_offset 112
+; NOLSX-NEXT: addi.d $sp, $sp, 112
+; NOLSX-NEXT: ret
+;
+; LSX-LABEL: test_frame_size_112:
+; LSX: # %bb.0:
+; LSX-NEXT: addi.d $sp, $sp, -128
+; LSX-NEXT: .cfi_def_cfa_offset 128
+; LSX-NEXT: addi.d $sp, $sp, 128
+; LSX-NEXT: ret
+ %1 = alloca i8, i32 112
+ ret void
+}
+
+;; LSX 128 + 16(emergency solt) = 144
+define void @test_frame_size_128() {
+; NOLSX-LABEL: test_frame_size_128:
+; NOLSX: # %bb.0:
+; NOLSX-NEXT: addi.d $sp, $sp, -128
+; NOLSX-NEXT: .cfi_def_cfa_offset 128
+; NOLSX-NEXT: addi.d $sp, $sp, 128
+; NOLSX-NEXT: ret
+;
+; LSX-LABEL: test_frame_size_128:
+; LSX: # %bb.0:
+; LSX-NEXT: addi.d $sp, $sp, -144
+; LSX-NEXT: .cfi_def_cfa_offset 144
+; LSX-NEXT: addi.d $sp, $sp, 144
+; LSX-NEXT: ret
+ %1 = alloca i8, i32 128
+ ret void
+}
+
+;; LSX 144 + 16(emergency solt) = 160
+define void @test_frame_size_144() {
+; NOLSX-LABEL: test_frame_size_144:
+; NOLSX: # %bb.0:
+; NOLSX-NEXT: addi.d $sp, $sp, -144
+; NOLSX-NEXT: .cfi_def_cfa_offset 144
+; NOLSX-NEXT: addi.d $sp, $sp, 144
+; NOLSX-NEXT: ret
+;
+; LSX-LABEL: test_frame_size_144:
+; LSX: # %bb.0:
+; LSX-NEXT: addi.d $sp, $sp, -160
+; LSX-NEXT: .cfi_def_cfa_offset 160
+; LSX-NEXT: addi.d $sp, $sp, 160
+; LSX-NEXT: ret
+ %1 = alloca i8, i32 144
+ ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
index 402ddb9ad941b..5a55b253c77bb 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-memcpy.ll
@@ -6,11 +6,11 @@
define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 dereferenceable(48) %b, i64 %i) {
; CHECK-LABEL: box:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -96
-; CHECK-NEXT: .cfi_def_cfa_offset 96
+; CHECK-NEXT: addi.d $sp, $sp, -112
+; CHECK-NEXT: .cfi_def_cfa_offset 112
; CHECK-NEXT: slli.d $a2, $a1, 5
; CHECK-NEXT: alsl.d $a1, $a1, $a2, 4
-; CHECK-NEXT: addi.d $a2, $sp, 0
+; CHECK-NEXT: addi.d $a2, $sp, 16
; CHECK-NEXT: add.d $a3, $a2, $a1
; CHECK-NEXT: vldx $vr0, $a1, $a2
; CHECK-NEXT: vld $vr1, $a3, 32
@@ -18,7 +18,7 @@ define void @box(ptr noalias nocapture noundef writeonly sret(%Box) align 16 der
; CHECK-NEXT: vst $vr0, $a0, 0
; CHECK-NEXT: vst $vr1, $a0, 32
; CHECK-NEXT: vst $vr2, $a0, 16
-; CHECK-NEXT: addi.d $sp, $sp, 96
+; CHECK-NEXT: addi.d $sp, $sp, 112
; CHECK-NEXT: ret
%1 = alloca [2 x %Box], align 16
%2 = getelementptr inbounds [2 x %Box], ptr %1, i64 0, i64 %i
diff --git a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
index 789b51d9b5e5b..9528280d181a3 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/fpowi.ll
@@ -6,10 +6,10 @@ declare <8 x float> @llvm.powi.v8f32.i32(<8 x float>, i32)
define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
; CHECK-LABEL: powi_v8f32:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: addi.d $sp, $sp, -80
-; CHECK-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT: xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: xvst $xr0, $sp, 16 # 32-byte Folded Spill
; CHECK-NEXT: addi.w $fp, $a0, 0
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 0
; CHECK-NEXT: movgr2fr.w $fa0, $a0
@@ -18,79 +18,79 @@ define <8 x float> @powi_v8f32(<8 x float> %va, i32 %b) nounwind {
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 1
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 1
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 2
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 2
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 3
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 3
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 4
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 4
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 5
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 5
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 6
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 6
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.w $a0, $xr0, 7
; CHECK-NEXT: movgr2fr.w $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powisf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.s $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.w $xr0, $a0, 7
-; CHECK-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 80
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
entry:
%res = call <8 x float> @llvm.powi.v8f32.i32(<8 x float> %va, i32 %b)
@@ -102,10 +102,10 @@ declare <4 x double> @llvm.powi.v4f64.i32(<4 x double>, i32)
define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
; CHECK-LABEL: powi_v4f64:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: addi.d $sp, $sp, -80
-; CHECK-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill
-; CHECK-NEXT: xvst $xr0, $sp, 0 # 32-byte Folded Spill
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: xvst $xr0, $sp, 16 # 32-byte Folded Spill
; CHECK-NEXT: addi.w $fp, $a0, 0
; CHECK-NEXT: xvpickve2gr.d $a0, $xr0, 0
; CHECK-NEXT: movgr2fr.d $fa0, $a0
@@ -114,39 +114,39 @@ define <4 x double> @powi_v4f64(<4 x double> %va, i32 %b) nounwind {
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.d $a0, $fa0
; CHECK-NEXT: xvinsgr2vr.d $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.d $a0, $xr0, 1
; CHECK-NEXT: movgr2fr.d $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powidf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.d $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.d $xr0, $a0, 1
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.d $a0, $xr0, 2
; CHECK-NEXT: movgr2fr.d $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powidf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.d $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.d $xr0, $a0, 2
-; CHECK-NEXT: xvst $xr0, $sp, 32 # 32-byte Folded Spill
-; CHECK-NEXT: xvld $xr0, $sp, 0 # 32-byte Folded Reload
+; CHECK-NEXT: xvst $xr0, $sp, 48 # 32-byte Folded Spill
+; CHECK-NEXT: xvld $xr0, $sp, 16 # 32-byte Folded Reload
; CHECK-NEXT: xvpickve2gr.d $a0, $xr0, 3
; CHECK-NEXT: movgr2fr.d $fa0, $a0
; CHECK-NEXT: move $a0, $fp
; CHECK-NEXT: pcaddu18i $ra, %call36(__powidf2)
; CHECK-NEXT: jirl $ra, $ra, 0
; CHECK-NEXT: movfr2gr.d $a0, $fa0
-; CHECK-NEXT: xvld $xr0, $sp, 32 # 32-byte Folded Reload
+; CHECK-NEXT: xvld $xr0, $sp, 48 # 32-byte Folded Reload
; CHECK-NEXT: xvinsgr2vr.d $xr0, $a0, 3
-; CHECK-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 80
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
entry:
%res = call <4 x double> @llvm.powi.v4f64.i32(<4 x double> %va, i32 %b)
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
index 04214f5dfa9d2..2e1618748688a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/extractelement.ll
@@ -76,21 +76,21 @@ define void @extract_4xdouble(ptr %src, ptr %dst) nounwind {
define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
; CHECK-LABEL: extract_32xi8_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 0
; CHECK-NEXT: ld.b $a0, $a0, 0
; CHECK-NEXT: st.b $a0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <32 x i8>, ptr %src
%e = extractelement <32 x i8> %v, i32 %idx
@@ -101,21 +101,21 @@ define void @extract_32xi8_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
; CHECK-LABEL: extract_16xi16_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 1
; CHECK-NEXT: ld.h $a0, $a0, 0
; CHECK-NEXT: st.h $a0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <16 x i16>, ptr %src
%e = extractelement <16 x i16> %v, i32 %idx
@@ -126,21 +126,21 @@ define void @extract_16xi16_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
; CHECK-LABEL: extract_8xi32_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 2
; CHECK-NEXT: ld.w $a0, $a0, 0
; CHECK-NEXT: st.w $a0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <8 x i32>, ptr %src
%e = extractelement <8 x i32> %v, i32 %idx
@@ -151,21 +151,21 @@ define void @extract_8xi32_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
; CHECK-LABEL: extract_4xi64_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 3
; CHECK-NEXT: ld.d $a0, $a0, 0
; CHECK-NEXT: st.d $a0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <4 x i64>, ptr %src
%e = extractelement <4 x i64> %v, i32 %idx
@@ -176,21 +176,21 @@ define void @extract_4xi64_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
; CHECK-LABEL: extract_8xfloat_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 2
; CHECK-NEXT: fld.s $fa0, $a0, 0
; CHECK-NEXT: fst.s $fa0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <8 x float>, ptr %src
%e = extractelement <8 x float> %v, i32 %idx
@@ -201,21 +201,21 @@ define void @extract_8xfloat_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
define void @extract_4xdouble_idx(ptr %src, ptr %dst, i32 %idx) nounwind {
; CHECK-LABEL: extract_4xdouble_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 3
; CHECK-NEXT: fld.d $fa0, $a0, 0
; CHECK-NEXT: fst.d $fa0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <4 x double>, ptr %src
%e = extractelement <4 x double> %v, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
index 3a4f6efd2c893..b24f95e676d10 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
@@ -114,22 +114,22 @@ define void @insert_4xdouble(ptr %src, ptr %dst, double %in) nounwind {
define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
; CHECK-LABEL: insert_32xi8_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a3, 4, 0
; CHECK-NEXT: st.b $a2, $a0, 0
-; CHECK-NEXT: xvld $xr0, $sp, 0
+; CHECK-NEXT: xvld $xr0, $sp, 32
; CHECK-NEXT: xvst $xr0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <32 x i8>, ptr %src
%v_new = insertelement <32 x i8> %v, i8 %in, i32 %idx
@@ -140,22 +140,22 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
; CHECK-LABEL: insert_16xi16_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a3, 4, 1
; CHECK-NEXT: st.h $a2, $a0, 0
-; CHECK-NEXT: xvld $xr0, $sp, 0
+; CHECK-NEXT: xvld $xr0, $sp, 32
; CHECK-NEXT: xvst $xr0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <16 x i16>, ptr %src
%v_new = insertelement <16 x i16> %v, i16 %in, i32 %idx
@@ -166,22 +166,22 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
; CHECK-LABEL: insert_8xi32_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a3, 4, 2
; CHECK-NEXT: st.w $a2, $a0, 0
-; CHECK-NEXT: xvld $xr0, $sp, 0
+; CHECK-NEXT: xvld $xr0, $sp, 32
; CHECK-NEXT: xvst $xr0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <8 x i32>, ptr %src
%v_new = insertelement <8 x i32> %v, i32 %in, i32 %idx
@@ -192,22 +192,22 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
; CHECK-LABEL: insert_4xi64_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr0, $a0, 0
-; CHECK-NEXT: xvst $xr0, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr0, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a3, 4, 3
; CHECK-NEXT: st.d $a2, $a0, 0
-; CHECK-NEXT: xvld $xr0, $sp, 0
+; CHECK-NEXT: xvld $xr0, $sp, 32
; CHECK-NEXT: xvst $xr0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <4 x i64>, ptr %src
%v_new = insertelement <4 x i64> %v, i64 %in, i32 %idx
@@ -218,22 +218,22 @@ define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwind {
; CHECK-LABEL: insert_8xfloat_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr1, $a0, 0
-; CHECK-NEXT: xvst $xr1, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr1, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 2
; CHECK-NEXT: fst.s $fa0, $a0, 0
-; CHECK-NEXT: xvld $xr0, $sp, 0
+; CHECK-NEXT: xvld $xr0, $sp, 32
; CHECK-NEXT: xvst $xr0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <8 x float>, ptr %src
%v_new = insertelement <8 x float> %v, float %in, i32 %idx
@@ -244,22 +244,22 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin
define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounwind {
; CHECK-LABEL: insert_4xdouble_idx:
; CHECK: # %bb.0:
-; CHECK-NEXT: addi.d $sp, $sp, -64
-; CHECK-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; CHECK-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; CHECK-NEXT: addi.d $fp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $sp, -96
+; CHECK-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $fp, $sp, 96
; CHECK-NEXT: bstrins.d $sp, $zero, 4, 0
; CHECK-NEXT: xvld $xr1, $a0, 0
-; CHECK-NEXT: xvst $xr1, $sp, 0
-; CHECK-NEXT: addi.d $a0, $sp, 0
+; CHECK-NEXT: xvst $xr1, $sp, 32
+; CHECK-NEXT: addi.d $a0, $sp, 32
; CHECK-NEXT: bstrins.d $a0, $a2, 4, 3
; CHECK-NEXT: fst.d $fa0, $a0, 0
-; CHECK-NEXT: xvld $xr0, $sp, 0
+; CHECK-NEXT: xvld $xr0, $sp, 32
; CHECK-NEXT: xvst $xr0, $a1, 0
-; CHECK-NEXT: addi.d $sp, $fp, -64
-; CHECK-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; CHECK-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; CHECK-NEXT: addi.d $sp, $sp, 64
+; CHECK-NEXT: addi.d $sp, $fp, -96
+; CHECK-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 96
; CHECK-NEXT: ret
%v = load volatile <4 x double>, ptr %src
%v_new = insertelement <4 x double> %v, double %in, i32 %idx
diff --git a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
index ffedd7f9e9438..648c19d509715 100644
--- a/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
+++ b/llvm/test/CodeGen/LoongArch/llvm.sincos.ll
@@ -347,42 +347,42 @@ define { <2 x float>, <2 x float> } @test_sincos_v2f32(<2 x float> %a) #0 {
;
; LA64-LABEL: test_sincos_v2f32:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -64
-; LA64-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT: vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -80
+; LA64-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
; LA64-NEXT: vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sinf)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT: vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
; LA64-NEXT: vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT: vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sinf)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT: vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT: vld $vr1, $sp, 32 # 16-byte Folded Reload
; LA64-NEXT: vpackev.w $vr0, $vr0, $vr1
-; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cosf)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT: vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cosf)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: # kill: def $f0 killed $f0 def $vr0
-; LA64-NEXT: vld $vr1, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: vld $vr1, $sp, 48 # 16-byte Folded Reload
; LA64-NEXT: vpackev.w $vr1, $vr0, $vr1
-; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 64
+; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 80
; LA64-NEXT: ret
%result = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> %a)
ret { <2 x float>, <2 x float> } %result
@@ -439,48 +439,48 @@ define { <3 x float>, <3 x float> } @test_sincos_v3f32(<3 x float> %a) #0 {
;
; LA64-LABEL: test_sincos_v3f32:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -96
-; LA64-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
-; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -112
+; LA64-NEXT: st.d $ra, $sp, 104 # 8-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill
; LA64-NEXT: vreplvei.w $vr0, $vr0, 2
-; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sinf)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: fst.s $fa0, $sp, 72
-; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: fst.s $fa0, $sp, 88
+; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload
; LA64-NEXT: vreplvei.w $vr0, $vr0, 1
-; LA64-NEXT: vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sinf)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: fst.s $fa0, $sp, 68
-; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: fst.s $fa0, $sp, 84
+; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload
; LA64-NEXT: vreplvei.w $vr0, $vr0, 0
-; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sinf)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: fst.s $fa0, $sp, 64
-; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT: fst.s $fa0, $sp, 80
+; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cosf)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: fst.s $fa0, $sp, 56
-; LA64-NEXT: vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT: fst.s $fa0, $sp, 72
+; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cosf)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: fst.s $fa0, $sp, 52
-; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: fst.s $fa0, $sp, 68
+; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0 killed $f0 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cosf)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: fst.s $fa0, $sp, 48
-; LA64-NEXT: vld $vr0, $sp, 64
-; LA64-NEXT: vld $vr1, $sp, 48
-; LA64-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 96
+; LA64-NEXT: fst.s $fa0, $sp, 64
+; LA64-NEXT: vld $vr0, $sp, 80
+; LA64-NEXT: vld $vr1, $sp, 64
+; LA64-NEXT: ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 112
; LA64-NEXT: ret
%result = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %a)
ret { <3 x float>, <3 x float> } %result
@@ -568,44 +568,44 @@ define { <2 x double>, <2 x double> } @test_sincos_v2f64(<2 x double> %a) #0 {
;
; LA64-LABEL: test_sincos_v2f64:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -64
-; LA64-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT: vst $vr0, $sp, 0 # 16-byte Folded Spill
-; LA64-NEXT: vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT: addi.d $sp, $sp, -80
+; LA64-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
+; LA64-NEXT: vreplvei.d $vr0, $vr0, 0
+; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0_64 killed $f0_64 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sin)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: movfr2gr.d $a0, $fa0
; LA64-NEXT: vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT: vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
; LA64-NEXT: vreplvei.d $vr0, $vr0, 1
-; LA64-NEXT: vst $vr0, $sp, 0 # 16-byte Folded Spill
+; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
; LA64-NEXT: # kill: def $f0_64 killed $f0_64 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(sin)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: movfr2gr.d $a0, $fa0
-; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
+; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload
; LA64-NEXT: vinsgr2vr.d $vr0, $a0, 1
-; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
-; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT: vst $vr0, $sp, 48 # 16-byte Folded Spill
+; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0_64 killed $f0_64 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cos)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: movfr2gr.d $a0, $fa0
; LA64-NEXT: vinsgr2vr.d $vr0, $a0, 0
-; LA64-NEXT: vst $vr0, $sp, 16 # 16-byte Folded Spill
-; LA64-NEXT: vld $vr0, $sp, 0 # 16-byte Folded Reload
+; LA64-NEXT: vst $vr0, $sp, 32 # 16-byte Folded Spill
+; LA64-NEXT: vld $vr0, $sp, 16 # 16-byte Folded Reload
; LA64-NEXT: # kill: def $f0_64 killed $f0_64 killed $vr0
; LA64-NEXT: pcaddu18i $ra, %call36(cos)
; LA64-NEXT: jirl $ra, $ra, 0
; LA64-NEXT: movfr2gr.d $a0, $fa0
-; LA64-NEXT: vld $vr1, $sp, 16 # 16-byte Folded Reload
+; LA64-NEXT: vld $vr1, $sp, 32 # 16-byte Folded Reload
; LA64-NEXT: vinsgr2vr.d $vr1, $a0, 1
-; LA64-NEXT: vld $vr0, $sp, 32 # 16-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 64
+; LA64-NEXT: vld $vr0, $sp, 48 # 16-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 80
; LA64-NEXT: ret
%result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %a)
ret { <2 x double>, <2 x double> } %result
@@ -801,17 +801,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
;
; LA64-LABEL: test_sincos_v2f128:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -80
-; LA64-NEXT: st.d $ra, $sp, 72 # 8-byte Folded Spill
-; LA64-NEXT: st.d $fp, $sp, 64 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s0, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s1, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s2, $sp, 40 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s3, $sp, 32 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s4, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s5, $sp, 16 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s6, $sp, 8 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s7, $sp, 0 # 8-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -96
+; LA64-NEXT: st.d $ra, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT: st.d $fp, $sp, 80 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s0, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s1, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s2, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s3, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s4, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s5, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s6, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s7, $sp, 16 # 8-byte Folded Spill
; LA64-NEXT: ld.d $fp, $a1, 16
; LA64-NEXT: ld.d $s0, $a1, 24
; LA64-NEXT: ld.d $s1, $a1, 0
@@ -847,17 +847,17 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
; LA64-NEXT: st.d $s6, $s3, 16
; LA64-NEXT: st.d $s5, $s3, 8
; LA64-NEXT: st.d $s4, $s3, 0
-; LA64-NEXT: ld.d $s7, $sp, 0 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s6, $sp, 8 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s5, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s4, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s3, $sp, 32 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s2, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s1, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $s0, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $fp, $sp, 64 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 72 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 80
+; LA64-NEXT: ld.d $s7, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s6, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s5, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s4, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s3, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s2, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s1, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $s0, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $fp, $sp, 80 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 96
; LA64-NEXT: ret
%result = call { <2 x fp128>, <2 x fp128> } @llvm.sincos.v2f128(<2 x fp128> %a)
ret { <2 x fp128>, <2 x fp128> } %result
diff --git a/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
new file mode 100644
index 0000000000000..96159e5884d3f
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/pr146455.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 -mattr=+d,+lsx --verify-machineinstrs < %s | FileCheck %s
+define void @eliminate_frame_index(<16 x i8> %a) nounwind {
+; CHECK-LABEL: eliminate_frame_index:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi.d $sp, $sp, -240
+; CHECK-NEXT: st.d $ra, $sp, 232 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $fp, $sp, 224 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s0, $sp, 216 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s1, $sp, 208 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s2, $sp, 200 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s3, $sp, 192 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s4, $sp, 184 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s5, $sp, 176 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s6, $sp, 168 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s7, $sp, 160 # 8-byte Folded Spill
+; CHECK-NEXT: st.d $s8, $sp, 152 # 8-byte Folded Spill
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $zero, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $ra, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $tp, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a0, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a1, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a2, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a3, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a4, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a5, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a6, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $a7, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t0, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t1, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t2, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t3, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t4, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t5, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t6, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t7, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $t8, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $fp, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s0, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s1, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s2, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s3, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s4, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s5, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s6, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s7, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: addi.d $s8, $zero, 1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: st.d $a0, $sp, 0 # 8-byte Folded Spill
+; CHECK-NEXT: addi.d $a0, $sp, 136
+; CHECK-NEXT: vstelm.b $vr0, $a0, 0, 0
+; CHECK-NEXT: ld.d $a0, $sp, 0 # 8-byte Folded Reload
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $zero
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $ra
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $tp
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a3
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a4
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a5
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a6
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $a7
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t3
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t4
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t5
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t6
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t7
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $t8
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $fp
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s0
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s1
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s2
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s3
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s4
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s5
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s6
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s7
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: #APP
+; CHECK-NEXT: # reg use $s8
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: ld.d $s8, $sp, 152 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s7, $sp, 160 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s6, $sp, 168 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s5, $sp, 176 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s4, $sp, 184 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s3, $sp, 192 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s2, $sp, 200 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s1, $sp, 208 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $s0, $sp, 216 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $fp, $sp, 224 # 8-byte Folded Reload
+; CHECK-NEXT: ld.d $ra, $sp, 232 # 8-byte Folded Reload
+; CHECK-NEXT: addi.d $sp, $sp, 240
+; CHECK-NEXT: ret
+ %s = alloca [16 x i8]
+ %ss = alloca [128 x i8]
+
+ %zero = call i64 asm sideeffect "addi.d $$zero, $$zero, 1", "={r0}"()
+ %ra = call i64 asm sideeffect "addi.d $$ra, $$zero, 1", "={r1}"()
+ %tp = call i64 asm sideeffect "addi.d $$tp, $$zero, 1", "={r2}"()
+ %a0 = call i64 asm sideeffect "addi.d $$a0, $$zero, 1", "={r4}"()
+ %a1 = call i64 asm sideeffect "addi.d $$a1, $$zero, 1", "={r5}"()
+ %a2 = call i64 asm sideeffect "addi.d $$a2, $$zero, 1", "={r6}"()
+ %a3 = call i64 asm sideeffect "addi.d $$a3, $$zero, 1", "={r7}"()
+ %a4 = call i64 asm sideeffect "addi.d $$a4, $$zero, 1", "={r8}"()
+ %a5 = call i64 asm sideeffect "addi.d $$a5, $$zero, 1", "={r9}"()
+ %a6 = call i64 asm sideeffect "addi.d $$a6, $$zero, 1", "={r10}"()
+ %a7 = call i64 asm sideeffect "addi.d $$a7, $$zero, 1", "={r11}"()
+ %t0 = call i64 asm sideeffect "addi.d $$t0, $$zero, 1", "={r12}"()
+ %t1 = call i64 asm sideeffect "addi.d $$t1, $$zero, 1", "={r13}"()
+ %t2 = call i64 asm sideeffect "addi.d $$t2, $$zero, 1", "={r14}"()
+ %t3 = call i64 asm sideeffect "addi.d $$t3, $$zero, 1", "={r15}"()
+ %t4 = call i64 asm sideeffect "addi.d $$t4, $$zero, 1", "={r16}"()
+ %t5 = call i64 asm sideeffect "addi.d $$t5, $$zero, 1", "={r17}"()
+ %t6 = call i64 asm sideeffect "addi.d $$t6, $$zero, 1", "={r18}"()
+ %t7 = call i64 asm sideeffect "addi.d $$t7, $$zero, 1", "={r19}"()
+ %t8 = call i64 asm sideeffect "addi.d $$t8, $$zero, 1", "={r20}"()
+ ;; r21 Reserved (Non-allocatable)
+ %s9 = call i64 asm sideeffect "addi.d $$s9, $$zero, 1", "={r22}"()
+ %s0 = call i64 asm sideeffect "addi.d $$s0, $$zero, 1", "={r23}"()
+ %s1 = call i64 asm sideeffect "addi.d $$s1, $$zero, 1", "={r24}"()
+ %s2 = call i64 asm sideeffect "addi.d $$s2, $$zero, 1", "={r25}"()
+ %s3 = call i64 asm sideeffect "addi.d $$s3, $$zero, 1", "={r26}"()
+ %s4 = call i64 asm sideeffect "addi.d $$s4, $$zero, 1", "={r27}"()
+ %s5 = call i64 asm sideeffect "addi.d $$s5, $$zero, 1", "={r28}"()
+ %s6 = call i64 asm sideeffect "addi.d $$s6, $$zero, 1", "={r29}"()
+ %s7 = call i64 asm sideeffect "addi.d $$s7, $$zero, 1", "={r30}"()
+ %s8 = call i64 asm sideeffect "addi.d $$s8, $$zero, 1", "={r31}"()
+
+ %e = extractelement <16 x i8> %a, i64 0
+
+ store volatile i8 %e, ptr %s
+
+ call void asm sideeffect "# reg use $0", "{r0}"(i64 %zero)
+ call void asm sideeffect "# reg use $0", "{r1}"(i64 %ra)
+ call void asm sideeffect "# reg use $0", "{r2}"(i64 %tp)
+ call void asm sideeffect "# reg use $0", "{r4}"(i64 %a0)
+ call void asm sideeffect "# reg use $0", "{r5}"(i64 %a1)
+ call void asm sideeffect "# reg use $0", "{r6}"(i64 %a2)
+ call void asm sideeffect "# reg use $0", "{r7}"(i64 %a3)
+ call void asm sideeffect "# reg use $0", "{r8}"(i64 %a4)
+ call void asm sideeffect "# reg use $0", "{r9}"(i64 %a5)
+ call void asm sideeffect "# reg use $0", "{r10}"(i64 %a6)
+ call void asm sideeffect "# reg use $0", "{r11}"(i64 %a7)
+ call void asm sideeffect "# reg use $0", "{r12}"(i64 %t0)
+ call void asm sideeffect "# reg use $0", "{r13}"(i64 %t1)
+ call void asm sideeffect "# reg use $0", "{r14}"(i64 %t2)
+ call void asm sideeffect "# reg use $0", "{r15}"(i64 %t3)
+ call void asm sideeffect "# reg use $0", "{r16}"(i64 %t4)
+ call void asm sideeffect "# reg use $0", "{r17}"(i64 %t5)
+ call void asm sideeffect "# reg use $0", "{r18}"(i64 %t6)
+ call void asm sideeffect "# reg use $0", "{r19}"(i64 %t7)
+ call void asm sideeffect "# reg use $0", "{r20}"(i64 %t8)
+ ;; r21 Reserved (Non-allocatable)
+ call void asm sideeffect "# reg use $0", "{r22}"(i64 %s9)
+ call void asm sideeffect "# reg use $0", "{r23}"(i64 %s0)
+ call void asm sideeffect "# reg use $0", "{r24}"(i64 %s1)
+ call void asm sideeffect "# reg use $0", "{r25}"(i64 %s2)
+ call void asm sideeffect "# reg use $0", "{r26}"(i64 %s3)
+ call void asm sideeffect "# reg use $0", "{r27}"(i64 %s4)
+ call void asm sideeffect "# reg use $0", "{r28}"(i64 %s5)
+ call void asm sideeffect "# reg use $0", "{r29}"(i64 %s6)
+ call void asm sideeffect "# reg use $0", "{r30}"(i64 %s7)
+ call void asm sideeffect "# reg use $0", "{r31}"(i64 %s8)
+
+ ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
index 9f15604fcca6b..69995a0721f8a 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment-with-variable-sized-objects.ll
@@ -36,15 +36,15 @@ define void @caller(i32 %n) {
;
; LA64-LABEL: caller:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -64
-; LA64-NEXT: .cfi_def_cfa_offset 64
-; LA64-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
-; LA64-NEXT: st.d $s8, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -128
+; LA64-NEXT: .cfi_def_cfa_offset 128
+; LA64-NEXT: st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT: st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT: st.d $s8, $sp, 104 # 8-byte Folded Spill
; LA64-NEXT: .cfi_offset 1, -8
; LA64-NEXT: .cfi_offset 22, -16
; LA64-NEXT: .cfi_offset 31, -24
-; LA64-NEXT: addi.d $fp, $sp, 64
+; LA64-NEXT: addi.d $fp, $sp, 128
; LA64-NEXT: .cfi_def_cfa 22, 0
; LA64-NEXT: bstrins.d $sp, $zero, 5, 0
; LA64-NEXT: move $s8, $sp
@@ -54,14 +54,14 @@ define void @caller(i32 %n) {
; LA64-NEXT: slli.d $a0, $a0, 4
; LA64-NEXT: sub.d $a0, $sp, $a0
; LA64-NEXT: move $sp, $a0
-; LA64-NEXT: addi.d $a1, $s8, 0
+; LA64-NEXT: addi.d $a1, $s8, 64
; LA64-NEXT: pcaddu18i $ra, %call36(callee)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: addi.d $sp, $fp, -64
-; LA64-NEXT: ld.d $s8, $sp, 40 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 64
+; LA64-NEXT: addi.d $sp, $fp, -128
+; LA64-NEXT: ld.d $s8, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 128
; LA64-NEXT: ret
%1 = alloca i8, i32 %n
%2 = alloca i32, align 64
diff --git a/llvm/test/CodeGen/LoongArch/stack-realignment.ll b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
index 0645339358b64..0188884543adb 100644
--- a/llvm/test/CodeGen/LoongArch/stack-realignment.ll
+++ b/llvm/test/CodeGen/LoongArch/stack-realignment.ll
@@ -28,22 +28,22 @@ define void @caller32() {
;
; LA64-LABEL: caller32:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -32
-; LA64-NEXT: .cfi_def_cfa_offset 32
-; LA64-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -64
+; LA64-NEXT: .cfi_def_cfa_offset 64
+; LA64-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
; LA64-NEXT: .cfi_offset 1, -8
; LA64-NEXT: .cfi_offset 22, -16
-; LA64-NEXT: addi.d $fp, $sp, 32
+; LA64-NEXT: addi.d $fp, $sp, 64
; LA64-NEXT: .cfi_def_cfa 22, 0
; LA64-NEXT: bstrins.d $sp, $zero, 4, 0
-; LA64-NEXT: addi.d $a0, $sp, 0
+; LA64-NEXT: addi.d $a0, $sp, 32
; LA64-NEXT: pcaddu18i $ra, %call36(callee)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: addi.d $sp, $fp, -32
-; LA64-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 32
+; LA64-NEXT: addi.d $sp, $fp, -64
+; LA64-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 64
; LA64-NEXT: ret
%1 = alloca i8, align 32
call void @callee(ptr %1)
@@ -102,22 +102,22 @@ define void @caller64() {
;
; LA64-LABEL: caller64:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -64
-; LA64-NEXT: .cfi_def_cfa_offset 64
-; LA64-NEXT: st.d $ra, $sp, 56 # 8-byte Folded Spill
-; LA64-NEXT: st.d $fp, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -128
+; LA64-NEXT: .cfi_def_cfa_offset 128
+; LA64-NEXT: st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-NEXT: st.d $fp, $sp, 112 # 8-byte Folded Spill
; LA64-NEXT: .cfi_offset 1, -8
; LA64-NEXT: .cfi_offset 22, -16
-; LA64-NEXT: addi.d $fp, $sp, 64
+; LA64-NEXT: addi.d $fp, $sp, 128
; LA64-NEXT: .cfi_def_cfa 22, 0
; LA64-NEXT: bstrins.d $sp, $zero, 5, 0
-; LA64-NEXT: addi.d $a0, $sp, 0
+; LA64-NEXT: addi.d $a0, $sp, 64
; LA64-NEXT: pcaddu18i $ra, %call36(callee)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: addi.d $sp, $fp, -64
-; LA64-NEXT: ld.d $fp, $sp, 48 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 56 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 64
+; LA64-NEXT: addi.d $sp, $fp, -128
+; LA64-NEXT: ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 128
; LA64-NEXT: ret
%1 = alloca i8, align 64
call void @callee(ptr %1)
@@ -176,22 +176,22 @@ define void @caller128() {
;
; LA64-LABEL: caller128:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -128
-; LA64-NEXT: .cfi_def_cfa_offset 128
-; LA64-NEXT: st.d $ra, $sp, 120 # 8-byte Folded Spill
-; LA64-NEXT: st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -256
+; LA64-NEXT: .cfi_def_cfa_offset 256
+; LA64-NEXT: st.d $ra, $sp, 248 # 8-byte Folded Spill
+; LA64-NEXT: st.d $fp, $sp, 240 # 8-byte Folded Spill
; LA64-NEXT: .cfi_offset 1, -8
; LA64-NEXT: .cfi_offset 22, -16
-; LA64-NEXT: addi.d $fp, $sp, 128
+; LA64-NEXT: addi.d $fp, $sp, 256
; LA64-NEXT: .cfi_def_cfa 22, 0
; LA64-NEXT: bstrins.d $sp, $zero, 6, 0
-; LA64-NEXT: addi.d $a0, $sp, 0
+; LA64-NEXT: addi.d $a0, $sp, 128
; LA64-NEXT: pcaddu18i $ra, %call36(callee)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: addi.d $sp, $fp, -128
-; LA64-NEXT: ld.d $fp, $sp, 112 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 120 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 128
+; LA64-NEXT: addi.d $sp, $fp, -256
+; LA64-NEXT: ld.d $fp, $sp, 240 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 248 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 256
; LA64-NEXT: ret
%1 = alloca i8, align 128
call void @callee(ptr %1)
@@ -250,22 +250,22 @@ define void @caller256() {
;
; LA64-LABEL: caller256:
; LA64: # %bb.0:
-; LA64-NEXT: addi.d $sp, $sp, -256
-; LA64-NEXT: .cfi_def_cfa_offset 256
-; LA64-NEXT: st.d $ra, $sp, 248 # 8-byte Folded Spill
-; LA64-NEXT: st.d $fp, $sp, 240 # 8-byte Folded Spill
+; LA64-NEXT: addi.d $sp, $sp, -512
+; LA64-NEXT: .cfi_def_cfa_offset 512
+; LA64-NEXT: st.d $ra, $sp, 504 # 8-byte Folded Spill
+; LA64-NEXT: st.d $fp, $sp, 496 # 8-byte Folded Spill
; LA64-NEXT: .cfi_offset 1, -8
; LA64-NEXT: .cfi_offset 22, -16
-; LA64-NEXT: addi.d $fp, $sp, 256
+; LA64-NEXT: addi.d $fp, $sp, 512
; LA64-NEXT: .cfi_def_cfa 22, 0
; LA64-NEXT: bstrins.d $sp, $zero, 7, 0
-; LA64-NEXT: addi.d $a0, $sp, 0
+; LA64-NEXT: addi.d $a0, $sp, 256
; LA64-NEXT: pcaddu18i $ra, %call36(callee)
; LA64-NEXT: jirl $ra, $ra, 0
-; LA64-NEXT: addi.d $sp, $fp, -256
-; LA64-NEXT: ld.d $fp, $sp, 240 # 8-byte Folded Reload
-; LA64-NEXT: ld.d $ra, $sp, 248 # 8-byte Folded Reload
-; LA64-NEXT: addi.d $sp, $sp, 256
+; LA64-NEXT: addi.d $sp, $fp, -512
+; LA64-NEXT: ld.d $fp, $sp, 496 # 8-byte Folded Reload
+; LA64-NEXT: ld.d $ra, $sp, 504 # 8-byte Folded Reload
+; LA64-NEXT: addi.d $sp, $sp, 512
; LA64-NEXT: ret
%1 = alloca i8, align 256
call void @callee(ptr %1)
diff --git a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
index 925fdf3d60646..0d441e66a0c84 100644
--- a/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
+++ b/llvm/test/CodeGen/LoongArch/unaligned-memcpy-inline.ll
@@ -121,19 +121,19 @@ define void @t3() {
;
; LA64-LABEL: t3:
; LA64: # %bb.0: # %entry
-; LA64-NEXT: addi.d $sp, $sp, -64
-; LA64-NEXT: .cfi_def_cfa_offset 64
+; LA64-NEXT: addi.d $sp, $sp, -80
+; LA64-NEXT: .cfi_def_cfa_offset 80
; LA64-NEXT: pcalau12i $a0, %pc_hi20(.L.str)
; LA64-NEXT: addi.d $a0, $a0, %pc_lo12(.L.str)
; LA64-NEXT: ld.h $a1, $a0, 20
; LA64-NEXT: ld.w $a2, $a0, 16
; LA64-NEXT: ld.d $a3, $a0, 8
; LA64-NEXT: ld.d $a0, $a0, 0
-; LA64-NEXT: st.h $a1, $sp, 20
-; LA64-NEXT: st.w $a2, $sp, 16
-; LA64-NEXT: st.d $a3, $sp, 8
-; LA64-NEXT: st.d $a0, $sp, 0
-; LA64-NEXT: addi.d $sp, $sp, 64
+; LA64-NEXT: st.h $a1, $sp, 36
+; LA64-NEXT: st.w $a2, $sp, 32
+; LA64-NEXT: st.d $a3, $sp, 24
+; LA64-NEXT: st.d $a0, $sp, 16
+; LA64-NEXT: addi.d $sp, $sp, 80
; LA64-NEXT: ret
entry:
%msgbuf = alloca [64 x i8], align 1
diff --git a/llvm/test/CodeGen/LoongArch/vararg.ll b/llvm/test/CodeGen/LoongArch/vararg.ll
index 939cd2015c5b1..bc4b8a77c7e15 100644
--- a/llvm/test/CodeGen/LoongArch/vararg.ll
+++ b/llvm/test/CodeGen/LoongArch/vararg.ll
@@ -47,7 +47,7 @@ define i64 @va1(ptr %fmt, ...) {
; LA64-WITHFP-NEXT: st.d $a2, $fp, 16
; LA64-WITHFP-NEXT: st.d $a1, $fp, 8
; LA64-WITHFP-NEXT: addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT: st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT: st.d $a1, $fp, -32
; LA64-WITHFP-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload
; LA64-WITHFP-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
; LA64-WITHFP-NEXT: addi.d $sp, $sp, 96
@@ -94,7 +94,7 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
; LA64-WITHFP-NEXT: st.d $a2, $fp, 16
; LA64-WITHFP-NEXT: st.d $a1, $fp, 8
; LA64-WITHFP-NEXT: addi.d $a1, $fp, 16
-; LA64-WITHFP-NEXT: st.d $a1, $fp, -24
+; LA64-WITHFP-NEXT: st.d $a1, $fp, -32
; LA64-WITHFP-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload
; LA64-WITHFP-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
; LA64-WITHFP-NEXT: addi.d $sp, $sp, 96
@@ -112,11 +112,11 @@ define i64 @va1_va_arg(ptr %fmt, ...) nounwind {
define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
; LA64-FPELIM-LABEL: va1_va_arg_alloca:
; LA64-FPELIM: # %bb.0:
-; LA64-FPELIM-NEXT: addi.d $sp, $sp, -96
-; LA64-FPELIM-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-FPELIM-NEXT: addi.d $fp, $sp, 32
+; LA64-FPELIM-NEXT: addi.d $sp, $sp, -112
+; LA64-FPELIM-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT: st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT: st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-FPELIM-NEXT: addi.d $fp, $sp, 48
; LA64-FPELIM-NEXT: move $s0, $a1
; LA64-FPELIM-NEXT: st.d $a7, $fp, 56
; LA64-FPELIM-NEXT: st.d $a6, $fp, 48
@@ -126,7 +126,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
; LA64-FPELIM-NEXT: st.d $a2, $fp, 16
; LA64-FPELIM-NEXT: st.d $a1, $fp, 8
; LA64-FPELIM-NEXT: addi.d $a0, $fp, 16
-; LA64-FPELIM-NEXT: st.d $a0, $fp, -32
+; LA64-FPELIM-NEXT: st.d $a0, $fp, -40
; LA64-FPELIM-NEXT: addi.d $a0, $a1, 15
; LA64-FPELIM-NEXT: bstrins.d $a0, $zero, 3, 0
; LA64-FPELIM-NEXT: sub.d $a0, $sp, $a0
@@ -134,20 +134,20 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
; LA64-FPELIM-NEXT: pcaddu18i $ra, %call36(notdead)
; LA64-FPELIM-NEXT: jirl $ra, $ra, 0
; LA64-FPELIM-NEXT: move $a0, $s0
-; LA64-FPELIM-NEXT: addi.d $sp, $fp, -32
-; LA64-FPELIM-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-FPELIM-NEXT: addi.d $sp, $sp, 96
+; LA64-FPELIM-NEXT: addi.d $sp, $fp, -48
+; LA64-FPELIM-NEXT: ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT: ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-FPELIM-NEXT: addi.d $sp, $sp, 112
; LA64-FPELIM-NEXT: ret
;
; LA64-WITHFP-LABEL: va1_va_arg_alloca:
; LA64-WITHFP: # %bb.0:
-; LA64-WITHFP-NEXT: addi.d $sp, $sp, -96
-; LA64-WITHFP-NEXT: st.d $ra, $sp, 24 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT: st.d $fp, $sp, 16 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT: st.d $s0, $sp, 8 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT: addi.d $fp, $sp, 32
+; LA64-WITHFP-NEXT: addi.d $sp, $sp, -112
+; LA64-WITHFP-NEXT: st.d $ra, $sp, 40 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT: st.d $fp, $sp, 32 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT: st.d $s0, $sp, 24 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT: addi.d $fp, $sp, 48
; LA64-WITHFP-NEXT: move $s0, $a1
; LA64-WITHFP-NEXT: st.d $a7, $fp, 56
; LA64-WITHFP-NEXT: st.d $a6, $fp, 48
@@ -157,7 +157,7 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
; LA64-WITHFP-NEXT: st.d $a2, $fp, 16
; LA64-WITHFP-NEXT: st.d $a1, $fp, 8
; LA64-WITHFP-NEXT: addi.d $a0, $fp, 16
-; LA64-WITHFP-NEXT: st.d $a0, $fp, -32
+; LA64-WITHFP-NEXT: st.d $a0, $fp, -40
; LA64-WITHFP-NEXT: addi.d $a0, $a1, 15
; LA64-WITHFP-NEXT: bstrins.d $a0, $zero, 3, 0
; LA64-WITHFP-NEXT: sub.d $a0, $sp, $a0
@@ -165,11 +165,11 @@ define i64 @va1_va_arg_alloca(ptr %fmt, ...) nounwind {
; LA64-WITHFP-NEXT: pcaddu18i $ra, %call36(notdead)
; LA64-WITHFP-NEXT: jirl $ra, $ra, 0
; LA64-WITHFP-NEXT: move $a0, $s0
-; LA64-WITHFP-NEXT: addi.d $sp, $fp, -32
-; LA64-WITHFP-NEXT: ld.d $s0, $sp, 8 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT: ld.d $fp, $sp, 16 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT: ld.d $ra, $sp, 24 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT: addi.d $sp, $sp, 96
+; LA64-WITHFP-NEXT: addi.d $sp, $fp, -48
+; LA64-WITHFP-NEXT: ld.d $s0, $sp, 24 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT: ld.d $fp, $sp, 32 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT: ld.d $ra, $sp, 40 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT: addi.d $sp, $sp, 112
; LA64-WITHFP-NEXT: ret
%va = alloca ptr, align 8
call void @llvm.va_start(ptr %va)
@@ -314,10 +314,10 @@ define void @va_aligned_stack_caller() nounwind {
;
; LA64-WITHFP-LABEL: va_aligned_stack_caller:
; LA64-WITHFP: # %bb.0:
-; LA64-WITHFP-NEXT: addi.d $sp, $sp, -112
-; LA64-WITHFP-NEXT: st.d $ra, $sp, 104 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT: st.d $fp, $sp, 96 # 8-byte Folded Spill
-; LA64-WITHFP-NEXT: addi.d $fp, $sp, 112
+; LA64-WITHFP-NEXT: addi.d $sp, $sp, -128
+; LA64-WITHFP-NEXT: st.d $ra, $sp, 120 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT: st.d $fp, $sp, 112 # 8-byte Folded Spill
+; LA64-WITHFP-NEXT: addi.d $fp, $sp, 128
; LA64-WITHFP-NEXT: ori $a0, $zero, 17
; LA64-WITHFP-NEXT: st.d $a0, $sp, 48
; LA64-WITHFP-NEXT: ori $a0, $zero, 16
@@ -336,23 +336,23 @@ define void @va_aligned_stack_caller() nounwind {
; LA64-WITHFP-NEXT: lu32i.d $a0, 335544
; LA64-WITHFP-NEXT: lu52i.d $a0, $a0, -328
; LA64-WITHFP-NEXT: st.d $a0, $sp, 16
-; LA64-WITHFP-NEXT: st.d $zero, $fp, -24
+; LA64-WITHFP-NEXT: st.d $zero, $fp, -40
; LA64-WITHFP-NEXT: vrepli.b $vr0, 0
-; LA64-WITHFP-NEXT: vst $vr0, $fp, -40
+; LA64-WITHFP-NEXT: vst $vr0, $fp, -56
; LA64-WITHFP-NEXT: ori $a5, $zero, 1000
; LA64-WITHFP-NEXT: ori $a0, $zero, 1
; LA64-WITHFP-NEXT: ori $a1, $zero, 11
-; LA64-WITHFP-NEXT: addi.d $a2, $fp, -48
+; LA64-WITHFP-NEXT: addi.d $a2, $fp, -64
; LA64-WITHFP-NEXT: ori $a3, $zero, 12
; LA64-WITHFP-NEXT: ori $a4, $zero, 13
; LA64-WITHFP-NEXT: ori $a7, $zero, 1
-; LA64-WITHFP-NEXT: st.d $a5, $fp, -48
+; LA64-WITHFP-NEXT: st.d $a5, $fp, -64
; LA64-WITHFP-NEXT: move $a6, $zero
; LA64-WITHFP-NEXT: pcaddu18i $ra, %call36(va_aligned_stack_callee)
; LA64-WITHFP-NEXT: jirl $ra, $ra, 0
-; LA64-WITHFP-NEXT: ld.d $fp, $sp, 96 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT: ld.d $ra, $sp, 104 # 8-byte Folded Reload
-; LA64-WITHFP-NEXT: addi.d $sp, $sp, 112
+; LA64-WITHFP-NEXT: ld.d $fp, $sp, 112 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT: ld.d $ra, $sp, 120 # 8-byte Folded Reload
+; LA64-WITHFP-NEXT: addi.d $sp, $sp, 128
; LA64-WITHFP-NEXT: ret
%1 = call i32 (i32, ...) @va_aligned_stack_callee(i32 1, i32 11,
i256 1000, i32 12, i32 13, i128 18446744073709551616, i32 14,
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index cbc9f700b1f01..aba20e6b0f27f 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -2311,4 +2311,51 @@ entry:
ret void
}
+define <4 x float> @test_uitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_uitofp_v4i8(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [test_uitofp_v4i8_param_0];
+; CHECK-NEXT: prmt.b32 %r2, %r1, 0, 0x7773U;
+; CHECK-NEXT: cvt.rn.f32.u32 %r3, %r2;
+; CHECK-NEXT: prmt.b32 %r4, %r1, 0, 0x7772U;
+; CHECK-NEXT: cvt.rn.f32.u32 %r5, %r4;
+; CHECK-NEXT: prmt.b32 %r6, %r1, 0, 0x7771U;
+; CHECK-NEXT: cvt.rn.f32.u32 %r7, %r6;
+; CHECK-NEXT: prmt.b32 %r8, %r1, 0, 0x7770U;
+; CHECK-NEXT: cvt.rn.f32.u32 %r9, %r8;
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT: ret;
+ %r = uitofp <4 x i8> %a to <4 x float>
+ ret <4 x float> %r
+}
+
+define <4 x float> @test_sitofp_v4i8(<4 x i8> %a) {
+; CHECK-LABEL: test_sitofp_v4i8(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [test_sitofp_v4i8_param_0];
+; CHECK-NEXT: prmt.b32 %r2, %r1, 0, 0xbbb3U;
+; CHECK-NEXT: cvt.u16.u32 %rs1, %r2;
+; CHECK-NEXT: cvt.rn.f32.s16 %r3, %rs1;
+; CHECK-NEXT: prmt.b32 %r4, %r1, 0, 0xaaa2U;
+; CHECK-NEXT: cvt.u16.u32 %rs2, %r4;
+; CHECK-NEXT: cvt.rn.f32.s16 %r5, %rs2;
+; CHECK-NEXT: prmt.b32 %r6, %r1, 0, 0x9991U;
+; CHECK-NEXT: cvt.u16.u32 %rs3, %r6;
+; CHECK-NEXT: cvt.rn.f32.s16 %r7, %rs3;
+; CHECK-NEXT: prmt.b32 %r8, %r1, 0, 0x8880U;
+; CHECK-NEXT: cvt.u16.u32 %rs4, %r8;
+; CHECK-NEXT: cvt.rn.f32.s16 %r9, %rs4;
+; CHECK-NEXT: st.param.v4.b32 [func_retval0], {%r9, %r7, %r5, %r3};
+; CHECK-NEXT: ret;
+ %r = sitofp <4 x i8> %a to <4 x float>
+ ret <4 x float> %r
+}
+
attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/NVPTX/trunc-tofp.ll b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
new file mode 100644
index 0000000000000..404c423cc026a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/trunc-tofp.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mcpu=sm_80 | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define float @uitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nuw(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [uitofp_trunc_nuw_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [uitofp_trunc_nuw_param_1];
+; CHECK-NEXT: add.s32 %r3, %r1, %r2;
+; CHECK-NEXT: cvt.rn.f32.u32 %r4, %r3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+ %v = add i32 %x, %y
+ %t = trunc nuw i32 %v to i16
+ %f = uitofp i16 %t to float
+ ret float %f
+}
+
+define float @sitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nsw(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sitofp_trunc_nsw_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sitofp_trunc_nsw_param_1];
+; CHECK-NEXT: add.s32 %r3, %r1, %r2;
+; CHECK-NEXT: cvt.rn.f32.s32 %r4, %r3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+ %v = add i32 %x, %y
+ %t = trunc nsw i32 %v to i16
+ %f = sitofp i16 %t to float
+ ret float %f
+}
+
+define float @uitofp_trunc_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: uitofp_trunc_nsw(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [uitofp_trunc_nsw_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [uitofp_trunc_nsw_param_1];
+; CHECK-NEXT: add.s32 %r3, %r1, %r2;
+; CHECK-NEXT: cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT: cvt.rn.f32.u16 %r4, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+ %v = add i32 %x, %y
+ %t = trunc nsw i32 %v to i16
+ %f = uitofp i16 %t to float
+ ret float %f
+}
+
+define float @sitofp_trunc_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: sitofp_trunc_nuw(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<2>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sitofp_trunc_nuw_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sitofp_trunc_nuw_param_1];
+; CHECK-NEXT: add.s32 %r3, %r1, %r2;
+; CHECK-NEXT: cvt.u16.u32 %rs1, %r3;
+; CHECK-NEXT: cvt.rn.f32.s16 %r4, %rs1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+ %v = add i32 %x, %y
+ %t = trunc nuw i32 %v to i16
+ %f = sitofp i16 %t to float
+ ret float %f
+}
diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index 061435c45ad0e..59a702ab6b17f 100644
--- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -798,12 +798,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccor:
; RV64SFBSIFIVEU74: # %bb.0: # %bb
; RV64SFBSIFIVEU74-NEXT: addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: mv s0, a3
+; RV64SFBSIFIVEU74-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: andi a0, a0, 1
; RV64SFBSIFIVEU74-NEXT: mv s1, a2
+; RV64SFBSIFIVEU74-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: beqz a0, .LBB15_4
; RV64SFBSIFIVEU74-NEXT: # %bb.3: # %bb
; RV64SFBSIFIVEU74-NEXT: or s0, a3, a1
@@ -824,11 +824,11 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
; RV64SFBANDESAX45-LABEL: sextw_removal_ccor:
; RV64SFBANDESAX45: # %bb.0: # %bb
; RV64SFBANDESAX45-NEXT: addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: mv s0, a3
+; RV64SFBANDESAX45-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: mv s1, a2
+; RV64SFBANDESAX45-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: nds.bbc a0, 0, .LBB15_2
; RV64SFBANDESAX45-NEXT: # %bb.1:
; RV64SFBANDESAX45-NEXT: or s0, s0, a1
@@ -848,12 +848,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
; ZICOND-LABEL: sextw_removal_ccor:
; ZICOND: # %bb.0: # %bb
; ZICOND-NEXT: addi sp, sp, -32
-; ZICOND-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; ZICOND-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; ZICOND-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; ZICOND-NEXT: mv s0, a3
+; ZICOND-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; ZICOND-NEXT: andi a0, a0, 1
; ZICOND-NEXT: mv s1, a2
+; ZICOND-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; ZICOND-NEXT: beqz a0, .LBB15_4
; ZICOND-NEXT: # %bb.3: # %bb
; ZICOND-NEXT: or s0, a3, a1
@@ -874,12 +874,12 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
; RV32SFB-LABEL: sextw_removal_ccor:
; RV32SFB: # %bb.0: # %bb
; RV32SFB-NEXT: addi sp, sp, -16
-; RV32SFB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: mv s0, a3
+; RV32SFB-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: andi a0, a0, 1
; RV32SFB-NEXT: mv s1, a2
+; RV32SFB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: beqz a0, .LBB15_4
; RV32SFB-NEXT: # %bb.3: # %bb
; RV32SFB-NEXT: or s0, a3, a1
@@ -941,11 +941,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccaddw:
; RV64SFBSIFIVEU74: # %bb.0: # %bb
; RV64SFBSIFIVEU74-NEXT: addi sp, sp, -32
-; RV64SFBSIFIVEU74-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFBSIFIVEU74-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: mv s1, a1
; RV64SFBSIFIVEU74-NEXT: andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBSIFIVEU74-NEXT: mv s0, a2
; RV64SFBSIFIVEU74-NEXT: beqz a0, .LBB16_4
; RV64SFBSIFIVEU74-NEXT: # %bb.3: # %bb
@@ -967,11 +967,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
; RV64SFBANDESAX45-LABEL: sextw_removal_ccaddw:
; RV64SFBANDESAX45: # %bb.0: # %bb
; RV64SFBANDESAX45-NEXT: addi sp, sp, -32
-; RV64SFBANDESAX45-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFBANDESAX45-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: mv s0, a2
+; RV64SFBANDESAX45-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: mv s1, a1
+; RV64SFBANDESAX45-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; RV64SFBANDESAX45-NEXT: nds.bbc a0, 0, .LBB16_2
; RV64SFBANDESAX45-NEXT: # %bb.1:
; RV64SFBANDESAX45-NEXT: addw s1, s1, a3
@@ -991,11 +991,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
; ZICOND-LABEL: sextw_removal_ccaddw:
; ZICOND: # %bb.0: # %bb
; ZICOND-NEXT: addi sp, sp, -32
-; ZICOND-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
-; ZICOND-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
; ZICOND-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
; ZICOND-NEXT: mv s1, a1
; ZICOND-NEXT: andi a0, a0, 1
+; ZICOND-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; ZICOND-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; ZICOND-NEXT: mv s0, a2
; ZICOND-NEXT: beqz a0, .LBB16_4
; ZICOND-NEXT: # %bb.3: # %bb
@@ -1017,11 +1017,11 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
; RV32SFB-LABEL: sextw_removal_ccaddw:
; RV32SFB: # %bb.0: # %bb
; RV32SFB-NEXT: addi sp, sp, -16
-; RV32SFB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
-; RV32SFB-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: sw s1, 4(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: mv s1, a1
; RV32SFB-NEXT: andi a0, a0, 1
+; RV32SFB-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32SFB-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32SFB-NEXT: mv s0, a2
; RV32SFB-NEXT: beqz a0, .LBB16_4
; RV32SFB-NEXT: # %bb.3: # %bb
diff --git a/llvm/test/CodeGen/RISCV/zdinx-spill.ll b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
index d7a700622bf8c..6f206fe571c17 100644
--- a/llvm/test/CodeGen/RISCV/zdinx-spill.ll
+++ b/llvm/test/CodeGen/RISCV/zdinx-spill.ll
@@ -9,7 +9,6 @@ define double @foo(double %x) nounwind {
; CHECK-NEXT: liveins: $x10, $x11, $x8, $x9, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -64
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 64
; CHECK-NEXT: frame-setup SW killed $x8, $x2, 60 :: (store (s32) into %stack.1)
; CHECK-NEXT: frame-setup SW killed $x9, $x2, 56 :: (store (s32) into %stack.2)
; CHECK-NEXT: frame-setup SW killed $x18, $x2, 52 :: (store (s32) into %stack.3)
@@ -22,18 +21,6 @@ define double @foo(double %x) nounwind {
; CHECK-NEXT: frame-setup SW killed $x25, $x2, 24 :: (store (s32) into %stack.10)
; CHECK-NEXT: frame-setup SW killed $x26, $x2, 20 :: (store (s32) into %stack.11)
; CHECK-NEXT: frame-setup SW killed $x27, $x2, 16 :: (store (s32) into %stack.12)
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x8, -4
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x9, -8
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x18, -12
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x19, -16
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x20, -20
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x21, -24
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x22, -28
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x23, -32
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x24, -36
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x25, -40
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x26, -44
- ; CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $x27, -48
; CHECK-NEXT: renamable $x10_x11 = nofpexcept FADD_D_IN32X killed renamable $x10_x11, renamable $x10_x11, 7, implicit $frm
; CHECK-NEXT: PseudoRV32ZdinxSD killed renamable $x10_x11, $x2, 8 :: (store (s64) into %stack.0, align 4)
; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $x6, 12 /* clobber */, implicit-def dead early-clobber $x7, 12 /* clobber */, implicit-def dead early-clobber $x8, 12 /* clobber */, implicit-def dead early-clobber $x9, 12 /* clobber */, implicit-def dead early-clobber $x10, 12 /* clobber */, implicit-def dead early-clobber $x11, 12 /* clobber */, implicit-def dead early-clobber $x12, 12 /* clobber */, implicit-def dead early-clobber $x13, 12 /* clobber */, implicit-def dead early-clobber $x14, 12 /* clobber */, implicit-def dead early-clobber $x15, 12 /* clobber */, implicit-def dead early-clobber $x16, 12 /* clobber */, implicit-def dead early-clobber $x17, 12 /* clobber */, implicit-def dead early-clobber $x18, 12 /* clobber */, implicit-def dead early-clobber $x19, 12 /* clobber */, implicit-def dead early-clobber $x20, 12 /* clobber */, implicit-def dead early-clobber $x21, 12 /* clobber */, implicit-def dead early-clobber $x22, 12 /* clobber */, implicit-def dead early-clobber $x23, 12 /* clobber */, implicit-def dead early-clobber $x24, 12 /* clobber */, implicit-def dead early-clobber $x25, 12 /* clobber */, implicit-def dead early-clobber $x26, 12 /* clobber */, implicit-def dead early-clobber $x27, 12 /* clobber */, implicit-def dead early-clobber $x28, 12 /* clobber */, implicit-def dead early-clobber $x29, 12 /* clobber */, implicit-def dead early-clobber $x31
@@ -50,20 +37,7 @@ define double @foo(double %x) nounwind {
; CHECK-NEXT: $x25 = frame-destroy LW $x2, 24 :: (load (s32) from %stack.10)
; CHECK-NEXT: $x26 = frame-destroy LW $x2, 20 :: (load (s32) from %stack.11)
; CHECK-NEXT: $x27 = frame-destroy LW $x2, 16 :: (load (s32) from %stack.12)
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x8
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x9
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x18
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x19
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x20
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x21
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x22
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x23
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x24
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x25
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x26
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION restore $x27
; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 64
- ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
%a = fadd double %x, %x
call void asm sideeffect "", "~{x6},~{x7},~{x8},~{x9},~{x10},~{x11},~{x12},~{x13},~{x14},~{x15},~{x16},~{x17},~{x18},~{x19},~{x20},~{x21},~{x22},~{x23},~{x24},~{x25},~{x26},~{x27},~{x28},~{x29},~{xr0},~{x31}"()
diff --git a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
index 1d073cdc3bdb9..a878dbe94d11d 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; Test handling of llvm.lifetime intrinsics.
; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-DEFAULT
; RUN: opt < %s -passes=asan -asan-use-after-scope -asan-use-after-return=never -asan-use-stack-safety=0 -asan-instrument-dynamic-allocas=0 -S | FileCheck %s --check-prefixes=CHECK,CHECK-NO-DYNAMIC
@@ -8,109 +9,389 @@ target triple = "x86_64-unknown-linux-gnu"
declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
-; CHECK-LABEL: define void @lifetime_no_size(
define void @lifetime_no_size(i64 %i) sanitize_address {
+; CHECK-LABEL: define void @lifetime_no_size(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT: store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT: store i64 ptrtoint (ptr @lifetime_no_size to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT: store i64 -868083117767659023, ptr [[TMP11]], align 1
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT: [[AI:%.*]] = getelementptr inbounds [2 x i32], ptr [[TMP2]], i64 0, i64 [[I]]
+; CHECK-NEXT: [[TMP12:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT: [[TMP13:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP13]], 2147450880
+; CHECK-NEXT: [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-NEXT: [[TMP16:%.*]] = load i8, ptr [[TMP15]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i8 [[TMP16]], 0
+; CHECK-NEXT: br i1 [[TMP17]], label %[[BB18:.*]], label %[[BB23:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: [[BB18]]:
+; CHECK-NEXT: [[TMP19:%.*]] = and i64 [[TMP12]], 7
+; CHECK-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i8
+; CHECK-NEXT: [[TMP21:%.*]] = icmp sge i8 [[TMP20]], [[TMP16]]
+; CHECK-NEXT: br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB23]]
+; CHECK: [[BB22]]:
+; CHECK-NEXT: call void @__asan_report_store1(i64 [[TMP12]]) #[[ATTR4:[0-9]+]]
+; CHECK-NEXT: unreachable
+; CHECK: [[BB23]]:
+; CHECK-NEXT: store volatile i8 0, ptr [[AI]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP2]])
+; CHECK-NEXT: store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT: [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT: store i64 0, ptr [[TMP25]], align 1
+; CHECK-NEXT: ret void
+;
entry:
%a = alloca [2 x i32], align 4
; Poison memory in prologue: 0xf3f3f300f1f1f1f1
- ; CHECK: store i64 -868083117767659023, ptr %[[#]]
call void @llvm.lifetime.start.p0(i64 -1, ptr %a)
; Check that lifetime with no size are ignored.
- ; CHECK-NOT: store
- ; CHECK: call void @llvm.lifetime.start
%ai = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
store volatile i8 0, ptr %ai, align 4
- ; CHECK: store volatile
call void @llvm.lifetime.end.p0(i64 -1, ptr %a)
; Check that lifetime with no size are ignored.
- ; CHECK-NOT: store
- ; CHECK: call void @llvm.lifetime.end
; Unpoison stack frame on exit.
- ; CHECK: store i64 0, ptr %{{[0-9]+}}
- ; CHECK: ret void
ret void
}
; Generic case of lifetime analysis.
define void @lifetime() sanitize_address {
- ; CHECK-LABEL: define void @lifetime()
+; CHECK-DEFAULT-LABEL: define void @lifetime(
+; CHECK-DEFAULT-SAME: ) #[[ATTR1]] {
+; CHECK-DEFAULT-NEXT: [[TMP1:%.*]] = alloca i64, align 32
+; CHECK-DEFAULT-NEXT: store i64 0, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT: [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-DEFAULT-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-DEFAULT-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 32
+; CHECK-DEFAULT-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-DEFAULT-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-DEFAULT-NEXT: store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT: [[TMP6:%.*]] = add i64 [[TMP2]], 8
+; CHECK-DEFAULT-NEXT: [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-DEFAULT-NEXT: store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP7]], align 8
+; CHECK-DEFAULT-NEXT: [[TMP8:%.*]] = add i64 [[TMP2]], 16
+; CHECK-DEFAULT-NEXT: [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-DEFAULT-NEXT: store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP9]], align 8
+; CHECK-DEFAULT-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP2]], 3
+; CHECK-DEFAULT-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-DEFAULT-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT: [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-DEFAULT-NEXT: store i64 -868082052615769615, ptr [[TMP13]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT: [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
+; CHECK-DEFAULT-NEXT: store i8 4, ptr [[TMP15]], align 1
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT: [[TMP16:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT: [[TMP17:%.*]] = lshr i64 [[TMP16]], 3
+; CHECK-DEFAULT-NEXT: [[TMP18:%.*]] = add i64 [[TMP17]], 2147450880
+; CHECK-DEFAULT-NEXT: [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-DEFAULT-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
+; CHECK-DEFAULT-NEXT: br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB27:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT: [[BB22]]:
+; CHECK-DEFAULT-NEXT: [[TMP23:%.*]] = and i64 [[TMP16]], 7
+; CHECK-DEFAULT-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
+; CHECK-DEFAULT-NEXT: [[TMP25:%.*]] = icmp sge i8 [[TMP24]], [[TMP20]]
+; CHECK-DEFAULT-NEXT: br i1 [[TMP25]], label %[[BB26:.*]], label %[[BB27]]
+; CHECK-DEFAULT: [[BB26]]:
+; CHECK-DEFAULT-NEXT: call void @__asan_report_store1(i64 [[TMP16]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT: unreachable
+; CHECK-DEFAULT: [[BB27]]:
+; CHECK-DEFAULT-NEXT: store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP28:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT: [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-DEFAULT-NEXT: store i8 -8, ptr [[TMP29]], align 1
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT: [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT: [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
+; CHECK-DEFAULT-NEXT: store i8 -8, ptr [[TMP31]], align 1
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT: [[TMP32:%.*]] = alloca i8, i64 128, align 32
+; CHECK-DEFAULT-NEXT: [[TMP33:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT: [[TMP34:%.*]] = add i64 [[TMP33]], 32
+; CHECK-DEFAULT-NEXT: call void @__asan_alloca_poison(i64 [[TMP34]], i64 40)
+; CHECK-DEFAULT-NEXT: [[TMP35:%.*]] = ptrtoint ptr [[TMP32]] to i64
+; CHECK-DEFAULT-NEXT: store i64 [[TMP35]], ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT: [[TMP36:%.*]] = inttoptr i64 [[TMP34]] to ptr
+; CHECK-DEFAULT-NEXT: [[TMP37:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT: call void @__asan_unpoison_stack_memory(i64 [[TMP37]], i64 40)
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.start.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT: [[TMP38:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT: [[TMP39:%.*]] = lshr i64 [[TMP38]], 3
+; CHECK-DEFAULT-NEXT: [[TMP40:%.*]] = add i64 [[TMP39]], 2147450880
+; CHECK-DEFAULT-NEXT: [[TMP41:%.*]] = inttoptr i64 [[TMP40]] to ptr
+; CHECK-DEFAULT-NEXT: [[TMP42:%.*]] = load i8, ptr [[TMP41]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP43:%.*]] = icmp ne i8 [[TMP42]], 0
+; CHECK-DEFAULT-NEXT: br i1 [[TMP43]], label %[[BB44:.*]], label %[[BB49:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT: [[BB44]]:
+; CHECK-DEFAULT-NEXT: [[TMP45:%.*]] = and i64 [[TMP38]], 7
+; CHECK-DEFAULT-NEXT: [[TMP46:%.*]] = trunc i64 [[TMP45]] to i8
+; CHECK-DEFAULT-NEXT: [[TMP47:%.*]] = icmp sge i8 [[TMP46]], [[TMP42]]
+; CHECK-DEFAULT-NEXT: br i1 [[TMP47]], label %[[BB48:.*]], label %[[BB49]]
+; CHECK-DEFAULT: [[BB48]]:
+; CHECK-DEFAULT-NEXT: call void @__asan_report_store1(i64 [[TMP38]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT: unreachable
+; CHECK-DEFAULT: [[BB49]]:
+; CHECK-DEFAULT-NEXT: store volatile i8 0, ptr [[TMP36]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP50:%.*]] = ptrtoint ptr [[TMP36]] to i64
+; CHECK-DEFAULT-NEXT: call void @__asan_poison_stack_memory(i64 [[TMP50]], i64 40)
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.end.p0(i64 40, ptr [[TMP36]])
+; CHECK-DEFAULT-NEXT: [[TMP51:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT: [[TMP52:%.*]] = inttoptr i64 [[TMP51]] to ptr
+; CHECK-DEFAULT-NEXT: store i8 4, ptr [[TMP52]], align 1
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT: [[TMP53:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-DEFAULT-NEXT: [[TMP54:%.*]] = lshr i64 [[TMP53]], 3
+; CHECK-DEFAULT-NEXT: [[TMP55:%.*]] = add i64 [[TMP54]], 2147450880
+; CHECK-DEFAULT-NEXT: [[TMP56:%.*]] = inttoptr i64 [[TMP55]] to ptr
+; CHECK-DEFAULT-NEXT: [[TMP57:%.*]] = load i8, ptr [[TMP56]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP58:%.*]] = icmp ne i8 [[TMP57]], 0
+; CHECK-DEFAULT-NEXT: br i1 [[TMP58]], label %[[BB59:.*]], label %[[BB64:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT: [[BB59]]:
+; CHECK-DEFAULT-NEXT: [[TMP60:%.*]] = and i64 [[TMP53]], 7
+; CHECK-DEFAULT-NEXT: [[TMP61:%.*]] = trunc i64 [[TMP60]] to i8
+; CHECK-DEFAULT-NEXT: [[TMP62:%.*]] = icmp sge i8 [[TMP61]], [[TMP57]]
+; CHECK-DEFAULT-NEXT: br i1 [[TMP62]], label %[[BB63:.*]], label %[[BB64]]
+; CHECK-DEFAULT: [[BB63]]:
+; CHECK-DEFAULT-NEXT: call void @__asan_report_store1(i64 [[TMP53]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT: unreachable
+; CHECK-DEFAULT: [[BB64]]:
+; CHECK-DEFAULT-NEXT: store volatile i8 0, ptr [[TMP4]], align 1
+; CHECK-DEFAULT-NEXT: [[TMP65:%.*]] = add i64 [[TMP11]], 4
+; CHECK-DEFAULT-NEXT: [[TMP66:%.*]] = inttoptr i64 [[TMP65]] to ptr
+; CHECK-DEFAULT-NEXT: store i8 -8, ptr [[TMP66]], align 1
+; CHECK-DEFAULT-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP4]])
+; CHECK-DEFAULT-NEXT: [[TMP67:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-DEFAULT-NEXT: [[TMP68:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-DEFAULT-NEXT: call void @__asan_allocas_unpoison(i64 [[TMP68]], i64 [[TMP67]])
+; CHECK-DEFAULT-NEXT: store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-DEFAULT-NEXT: [[TMP69:%.*]] = add i64 [[TMP11]], 0
+; CHECK-DEFAULT-NEXT: [[TMP70:%.*]] = inttoptr i64 [[TMP69]] to ptr
+; CHECK-DEFAULT-NEXT: store i64 0, ptr [[TMP70]], align 1
+; CHECK-DEFAULT-NEXT: ret void
+;
+; CHECK-NO-DYNAMIC-LABEL: define void @lifetime(
+; CHECK-NO-DYNAMIC-SAME: ) #[[ATTR1]] {
+; CHECK-NO-DYNAMIC-NEXT: [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NO-DYNAMIC-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NO-DYNAMIC-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 32
+; CHECK-NO-DYNAMIC-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i64 1102416563, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP5:%.*]] = add i64 [[TMP1]], 8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP6]], align 8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP7:%.*]] = add i64 [[TMP1]], 16
+; CHECK-NO-DYNAMIC-NEXT: [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP8]], align 8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP9:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NO-DYNAMIC-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT: [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i64 -868082052615769615, ptr [[TMP12]], align 1
+; CHECK-NO-DYNAMIC-NEXT: [[TMP13:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT: [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i8 4, ptr [[TMP14]], align 1
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.start.p0(i64 3, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT: [[TMP15:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP15]], 3
+; CHECK-NO-DYNAMIC-NEXT: [[TMP17:%.*]] = add i64 [[TMP16]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT: [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; CHECK-NO-DYNAMIC-NEXT: [[TMP20:%.*]] = icmp ne i8 [[TMP19]], 0
+; CHECK-NO-DYNAMIC-NEXT: br i1 [[TMP20]], label %[[BB21:.*]], label %[[BB26:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC: [[BB21]]:
+; CHECK-NO-DYNAMIC-NEXT: [[TMP22:%.*]] = and i64 [[TMP15]], 7
+; CHECK-NO-DYNAMIC-NEXT: [[TMP23:%.*]] = trunc i64 [[TMP22]] to i8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP24:%.*]] = icmp sge i8 [[TMP23]], [[TMP19]]
+; CHECK-NO-DYNAMIC-NEXT: br i1 [[TMP24]], label %[[BB25:.*]], label %[[BB26]]
+; CHECK-NO-DYNAMIC: [[BB25]]:
+; CHECK-NO-DYNAMIC-NEXT: call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT: unreachable
+; CHECK-NO-DYNAMIC: [[BB26]]:
+; CHECK-NO-DYNAMIC-NEXT: store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT: [[TMP27:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT: [[TMP28:%.*]] = inttoptr i64 [[TMP27]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i8 -8, ptr [[TMP28]], align 1
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT: [[TMP29:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT: [[TMP30:%.*]] = inttoptr i64 [[TMP29]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i8 -8, ptr [[TMP30]], align 1
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.end.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT: [[ARR:%.*]] = alloca [10 x i32], align 16
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.start.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT: [[TMP31:%.*]] = ptrtoint ptr [[ARR]] to i64
+; CHECK-NO-DYNAMIC-NEXT: [[TMP32:%.*]] = lshr i64 [[TMP31]], 3
+; CHECK-NO-DYNAMIC-NEXT: [[TMP33:%.*]] = add i64 [[TMP32]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT: [[TMP34:%.*]] = inttoptr i64 [[TMP33]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: [[TMP35:%.*]] = load i8, ptr [[TMP34]], align 1
+; CHECK-NO-DYNAMIC-NEXT: [[TMP36:%.*]] = icmp ne i8 [[TMP35]], 0
+; CHECK-NO-DYNAMIC-NEXT: br i1 [[TMP36]], label %[[BB37:.*]], label %[[BB42:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC: [[BB37]]:
+; CHECK-NO-DYNAMIC-NEXT: [[TMP38:%.*]] = and i64 [[TMP31]], 7
+; CHECK-NO-DYNAMIC-NEXT: [[TMP39:%.*]] = trunc i64 [[TMP38]] to i8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP40:%.*]] = icmp sge i8 [[TMP39]], [[TMP35]]
+; CHECK-NO-DYNAMIC-NEXT: br i1 [[TMP40]], label %[[BB41:.*]], label %[[BB42]]
+; CHECK-NO-DYNAMIC: [[BB41]]:
+; CHECK-NO-DYNAMIC-NEXT: call void @__asan_report_store1(i64 [[TMP31]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT: unreachable
+; CHECK-NO-DYNAMIC: [[BB42]]:
+; CHECK-NO-DYNAMIC-NEXT: store volatile i8 0, ptr [[ARR]], align 1
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.end.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT: [[TMP43:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT: [[TMP44:%.*]] = inttoptr i64 [[TMP43]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i8 4, ptr [[TMP44]], align 1
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.start.p0(i64 2, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT: [[TMP45:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NO-DYNAMIC-NEXT: [[TMP46:%.*]] = lshr i64 [[TMP45]], 3
+; CHECK-NO-DYNAMIC-NEXT: [[TMP47:%.*]] = add i64 [[TMP46]], 2147450880
+; CHECK-NO-DYNAMIC-NEXT: [[TMP48:%.*]] = inttoptr i64 [[TMP47]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: [[TMP49:%.*]] = load i8, ptr [[TMP48]], align 1
+; CHECK-NO-DYNAMIC-NEXT: [[TMP50:%.*]] = icmp ne i8 [[TMP49]], 0
+; CHECK-NO-DYNAMIC-NEXT: br i1 [[TMP50]], label %[[BB51:.*]], label %[[BB56:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC: [[BB51]]:
+; CHECK-NO-DYNAMIC-NEXT: [[TMP52:%.*]] = and i64 [[TMP45]], 7
+; CHECK-NO-DYNAMIC-NEXT: [[TMP53:%.*]] = trunc i64 [[TMP52]] to i8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP54:%.*]] = icmp sge i8 [[TMP53]], [[TMP49]]
+; CHECK-NO-DYNAMIC-NEXT: br i1 [[TMP54]], label %[[BB55:.*]], label %[[BB56]]
+; CHECK-NO-DYNAMIC: [[BB55]]:
+; CHECK-NO-DYNAMIC-NEXT: call void @__asan_report_store1(i64 [[TMP45]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT: unreachable
+; CHECK-NO-DYNAMIC: [[BB56]]:
+; CHECK-NO-DYNAMIC-NEXT: store volatile i8 0, ptr [[TMP3]], align 1
+; CHECK-NO-DYNAMIC-NEXT: [[TMP57:%.*]] = add i64 [[TMP10]], 4
+; CHECK-NO-DYNAMIC-NEXT: [[TMP58:%.*]] = inttoptr i64 [[TMP57]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i8 -8, ptr [[TMP58]], align 1
+; CHECK-NO-DYNAMIC-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP3]])
+; CHECK-NO-DYNAMIC-NEXT: store i64 1172321806, ptr [[TMP4]], align 8
+; CHECK-NO-DYNAMIC-NEXT: [[TMP59:%.*]] = add i64 [[TMP10]], 0
+; CHECK-NO-DYNAMIC-NEXT: [[TMP60:%.*]] = inttoptr i64 [[TMP59]] to ptr
+; CHECK-NO-DYNAMIC-NEXT: store i64 0, ptr [[TMP60]], align 1
+; CHECK-NO-DYNAMIC-NEXT: ret void
+;
; Regular variable lifetime intrinsics.
%i = alloca i32, align 4
; Poison memory in prologue: F1F1F1F1F8F3F3F3
- ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
; Memory is unpoisoned at llvm.lifetime.start
call void @llvm.lifetime.start.p0(i64 3, ptr %i)
- ; CHECK: store i8 4, ptr %{{[0-9]+}}
- ; CHECK-NEXT: llvm.lifetime.start
store volatile i8 0, ptr %i
- ; CHECK: store volatile
call void @llvm.lifetime.end.p0(i64 4, ptr %i)
- ; CHECK: store i8 -8, ptr %{{[0-9]+}}
- ; CHECK-NEXT: call void @llvm.lifetime.end
; Memory is poisoned at every call to llvm.lifetime.end
call void @llvm.lifetime.end.p0(i64 2, ptr %i)
- ; CHECK: store i8 -8, ptr %{{[0-9]+}}
- ; CHECK-NEXT: call void @llvm.lifetime.end
; Lifetime intrinsics for array.
%arr = alloca [10 x i32], align 16
call void @llvm.lifetime.start.p0(i64 40, ptr %arr)
- ; CHECK-DEFAULT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
- ; CHECK-NO-DYNAMIC-NOT: call void @__asan_unpoison_stack_memory(i64 %{{[^ ]+}}, i64 40)
store volatile i8 0, ptr %arr
- ; CHECK: store volatile
call void @llvm.lifetime.end.p0(i64 40, ptr %arr)
- ; CHECK-DEFAULT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
- ; CHECK-NO-DYNAMIC-NOT: call void @__asan_poison_stack_memory(i64 %{{[^ ]+}}, i64 40)
; One more lifetime start/end for the same variable %i.
call void @llvm.lifetime.start.p0(i64 2, ptr %i)
- ; CHECK: store i8 4, ptr %{{[0-9]+}}
- ; CHECK-NEXT: llvm.lifetime.start
store volatile i8 0, ptr %i
- ; CHECK: store volatile
call void @llvm.lifetime.end.p0(i64 4, ptr %i)
- ; CHECK: store i8 -8, ptr %{{[0-9]+}}
- ; CHECK-NEXT: llvm.lifetime.end
; Memory is unpoisoned at function exit (only once).
- ; CHECK: store i64 0, ptr %{{[0-9]+}}
- ; CHECK-NEXT: ret void
ret void
}
; Check that arguments of lifetime may come from phi nodes.
define void @phi_args(i1 %x) sanitize_address {
- ; CHECK-LABEL: define void @phi_args(i1 %x)
+; CHECK-LABEL: define void @phi_args(
+; CHECK-SAME: i1 [[X:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: store i64 1102416563, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT: store i64 ptrtoint (ptr @___asan_gen_stack.2 to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT: store i64 ptrtoint (ptr @phi_args to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT: store i64 -868082052615769615, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT: store i8 0, ptr [[TMP13]], align 1
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP2]])
+; CHECK-NEXT: [[TMP14:%.*]] = ptrtoint ptr [[TMP2]] to i64
+; CHECK-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP14]], 3
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP15]], 2147450880
+; CHECK-NEXT: [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, ptr [[TMP17]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = icmp ne i8 [[TMP18]], 0
+; CHECK-NEXT: br i1 [[TMP19]], label %[[BB20:.*]], label %[[BB25:.*]], !prof [[PROF1]]
+; CHECK: [[BB20]]:
+; CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP14]], 7
+; CHECK-NEXT: [[TMP22:%.*]] = trunc i64 [[TMP21]] to i8
+; CHECK-NEXT: [[TMP23:%.*]] = icmp sge i8 [[TMP22]], [[TMP18]]
+; CHECK-NEXT: br i1 [[TMP23]], label %[[BB24:.*]], label %[[BB25]]
+; CHECK: [[BB24]]:
+; CHECK-NEXT: call void @__asan_report_store1(i64 [[TMP14]]) #[[ATTR4]]
+; CHECK-NEXT: unreachable
+; CHECK: [[BB25]]:
+; CHECK-NEXT: store volatile i8 0, ptr [[TMP2]], align 1
+; CHECK-NEXT: br i1 [[X]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK: [[BB0]]:
+; CHECK-NEXT: br label %[[BB1]]
+; CHECK: [[BB1]]:
+; CHECK-NEXT: [[I_PHI:%.*]] = phi ptr [ [[TMP2]], %[[BB25]] ], [ [[TMP2]], %[[BB0]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[TMP9]], 4
+; CHECK-NEXT: [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
+; CHECK-NEXT: store i8 -8, ptr [[TMP27]], align 1
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[I_PHI]])
+; CHECK-NEXT: store i64 1172321806, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP28:%.*]] = add i64 [[TMP9]], 0
+; CHECK-NEXT: [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
+; CHECK-NEXT: store i64 0, ptr [[TMP29]], align 1
+; CHECK-NEXT: ret void
+;
entry:
%i = alloca i64, align 4
; Poison memory in prologue: F1F1F1F1F8F3F3F3
- ; CHECK: store i64 -868082052615769615, ptr %{{[0-9]+}}
call void @llvm.lifetime.start.p0(i64 8, ptr %i)
- ; CHECK: store i8 0, ptr %{{[0-9]+}}
- ; CHECK-NEXT: llvm.lifetime.start
store volatile i8 0, ptr %i
- ; CHECK: store volatile
br i1 %x, label %bb0, label %bb1
@@ -120,49 +401,101 @@ bb0:
bb1:
%i.phi = phi ptr [ %i, %entry ], [ %i, %bb0 ]
call void @llvm.lifetime.end.p0(i64 8, ptr %i.phi)
- ; CHECK: store i8 -8, ptr %{{[0-9]+}}
- ; CHECK-NEXT: llvm.lifetime.end
ret void
- ; CHECK: store i64 0, ptr %{{[0-9]+}}
- ; CHECK-NEXT: ret void
}
; Check that arguments of lifetime may come from getelementptr nodes.
define void @getelementptr_args(i64 %i) sanitize_address{
- ; CHECK-LABEL: define void @getelementptr_args
+; CHECK-LABEL: define void @getelementptr_args(
+; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MYALLOCA:%.*]] = alloca i8, i64 1216, align 32
+; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP0]], 1184
+; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i64 [[TMP0]] to ptr
+; CHECK-NEXT: store i64 1102416563, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT: [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+; CHECK-NEXT: store i64 ptrtoint (ptr @___asan_gen_stack.3 to i64), ptr [[TMP7]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[TMP0]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT: store i64 ptrtoint (ptr @getelementptr_args to i64), ptr [[TMP9]], align 8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP0]], 3
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 2147450880
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT: [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr
+; CHECK-NEXT: store i32 -235802127, ptr [[TMP13]], align 1
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT: call void @__asan_set_shadow_f8(i64 [[TMP14]], i64 128)
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP11]], 132
+; CHECK-NEXT: [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
+; CHECK-NEXT: store i64 -940422246894996750, ptr [[TMP16]], align 1
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[TMP11]], 140
+; CHECK-NEXT: [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; CHECK-NEXT: store i64 -940422246894996750, ptr [[TMP18]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT: [[TMP20:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT: store i16 -3085, ptr [[TMP20]], align 1
+; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT: call void @__asan_set_shadow_00(i64 [[TMP21]], i64 128)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT: [[AI:%.*]] = getelementptr inbounds [2 x ptr], ptr [[TMP4]], i64 0, i64 [[I]]
+; CHECK-NEXT: [[TMP22:%.*]] = ptrtoint ptr [[AI]] to i64
+; CHECK-NEXT: [[TMP23:%.*]] = lshr i64 [[TMP22]], 3
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], 2147450880
+; CHECK-NEXT: [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
+; CHECK-NEXT: [[TMP26:%.*]] = load i8, ptr [[TMP25]], align 1
+; CHECK-NEXT: [[TMP27:%.*]] = icmp ne i8 [[TMP26]], 0
+; CHECK-NEXT: br i1 [[TMP27]], label %[[BB28:.*]], label %[[BB29:.*]]
+; CHECK: [[BB28]]:
+; CHECK-NEXT: call void @__asan_report_store8(i64 [[TMP22]]) #[[ATTR4]]
+; CHECK-NEXT: unreachable
+; CHECK: [[BB29]]:
+; CHECK-NEXT: store ptr [[TMP2]], ptr [[AI]], align 8
+; CHECK-NEXT: [[TMP30:%.*]] = add i64 [[TMP11]], 4
+; CHECK-NEXT: call void @__asan_set_shadow_f8(i64 [[TMP30]], i64 128)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1024, ptr [[TMP2]])
+; CHECK-NEXT: store i64 1172321806, ptr [[TMP5]], align 8
+; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP11]], 0
+; CHECK-NEXT: call void @__asan_set_shadow_00(i64 [[TMP31]], i64 148)
+; CHECK-NEXT: [[TMP32:%.*]] = add i64 [[TMP11]], 150
+; CHECK-NEXT: [[TMP33:%.*]] = inttoptr i64 [[TMP32]] to ptr
+; CHECK-NEXT: store i16 0, ptr [[TMP33]], align 1
+; CHECK-NEXT: ret void
+;
entry:
%x = alloca [1024 x i8], align 16
%a = alloca [2 x ptr], align 8
; F1F1F1F1
- ; CHECK: store i32 -235802127, ptr %{{[0-9]+}}
- ; CHECK: call void @__asan_set_shadow_f8(i64 %[[#]], i64 128)
; 0xf2f2f2f2f2f2f2f2
- ; CHECK: store i64 -940422246894996750, ptr %[[#]]
; 0xf2f2f2f2f2f2f2f2
- ; CHECK: store i64 -940422246894996750, ptr %[[#]]
call void @llvm.lifetime.start.p0(i64 1024, ptr %x)
- ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 128)
- ; CHECK-NEXT: call void @llvm.lifetime.start
%ai = getelementptr inbounds [2 x ptr], ptr %a, i64 0, i64 %i
store ptr %x, ptr %ai, align 8
- ; CHECK: store ptr
call void @llvm.lifetime.end.p0(i64 1024, ptr %x)
- ; CHECK: call void @__asan_set_shadow_f8(i64 %{{[0-9]+}}, i64 128)
- ; CHECK-NEXT: call void @llvm.lifetime.end
ret void
- ; CHECK: call void @__asan_set_shadow_00(i64 %{{[0-9]+}}, i64 148)
- ; CHECK: store i16 0, ptr %[[#]], align 1
- ; CHECK-NEXT: ret void
}
define void @zero_sized(i64 %a) #0 {
-; CHECK-LABEL: define void @zero_sized(i64 %a)
+; CHECK-LABEL: define void @zero_sized(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_ADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT: [[B:%.*]] = alloca [0 x i8], align 1
+; CHECK-NEXT: store i64 [[A]], ptr [[A_ADDR]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 0, ptr [[B]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 0, ptr [[B]])
+; CHECK-NEXT: ret void
+;
entry:
%a.addr = alloca i64, align 8
@@ -170,11 +503,13 @@ entry:
store i64 %a, ptr %a.addr, align 8
call void @llvm.lifetime.start.p0(i64 0, ptr %b) #2
- ; CHECK: call void @llvm.lifetime.start
call void @llvm.lifetime.end.p0(i64 0, ptr %b) #2
- ; CHECK: call void @llvm.lifetime.end
ret void
- ; CHECK-NEXT: ret void
}
+;.
+; CHECK-DEFAULT: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
+; CHECK-NO-DYNAMIC: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
index 279bb262bff04..5f310a9954ad0 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
@@ -163,6 +163,51 @@ v_tanh_bf16 v5, src_scc
v_tanh_bf16 v127, 0x8000
// GFX1250: v_tanh_bf16_e32 v127, 0x8000 ; encoding: [0xff,0x94,0xfe,0x7e,0x00,0x80,0x00,0x00]
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1 ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1 ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0 ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1 ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5 ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
v_rcp_bf16 v5, v1
// GFX1250: v_rcp_bf16_e32 v5, v1 ; encoding: [0x01,0xf3,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
index 76272d25d92d4..aa2e028f661e1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
@@ -169,6 +169,51 @@ v_tanh_bf16 v127, 0x8000
v_tanh_bf16 v5.h, v1.h
// GFX1250: v_tanh_bf16_e32 v5.h, v1.h ; encoding: [0x81,0x95,0x0a,0x7f]
+v_prng_b32 v5, v1
+// GFX1250: v_prng_b32_e32 v5, v1 ; encoding: [0x01,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, v255
+// GFX1250: v_prng_b32_e32 v5, v255 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+v_prng_b32 v5, s1
+// GFX1250: v_prng_b32_e32 v5, s1 ; encoding: [0x01,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, s105
+// GFX1250: v_prng_b32_e32 v5, s105 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_lo
+// GFX1250: v_prng_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, vcc_hi
+// GFX1250: v_prng_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, ttmp15
+// GFX1250: v_prng_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, m0
+// GFX1250: v_prng_b32_e32 v5, m0 ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_lo
+// GFX1250: v_prng_b32_e32 v5, exec_lo ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, exec_hi
+// GFX1250: v_prng_b32_e32 v5, exec_hi ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, null
+// GFX1250: v_prng_b32_e32 v5, null ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, -1
+// GFX1250: v_prng_b32_e32 v5, -1 ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, 0.5
+// GFX1250: v_prng_b32_e32 v5, 0.5 ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+v_prng_b32 v5, src_scc
+// GFX1250: v_prng_b32_e32 v5, src_scc ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+v_prng_b32 v255, 0xaf123456
+// GFX1250: v_prng_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
v_rcp_bf16 v5, v1
// GFX1250: v_rcp_bf16_e32 v5, v1 ; encoding: [0x01,0xf3,0x0a,0x7e]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
index 0a8ee84561d33..e1cd2e3043693 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
@@ -170,6 +170,58 @@ v_tanh_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 f
// GFX1250: v_tanh_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0x94,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
// GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
index d4afb9d9b2d9a..c1d3238b65cbd 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
@@ -178,6 +178,58 @@ v_tanh_bf16 v5.h, v1.h quad_perm:[3,2,1,0]
// GFX1250: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shl:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_shr:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:1
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_ror:15
+// GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16 v5, v1 quad_perm:[3,2,1,0]
// GFX1250: v_rcp_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf2,0x0a,0x7e,0x01,0x1b,0x00,0xff]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
index a7cb6bf8de69c..100e9f92ff58b 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
@@ -38,6 +38,18 @@ v_tanh_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
// GFX1250: v_tanh_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x94,0xfe,0x7e,0x7f,0x00,0x00,0x00]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
// GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
index 6acab7edc0d49..2ae103545443c 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
@@ -46,6 +46,18 @@ v_tanh_bf16 v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0]
// GFX1250: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32 v255, v255 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
// GFX1250: v_rcp_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf2,0x0a,0x7e,0x01,0x77,0x39,0x05]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
index 7486d849253e8..9c6a9127d82e4 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
@@ -127,6 +127,42 @@ v_cvt_f32_fp8 v1, v3 byte_sel:1 clamp
v_cvt_f32_fp8 v1, v3 byte_sel:2 clamp
// GFX1250: v_cvt_f32_fp8_e64 v1, v3 byte_sel:2 clamp ; encoding: [0x01,0x88,0xec,0xd5,0x03,0x01,0x00,0x00]
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1 ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1 ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0 ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1 ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
v_tanh_f32_e64 v5, v1
// GFX1250: v_tanh_f32_e64 v5, v1 ; encoding: [0x05,0x00,0x9e,0xd5,0x01,0x01,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
index b59b8b31e2d5f..2f57d1c331c42 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
@@ -223,6 +223,42 @@ v_tanh_f16_e64 v255, -|0x8000| clamp div:2
v_tanh_f16 v5.l, v128.h
// GFX1250: v_tanh_f16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0x9f,0xd5,0x80,0x01,0x00,0x00]
+v_prng_b32_e64 v5, v1
+// GFX1250: v_prng_b32_e64 v5, v1 ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, v255
+// GFX1250: v_prng_b32_e64 v5, v255 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+v_prng_b32_e64 v5, s1
+// GFX1250: v_prng_b32_e64 v5, s1 ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, s105
+// GFX1250: v_prng_b32_e64 v5, s105 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_lo
+// GFX1250: v_prng_b32_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, vcc_hi
+// GFX1250: v_prng_b32_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, ttmp15
+// GFX1250: v_prng_b32_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, m0
+// GFX1250: v_prng_b32_e64 v5, m0 ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_lo
+// GFX1250: v_prng_b32_e64 v5, exec_lo ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, exec_hi
+// GFX1250: v_prng_b32_e64 v5, exec_hi ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, null
+// GFX1250: v_prng_b32_e64 v5, null ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+v_prng_b32_e64 v5, -1
+// GFX1250: v_prng_b32_e64 v5, -1 ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
v_rcp_bf16_e64 v5, v1
// GFX1250: v_rcp_bf16_e64 v5, v1 ; encoding: [0x05,0x00,0xf9,0xd5,0x01,0x01,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
index f7f20f46161ce..29bb842b529b7 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
@@ -170,6 +170,50 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mas
// GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xca,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
// GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
index e1241b01ccae1..7df92751c38d1 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
@@ -178,6 +178,50 @@ v_tanh_bf16_e64_dpp v5.h, v128.h quad_perm:[3,2,1,0]
// GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0xca,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
// GFX1250: v_rcp_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf9,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
index 0106175301d20..d235aeb9f3e62 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
@@ -50,6 +50,10 @@ v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
// GFX1250: v_tanh_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xca,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
// GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
index 93b86f3ffb841..f25e2a5882436 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
@@ -58,6 +58,10 @@ v_tanh_bf16_e64_dpp v5.h, v128.h dpp8:[7,6,5,4,3,2,1,0]
// GFX1250: v_tanh_bf16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0xca,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
// GFX1250: v_rcp_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf9,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
index 5f37ba91e071b..aa968b2bb2bee 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
@@ -200,6 +200,51 @@
0x81,0x95,0x0a,0x7f
# GFX1250-REAL16: v_tanh_bf16_e32 v5.h, v1.h ; encoding: [0x81,0x95,0x0a,0x7f]
+0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf
+# GFX1250: v_prng_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x96,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+
+0xc1,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, -1 ; encoding: [0xc1,0x96,0x0a,0x7e]
+
+0xf0,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, 0.5 ; encoding: [0xf0,0x96,0x0a,0x7e]
+
+0x7f,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_hi ; encoding: [0x7f,0x96,0x0a,0x7e]
+
+0x7e,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, exec_lo ; encoding: [0x7e,0x96,0x0a,0x7e]
+
+0x7d,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, m0 ; encoding: [0x7d,0x96,0x0a,0x7e]
+
+0x7c,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, null ; encoding: [0x7c,0x96,0x0a,0x7e]
+
+0x01,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s1 ; encoding: [0x01,0x96,0x0a,0x7e]
+
+0x69,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, s105 ; encoding: [0x69,0x96,0x0a,0x7e]
+
+0xfd,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, src_scc ; encoding: [0xfd,0x96,0x0a,0x7e]
+
+0x7b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x96,0x0a,0x7e]
+
+0x01,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v1 ; encoding: [0x01,0x97,0x0a,0x7e]
+
+0xff,0x97,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, v255 ; encoding: [0xff,0x97,0x0a,0x7e]
+
+0x6b,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x96,0x0a,0x7e]
+
+0x6a,0x96,0x0a,0x7e
+# GFX1250: v_prng_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x96,0x0a,0x7e]
+
0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00
# GFX1250-REAL16: v_rcp_bf16_e32 v127.l, 0x8000 ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
# GFX1250-FAKE16: v_rcp_bf16_e32 v127, 0x8000 ; encoding: [0xff,0xf2,0xfe,0x7e,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
index 57bee2766ce44..913a2a916ff62 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
@@ -162,6 +162,45 @@
0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff
# GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x94,0x0a,0x7f,0x81,0x1b,0x00,0xff]
+0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x41,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x40,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x21,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x50,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01
+# GFX1250: v_prng_b32_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x01,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x11,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+
+0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13
+# GFX1250: v_prng_b32_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0x96,0x0a,0x7e,0x01,0x60,0x09,0x13]
+
0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30
# GFX1250-REAL16: v_rcp_bf16_dpp v127.l, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
# GFX1250-FAKE16: v_rcp_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xf2,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
index 28ec6b11b4de3..4afe44e241bf3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
@@ -43,6 +43,15 @@
# GFX1250-REAL16: v_tanh_bf16_dpp v5.h, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x94,0x0a,0x7f,0x81,0x77,0x39,0x05]
# GFX1250-FAKE16: v_add_f64_e32 v[156:157], v[129:130], v[187:188] ; encoding: [0x81,0x77,0x39,0x05]
+0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00
+# GFX1250: v_prng_b32_dpp v255, v255 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0x96,0xfe,0x7f,0xff,0x00,0x00,0x00]
+
+0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0x96,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00
# GFX1250-REAL16: v_rcp_bf16_dpp v127.l, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
# GFX1250-FAKE16: v_rcp_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xf2,0xfe,0x7e,0x7f,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
index 5004762729701..1cf3b8807d044 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
@@ -175,6 +175,42 @@
# GFX1250-REAL16: v_tanh_bf16_e64 v5.l, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xca,0xd5,0x80,0x01,0x00,0x00]
# GFX1250-FAKE16: v_tanh_bf16_e64 v5, v128 ; encoding: [0x05,0x00,0xca,0xd5,0x80,0x01,0x00,0x00]
+0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, -1 ; encoding: [0x05,0x00,0xcb,0xd5,0xc1,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_hi ; encoding: [0x05,0x00,0xcb,0xd5,0x7f,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, exec_lo ; encoding: [0x05,0x00,0xcb,0xd5,0x7e,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, m0 ; encoding: [0x05,0x00,0xcb,0xd5,0x7d,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, null ; encoding: [0x05,0x00,0xcb,0xd5,0x7c,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s1 ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, s105 ; encoding: [0x05,0x00,0xcb,0xd5,0x69,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, ttmp15 ; encoding: [0x05,0x00,0xcb,0xd5,0x7b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v1 ; encoding: [0x05,0x00,0xcb,0xd5,0x01,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, v255 ; encoding: [0x05,0x00,0xcb,0xd5,0xff,0x01,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_hi ; encoding: [0x05,0x00,0xcb,0xd5,0x6b,0x00,0x00,0x00]
+
+0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00
+# GFX1250: v_prng_b32_e64 v5, vcc_lo ; encoding: [0x05,0x00,0xcb,0xd5,0x6a,0x00,0x00,0x00]
+
0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00
# GFX1250-REAL16: v_rcp_bf16_e64 v255.l, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
# GFX1250-FAKE16: v_rcp_bf16_e64 v255, -|0x8000| clamp div:2 ; encoding: [0xff,0x81,0xf9,0xd5,0xff,0x00,0x00,0x38,0x00,0x80,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
index de908b95d94f9..83a647ad7c658 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
@@ -104,6 +104,39 @@
# GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x48,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
# GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x9f,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+
+0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff
+# GFX1250: v_prng_b32_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xcb,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+
0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30
# GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
# GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x81,0xfb,0xd5,0xfa,0x00,0x00,0x38,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
index cfe7173c383b3..ef5ede4d1d453 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
@@ -1,6 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
# GFX1250: v_tanh_f32_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0x9e,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
@@ -34,6 +34,9 @@
# GFX1250-REAL16: v_tanh_f16_e64_dpp v5.h, v128.h op_sel:[1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x48,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
# GFX1250-FAKE16: v_tanh_f16_e64_dpp v5, v128 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x9f,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_prng_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xcb,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00
# GFX1250-REAL16: v_rsq_bf16_e64_dpp v255.l, -|v255.l| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
# GFX1250-FAKE16: v_rsq_bf16_e64_dpp v255, -|v255| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x81,0xfb,0xd5,0xe9,0x00,0x00,0x38,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
index e390d4bdca632..303afc207c023 100644
--- a/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/zeroed-missing.ll
@@ -12,6 +12,6 @@ define ptr @undeclared_customalloc(i64 %size, i64 %align) {
ret ptr %call
}
-declare ptr @customalloc2(i64, i64) allockind("alloc") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
+declare ptr @customalloc2(i64, i64) allockind("alloc,uninitialized") "alloc-family"="customalloc2" "alloc-variant-zeroed"="customalloc2_zeroed"
; CHECK-DAG: declare ptr @customalloc2_zeroed(i64, i64) #[[CA2ATTR:[0-9]+]]
; CHECK-DAG: attributes #[[CA2ATTR]] = { allockind("alloc,zeroed") "alloc-family"="customalloc2" }
diff --git a/llvm/test/Transforms/LoopInterchange/force-interchange.ll b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
new file mode 100644
index 0000000000000..c33ecdf7d9905
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/force-interchange.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t -disable-output -loop-interchange-profitabilities=ignore -S
+; RUN: FileCheck --input-file=%t %s
+
+; There should be no reason to interchange this, unless it is forced.
+;
+; for (int i = 0; i<1024; i++)
+; for (int j = 0; j<1024; j++)
+; A[i][j] = 42;
+;
+; CHECK: --- !Passed
+; CHECK-NEXT: Pass: loop-interchange
+; CHECK-NEXT: Name: Interchanged
+; CHECK-NEXT: Function: f
+; CHECK-NEXT: Args:
+; CHECK-NEXT: - String: Loop interchanged with enclosing loop.
+; CHECK-NEXT: ...
+
+ at A = dso_local local_unnamed_addr global [1024 x [1024 x i32]] zeroinitializer, align 4
+
+define dso_local void @f() local_unnamed_addr #0 {
+entry:
+ br label %outer.header
+
+outer.header:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %inner.header ]
+ br label %inner.body
+
+inner.header:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond20.not = icmp eq i64 %i.next, 1024
+ br i1 %exitcond20.not, label %exit, label %outer.header
+
+inner.body:
+ %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
+ %arrayidx6 = getelementptr inbounds nuw [1024 x [1024 x i32]], ptr @A, i64 0, i64 %i, i64 %j
+ store i32 42, ptr %arrayidx6, align 4
+ %j.next = add nuw nsw i64 %j, 1
+ %exitcond.not = icmp eq i64 %j.next, 1024
+ br i1 %exitcond.not, label %inner.header, label %inner.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
new file mode 100644
index 0000000000000..c7a0de22b200b
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=loop-reduce -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %p, i64 %idx) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [4 x [4 x i32]], align 16
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[IDX]], 6
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 48
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr nuw i8, ptr [[ALLOCA]], i64 48
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], %[[LOOP]] ], [ -8, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = shl nsw i64 [[LSR_IV]], 2
+; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; CHECK-NEXT: [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[SCEVGEP8]], i64 32
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[SCEVGEP9]], align 4
+; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[LSR_IV]]
+; CHECK-NEXT: [[SCEVGEP7:%.*]] = getelementptr i8, ptr [[SCEVGEP6]], i64 8
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[SCEVGEP7]], align 4
+; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[LSR_IV]]
+; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 8
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[SCEVGEP5]], align 4
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[LSR_IV]]
+; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 8
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[SCEVGEP2]], align 4
+; CHECK-NEXT: [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 4
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %alloca = alloca [4 x [4 x i32]], align 16
+ call void @llvm.lifetime.start.p0(i64 64, ptr %alloca)
+ br label %loop
+
+loop:
+ %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ]
+ %gep1 = getelementptr [4 x [12 x [4 x [4 x i32]]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv, i64 0
+ %0 = load i32, ptr %gep1, align 4
+ %gep2 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 0, i64 0, i64 %indvars.iv
+ %1 = load i32, ptr %gep2, align 4
+ %gep3 = getelementptr [4 x [4 x i32]], ptr %alloca, i64 0, i64 3, i64 %indvars.iv
+ %2 = load i32, ptr %gep3, align 4
+ %gep4 = getelementptr [6 x [4 x [4 x i32]]], ptr %p, i64 0, i64 %idx, i64 3, i64 %indvars.iv
+ %3 = load i32, ptr %gep4, align 4
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv, 1
+ br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+ call void @llvm.lifetime.end.p0(i64 64, ptr %alloca)
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
new file mode 100644
index 0000000000000..2d1543185098f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/maxbandwidth-regpressure.ll
@@ -0,0 +1,38 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
+; RUN: opt -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -force-target-num-vector-regs=1 -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NOREGS-VP
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @dotp(ptr %a, ptr %b) #0 {
+; CHECK-REGS-VP-NOT: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-REGS-VP: LV: Selecting VF: vscale x 8.
+;
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 8 because it uses too many registers
+; CHECK-NOREGS-VP: LV(REG): Not considering vector loop of width vscale x 16 because it uses too many registers
+; CHECK-NOREGS-VP: LV: Selecting VF: vscale x 4.
+entry:
+ br label %for.body
+
+for.body: ; preds = %for.body, %entry
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+ %gep.a = getelementptr i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %gep.a, align 1
+ %ext.a = zext i8 %load.a to i32
+ %gep.b = getelementptr i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %gep.b, align 1
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul i32 %ext.b, %ext.a
+ %sub = sub i32 0, %mul
+ %add = add i32 %accum, %sub
+ %iv.next = add i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit: ; preds = %for.body
+ ret i32 %add
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
index 8e09847e9264e..cfff11758a37a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reorder-node-with-ops.ll
@@ -58,7 +58,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
; CHECK-NEXT: br label %[[BB54:.*]]
; CHECK: [[BB54]]:
; CHECK-NEXT: [[TMP54:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP17]])
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 0, ptr null)
; CHECK-NEXT: [[TMP55:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP21]])
; CHECK-NEXT: [[TMP56:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <8 x float> [[TMP56]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison>
@@ -198,7 +197,6 @@ define void @test(i32 %0, i8 %1, i64 %2, float %3) {
%95 = or i64 %94, %91
%96 = or i64 %95, %37
store i64 %96, ptr null, align 1
- call void @llvm.lifetime.start.p0(i64 0, ptr null)
store i64 %42, ptr null, align 1
%97 = bitcast float %3 to i32
%98 = icmp ult i32 %97, 1325400064
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index 422eb855ba2cf..2b43d27f292a0 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -131,20 +131,6 @@ class CommentWriter : public AssemblyAnnotationWriter {
printDebugLoc(DL,OS);
OS << "]";
}
- if (const DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
- if (!Padded) {
- OS.PadToColumn(50);
- OS << ";";
- }
- OS << " [debug variable = " << DDI->getVariable()->getName() << "]";
- }
- else if (const DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
- if (!Padded) {
- OS.PadToColumn(50);
- OS << ";";
- }
- OS << " [debug variable = " << DVI->getVariable()->getName() << "]";
- }
}
}
};
diff --git a/llvm/unittests/CodeGen/LexicalScopesTest.cpp b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
index 3d707462fa615..563d496d1e600 100644
--- a/llvm/unittests/CodeGen/LexicalScopesTest.cpp
+++ b/llvm/unittests/CodeGen/LexicalScopesTest.cpp
@@ -67,7 +67,7 @@ class LexicalScopesTest : public testing::Test {
BeanInst.Opcode = 1;
BeanInst.Size = 1;
- memset(&DbgValueInst, 0, sizeof(DbgValueInst));
+ memset(&DbgValueInst, 0, sizeof(MCInstrDesc));
DbgValueInst.Opcode = TargetOpcode::DBG_VALUE;
DbgValueInst.Size = 1;
DbgValueInst.Flags = 1U << MCID::Meta;
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 41bf863420304..baa13e1199eea 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -188,6 +188,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgRecord) {
SmallVector<DbgValueInst *, 1> DVIs;
SmallVector<DbgVariableRecord *, 1> DVRs;
findDbgValues(DVIs, &I, &DVRs);
+ assert(DVIs.empty());
// Delete %b. The dbg.value should now point to undef.
I.eraseFromParent();
@@ -314,6 +315,7 @@ TEST(MetadataTest, DeleteInstUsedByDbgVariableRecord) {
SmallVector<DbgValueInst *, 2> DVIs;
SmallVector<DbgVariableRecord *, 2> DVRs;
findDbgValues(DVIs, &I, &DVRs);
+ assert(DVIs.empty());
ASSERT_EQ(DVRs.size(), 2u);
// Delete %b. The DbgVariableRecord should now point to undef.
diff --git a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
index 0b00734fc4d75..1daf381ee2862 100644
--- a/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
+++ b/llvm/unittests/Transforms/Utils/DebugifyTest.cpp
@@ -54,20 +54,13 @@ struct DebugInfoDrop : public FunctionPass {
struct DebugValueDrop : public FunctionPass {
static char ID;
bool runOnFunction(Function &F) override {
- SmallVector<DbgVariableIntrinsic *, 4> Dbgs;
for (BasicBlock &BB : F) {
- // Remove dbg var intrinsics.
for (Instruction &I : BB) {
- if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
- Dbgs.push_back(DVI);
- // If there are any non-intrinsic records (DbgRecords), drop those too.
+ // If there are any debug records, drop them.
I.dropDbgRecords();
}
}
- for (auto &I : Dbgs)
- I->eraseFromParent();
-
return true;
}
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index b922216ef8893..dd2a6249c7cf9 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -633,62 +633,6 @@ TEST(Local, ChangeToUnreachable) {
EXPECT_EQ(DLA, DLB);
}
-TEST(Local, FindDbgUsers) {
- LLVMContext Ctx;
- std::unique_ptr<Module> M = parseIR(Ctx,
- R"(
- define dso_local void @fun(ptr %a) #0 !dbg !11 {
- entry:
- #dbg_assign(ptr %a, !16, !DIExpression(), !15, ptr %a, !DIExpression(), !19)
- ret void
- }
-
- !llvm.dbg.cu = !{!0}
- !llvm.module.flags = !{!2, !3, !9}
- !llvm.ident = !{!10}
-
- !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
- !1 = !DIFile(filename: "test.cpp", directory: "/")
- !2 = !{i32 7, !"Dwarf Version", i32 5}
- !3 = !{i32 2, !"Debug Info Version", i32 3}
- !4 = !{i32 1, !"wchar_size", i32 4}
- !9 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
- !10 = !{!"clang version 17.0.0"}
- !11 = distinct !DISubprogram(name: "fun", linkageName: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !14)
- !12 = !DISubroutineType(types: !13)
- !13 = !{null}
- !14 = !{}
- !15 = distinct !DIAssignID()
- !16 = !DILocalVariable(name: "x", scope: !11, file: !1, line: 2, type: !17)
- !17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
- !18 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
- !19 = !DILocation(line: 0, scope: !11)
- )");
-
- bool BrokenDebugInfo = true;
- verifyModule(*M, &errs(), &BrokenDebugInfo);
- ASSERT_FALSE(BrokenDebugInfo);
-
- // Convert to debug intrinsics as we want to test findDbgUsers and
- // findDbgValue's debug-intrinsic-finding code here.
- // TODO: Remove this test when debug intrinsics are removed.
- M->convertFromNewDbgValues();
-
- Function &Fun = *cast<Function>(M->getNamedValue("fun"));
- Value *Arg = Fun.getArg(0);
- SmallVector<DbgVariableIntrinsic *> Users;
- // Arg (%a) is used twice by a single dbg.assign. Check findDbgUsers returns
- // only 1 pointer to it rather than 2.
- findDbgUsers(Users, Arg);
- EXPECT_EQ(Users.size(), 1u);
-
- SmallVector<DbgValueInst *> Vals;
- // Arg (%a) is used twice by a single dbg.assign. Check findDbgValues returns
- // only 1 pointer to it rather than 2.
- findDbgValues(Vals, Arg);
- EXPECT_EQ(Vals.size(), 1u);
-}
-
TEST(Local, FindDbgRecords) {
// DbgRecord copy of the FindDbgUsers test above.
LLVMContext Ctx;
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 82ec8121548c9..5309b5d095134 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -35,7 +35,6 @@ if (current_toolchain == default_toolchain) {
"_LIBCPP_HAS_LOCALIZATION=1",
"_LIBCPP_HAS_UNICODE=1",
"_LIBCPP_HAS_WIDE_CHARACTERS=1",
- "_LIBCPP_HAS_NO_STD_MODULES=",
"_LIBCPP_HAS_TERMINAL=1",
"_LIBCPP_INSTRUMENTED_WITH_ASAN=",
"_LIBCPP_ABI_DEFINES=",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index d4adeddd9b4e4..3d11ce566207a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -176,6 +176,7 @@ static_library("LLVMAMDGPUCodeGen") {
"AMDGPUPreLegalizerCombiner.cpp",
"AMDGPUPreloadKernArgProlog.cpp",
"AMDGPUPreloadKernelArguments.cpp",
+ "AMDGPUPrepareAGPRAlloc.cpp",
"AMDGPUPrintfRuntimeBinding.cpp",
"AMDGPUPromoteAlloca.cpp",
"AMDGPUPromoteKernelArguments.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 946b63f8a54fb..7ed0d3c6824e2 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -286,6 +286,7 @@ group("test") {
"//llvm/tools/llvm-extract",
"//llvm/tools/llvm-gsymutil:llvm-gsymutil",
"//llvm/tools/llvm-ifs",
+ "//llvm/tools/llvm-ir2vec",
"//llvm/tools/llvm-isel-fuzzer",
"//llvm/tools/llvm-jitlink",
"//llvm/tools/llvm-jitlink/llvm-jitlink-executor",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
new file mode 100644
index 0000000000000..07a795122c76c
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ir2vec/BUILD.gn
@@ -0,0 +1,9 @@
+executable("llvm-ir2vec") {
+ deps = [
+ "//llvm/lib/Analysis",
+ "//llvm/lib/IR",
+ "//llvm/lib/IRReader",
+ "//llvm/lib/Support",
+ ]
+ sources = [ "llvm-ir2vec.cpp" ]
+}
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index 0f45f5607bab9..08c4e0f9d859c 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -133,6 +133,8 @@ This will generate two files, `MyExtension.h.inc` and `MyExtension.cpp.inc`, tha
```c++
// In MyExtension.cpp.
+#include "MyExtension.h"
+
#define GET_OP_CLASSES
#include "MyExtension.cpp.inc"
@@ -283,7 +285,7 @@ void registerMyExtension(::mlir::DialectRegistry ®istry) {
}
```
-After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is.
+After registering the extension, it becomes possible to use our new operation in the Transform dialect interpreter. The upstream testing pass can be used as is. It actually exists in `mlir/test/Examples/transform/Ch2/sequence.mlir`, which contains the `microkernel` implementation.
```mlir
module attributes {transform.with_named_sequence} {
@@ -300,7 +302,7 @@ module attributes {transform.with_named_sequence} {
// The actual tiling transformation takes tile sizes as attributes. It
// produces a handle to the loop generated during tiling.
- %loop, %tiled = transform.structured.tile_using_forall %max
+ %tiled, %loop = transform.structured.tile_using_forall %max
tile_sizes [8, 32]
: (!transform.any_op) -> (!transform.any_op, !transform.any_op)
@@ -311,32 +313,32 @@ module attributes {transform.with_named_sequence} {
// a single handle to all operations and give it to
// `fuse_into_containing_op` that would take care of the ordering in this
// case.
- %add_fused = transform.structured.fuse_into_containing_op %add into %loop
- : (!transform.any_op, !transform.any_op) -> !transform.any_op
- %matmul_fused = transform.structured.fuse_into_containing_op %arg1
- into %loop
+ %add_fused, %loop2 = transform.structured.fuse_into_containing_op %add into %loop
+ : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+ %matmul_fused, %loop3 = transform.structured.fuse_into_containing_op %arg1
+ into %loop2
: (!transform.op<"linalg.matmul">, !transform.any_op)
- -> !transform.any_op
+ -> (!transform.any_op, !transform.any_op)
// Tile again to get the desired size. Note that this time this tiles the
// "add" operation and fuses matmul into the loop, but doesn't affect the
// "max" operation. This illustrates the precise targeting with the
// transform dialect. Otherwise, it is difficult to differentiate "add" and
// "max", both of which having the same kind.
- %loop_2, %tiled_2 = transform.structured.tile_using_forall %add_fused
+ %tiled_second, %loop_second = transform.structured.tile_using_forall %add_fused
tile_sizes [4, 4]
: (!transform.any_op) -> (!transform.any_op, !transform.any_op)
- %matmul_fused_2 = transform.structured.fuse_into_containing_op %matmul_fused
- into %loop_2
- : (!transform.any_op, !transform.any_op) -> !transform.any_op
+ %matmul_fused_2, %loop_second_2 = transform.structured.fuse_into_containing_op %matmul_fused
+ into %loop_second
+ : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
// Since outlining is currently only implemented for region-holding
// operations such as loops, use tiling to size 1 to materialize the outer
// loop that is going to be outlined.
- %outline_target, %_ = transform.structured.tile_using_forall %tiled_2 tile_sizes [1]
+ %_0, %loop_third = transform.structured.tile_using_forall %tiled_second tile_sizes [1]
: (!transform.any_op) -> (!transform.any_op, !transform.any_op)
- transform.structured.fuse_into_containing_op %matmul_fused_2 into %outline_target
- : (!transform.any_op, !transform.any_op) -> !transform.any_op
+ %_1, %outline_target = transform.structured.fuse_into_containing_op %matmul_fused_2 into %loop_third
+ : (!transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
%func, %call = transform.loop.outline %outline_target
{func_name = "outlined"}
: (!transform.any_op) -> (!transform.any_op, !transform.any_op)
diff --git a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
index ac4915901fdec..ff99d7ce96daf 100644
--- a/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
+++ b/mlir/include/mlir/Conversion/VectorToXeGPU/VectorToXeGPU.h
@@ -21,9 +21,6 @@ class RewritePatternSet;
/// Collect a set of patterns to convert from the vector to XeGPU ops.
void populateVectorToXeGPUConversionPatterns(RewritePatternSet &patterns);
-/// Create a pass to convert ops from vector to XeGPU.
-std::unique_ptr<Pass> createConvertVectorToXeGPUPass();
-
} // namespace mlir
#endif // MLIR_CONVERSION_VECTORTOXEGPU_VECTORTOXEGPU_H
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 12c7cc62905c9..d4400547f9568 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -1115,6 +1115,18 @@ struct AMDGPUStreamTy {
return Plugin::success();
}
+ /// Complete pending post actions until and including the event in target
+ /// slot.
+ Error completeUntil(uint32_t TargetSlot) {
+ for (uint32_t Slot = 0; Slot <= TargetSlot; ++Slot) {
+ // Take the post action of the operation if any.
+ if (auto Err = Slots[Slot].performAction())
+ return Err;
+ }
+
+ return Plugin::success();
+ }
+
/// Make the current stream wait on a specific operation of another stream.
/// The idea is to make the current stream waiting on two signals: 1) the last
/// signal of the current stream, and 2) the last signal of the other stream.
@@ -1502,6 +1514,11 @@ struct AMDGPUStreamTy {
return complete();
}
+ /// Synchronize the stream until the given event. The current thread waits
+ /// until the provided event is finalized, and it performs the pending post
+ /// actions for that and prior events.
+ Error synchronizeOn(AMDGPUEventTy &Event);
+
/// Query the stream and complete pending post actions if operations finished.
/// Return whether all the operations completed. This operation does not block
/// the calling thread.
@@ -1575,6 +1592,21 @@ struct AMDGPUEventTy {
return Stream.waitEvent(*this);
}
+ Error sync() {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ if (!RecordedStream)
+ return Plugin::error(ErrorCode::INVALID_ARGUMENT,
+ "event does not have any recorded stream");
+
+ // No need to wait on anything, the recorded stream already finished the
+ // corresponding operation.
+ if (RecordedSlot < 0)
+ return Plugin::success();
+
+ return RecordedStream->synchronizeOn(*this);
+ }
+
protected:
/// The stream registered in this event.
AMDGPUStreamTy *RecordedStream;
@@ -1630,6 +1662,22 @@ Error AMDGPUStreamTy::waitEvent(const AMDGPUEventTy &Event) {
return waitOnStreamOperation(RecordedStream, Event.RecordedSlot);
}
+Error AMDGPUStreamTy::synchronizeOn(AMDGPUEventTy &Event) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ // Wait until the requested slot has completed
+ if (auto Err = Slots[Event.RecordedSlot].Signal->wait(
+ StreamBusyWaitMicroseconds, &Device))
+ return Err;
+
+ // If the event is the last one in the stream, just do a full finalize
+ if (Event.RecordedSlot == last())
+ return complete();
+
+ // Otherwise, only finalize until the appropriate event
+ return completeUntil(Event.RecordedSlot);
+}
+
struct AMDGPUStreamManagerTy final
: GenericDeviceResourceManagerTy<AMDGPUResourceRef<AMDGPUStreamTy>> {
using ResourceRef = AMDGPUResourceRef<AMDGPUStreamTy>;
@@ -2540,8 +2588,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
/// Synchronize the current thread with the event.
Error syncEventImpl(void *EventPtr) override {
- return Plugin::error(ErrorCode::UNIMPLEMENTED,
- "synchronize event not implemented");
+ AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr);
+ return Event->sync();
}
/// Print information about the device.
@@ -2562,7 +2610,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
Status2 = hsa_system_get_info(HSA_SYSTEM_INFO_VERSION_MINOR, &Minor);
if (Status == HSA_STATUS_SUCCESS && Status2 == HSA_STATUS_SUCCESS)
Info.add("HSA Runtime Version",
- std::to_string(Major) + "." + std::to_string(Minor));
+ std::to_string(Major) + "." + std::to_string(Minor), "",
+ DeviceInfo::DRIVER_VERSION);
Info.add("HSA OpenMP Device Number", DeviceId);
@@ -2572,11 +2621,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
Status = getDeviceAttrRaw(HSA_AGENT_INFO_NAME, TmpChar);
if (Status == HSA_STATUS_SUCCESS)
- Info.add("Device Name", TmpChar);
+ Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
Status = getDeviceAttrRaw(HSA_AGENT_INFO_VENDOR_NAME, TmpChar);
if (Status == HSA_STATUS_SUCCESS)
- Info.add("Vendor Name", TmpChar);
+ Info.add("Vendor Name", TmpChar, "", DeviceInfo::VENDOR);
hsa_device_type_t DevType;
Status = getDeviceAttrRaw(HSA_AGENT_INFO_DEVICE, DevType);
@@ -2652,7 +2701,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
Status = getDeviceAttrRaw(HSA_AGENT_INFO_WORKGROUP_MAX_DIM, WorkgrpMaxDim);
if (Status == HSA_STATUS_SUCCESS) {
- auto &MaxSize = *Info.add("Workgroup Max Size per Dimension");
+ auto &MaxSize =
+ *Info.add("Workgroup Max Size per Dimension", std::monostate{}, "",
+ DeviceInfo::MAX_WORK_GROUP_SIZE);
MaxSize.add("x", WorkgrpMaxDim[0]);
MaxSize.add("y", WorkgrpMaxDim[1]);
MaxSize.add("z", WorkgrpMaxDim[2]);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 162b149ab483e..8c17a2ee07047 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -113,6 +113,12 @@ struct AsyncInfoWrapperTy {
__tgt_async_info *AsyncInfoPtr;
};
+enum class DeviceInfo {
+#define OFFLOAD_DEVINFO(Name, _, Value) Name = Value,
+#include "OffloadInfo.inc"
+#undef OFFLOAD_DEVINFO
+};
+
/// Tree node for device information
///
/// This information is either printed or used by liboffload to extract certain
@@ -133,6 +139,8 @@ struct InfoTreeNode {
// * The same key can appear multiple times
std::unique_ptr<llvm::SmallVector<InfoTreeNode, 8>> Children;
+ llvm::DenseMap<DeviceInfo, size_t> DeviceInfoMap;
+
InfoTreeNode() : InfoTreeNode("", std::monostate{}, "") {}
InfoTreeNode(std::string Key, VariantType Value, std::string Units)
: Key(Key), Value(Value), Units(Units) {}
@@ -140,10 +148,12 @@ struct InfoTreeNode {
/// Add a new info entry as a child of this node. The entry requires at least
/// a key string in \p Key. The value in \p Value is optional and can be any
/// type that is representable as a string. The units in \p Units is optional
- /// and must be a string.
+ /// and must be a string. Providing a device info key allows liboffload to
+ /// use that value for an appropriate olGetDeviceInfo query
template <typename T = std::monostate>
InfoTreeNode *add(std::string Key, T Value = T(),
- const std::string &Units = std::string()) {
+ const std::string &Units = std::string(),
+ std::optional<DeviceInfo> DeviceInfoKey = std::nullopt) {
assert(!Key.empty() && "Invalid info key");
if (!Children)
@@ -157,7 +167,12 @@ struct InfoTreeNode {
else
ValueVariant = std::string{Value};
- return &Children->emplace_back(Key, ValueVariant, Units);
+ auto Ptr = &Children->emplace_back(Key, ValueVariant, Units);
+
+ if (DeviceInfoKey)
+ DeviceInfoMap[*DeviceInfoKey] = Children->size() - 1;
+
+ return Ptr;
}
std::optional<InfoTreeNode *> get(StringRef Key) {
@@ -171,6 +186,13 @@ struct InfoTreeNode {
return It;
}
+ std::optional<InfoTreeNode *> get(DeviceInfo Info) {
+ auto Result = DeviceInfoMap.find(Info);
+ if (Result != DeviceInfoMap.end())
+ return &(*Children)[Result->second];
+ return std::nullopt;
+ }
+
/// Print all info entries in the tree
void print() const {
// Fake an additional indent so that values are offset from the keys
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index b787376eb1770..728bf07c572bb 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -935,15 +935,16 @@ struct CUDADeviceTy : public GenericDeviceTy {
if (Res == CUDA_SUCCESS)
// For consistency with other drivers, store the version as a string
// rather than an integer
- Info.add("CUDA Driver Version", std::to_string(TmpInt));
+ Info.add("CUDA Driver Version", std::to_string(TmpInt), "",
+ DeviceInfo::DRIVER_VERSION);
Info.add("CUDA OpenMP Device Number", DeviceId);
Res = cuDeviceGetName(TmpChar, 1000, Device);
if (Res == CUDA_SUCCESS)
- Info.add("Device Name", TmpChar);
+ Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
- Info.add("Vendor Name", "NVIDIA");
+ Info.add("Vendor Name", "NVIDIA", "", DeviceInfo::VENDOR);
Res = cuDeviceTotalMem(&TmpSt, Device);
if (Res == CUDA_SUCCESS)
@@ -978,7 +979,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
if (Res == CUDA_SUCCESS)
Info.add("Maximum Threads per Block", TmpInt);
- auto &MaxBlock = *Info.add("Maximum Block Dimensions", "");
+ auto &MaxBlock = *Info.add("Maximum Block Dimensions", std::monostate{}, "",
+ DeviceInfo::MAX_WORK_GROUP_SIZE);
Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, TmpInt);
if (Res == CUDA_SUCCESS)
MaxBlock.add("x", TmpInt);
diff --git a/offload/unittests/OffloadAPI/common/Fixtures.hpp b/offload/unittests/OffloadAPI/common/Fixtures.hpp
index e5d815ecda965..546921164f691 100644
--- a/offload/unittests/OffloadAPI/common/Fixtures.hpp
+++ b/offload/unittests/OffloadAPI/common/Fixtures.hpp
@@ -171,9 +171,6 @@ struct OffloadQueueTest : OffloadDeviceTest {
struct OffloadEventTest : OffloadQueueTest {
void SetUp() override {
RETURN_ON_FATAL_FAILURE(OffloadQueueTest::SetUp());
- if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
- GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
// Get an event from a memcpy. We can still use it in olGetEventInfo etc
// after it has been waited on.
void *Alloc;
diff --git a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
index 05356d4ef8d75..f80dabb4fc93f 100644
--- a/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
+++ b/offload/unittests/OffloadAPI/event/olWaitEvent.cpp
@@ -14,9 +14,6 @@ using olWaitEventTest = OffloadQueueTest;
OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olWaitEventTest);
TEST_P(olWaitEventTest, Success) {
- if (getPlatformBackend() == OL_PLATFORM_BACKEND_AMDGPU)
- GTEST_SKIP() << "AMDGPU synchronize event not implemented";
-
uint32_t Src = 42;
void *DstPtr;
>From bfe68539b78d8610bd0cd315c9a3e8fe9b456a3f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 18 Jul 2025 16:45:02 +0100
Subject: [PATCH 6/6] !fixup address latest comments, thanks
---
llvm/lib/Analysis/IVDescriptors.cpp | 6 +-
.../Vectorize/VPlanConstruction.cpp | 49 ++++----
.../Transforms/Vectorize/VPlanTransforms.h | 8 +-
.../fmax-without-fast-math-flags.ll | 116 ++++++++++++++++++
4 files changed, 150 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index a88bbd7b65d53..8be5de3bf356f 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -947,10 +947,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
InstDesc Res = isMinMaxPattern(I, Kind, Prev);
if (!Res.isRecurrence())
return InstDesc(false, I);
-
if (HasRequiredFMF())
return Res;
-
// We may be able to vectorize FMax/FMin reductions using maxnum/minnum
// intrinsics with extra checks ensuring the vector loop handles only
// non-NaN inputs.
@@ -964,9 +962,9 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
"unexpected recurrence kind for minnum");
return InstDesc(I, RecurKind::FMinNum);
}
-
return InstDesc(false, I);
- } else if (isFMulAddIntrinsic(I))
+ }
+ if (isFMulAddIntrinsic(I))
return InstDesc(Kind == RecurKind::FMulAdd, I,
I->hasAllowReassoc() ? nullptr : I);
return InstDesc(false, I);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index a426d06b4befe..98c2fe2d6129d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -654,10 +654,6 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
}
bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPReductionPHIRecipe *RedPhiR = nullptr;
- bool HasUnsupportedPhi = false;
-
auto GetMinMaxCompareValue = [](VPReductionPHIRecipe *RedPhiR) -> VPValue * {
auto *MinMaxR = dyn_cast<VPRecipeWithIRFlags>(
RedPhiR->getBackedgeValue()->getDefiningRecipe());
@@ -666,7 +662,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxR);
if (!isa<VPWidenIntrinsicRecipe>(MinMaxR) &&
- !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+ !(RepR && isa<IntrinsicInst>(RepR->getUnderlyingInstr())))
return nullptr;
#ifndef NDEBUG
@@ -690,41 +686,47 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
return MinMaxR->getOperand(0);
};
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPReductionPHIRecipe *RedPhiR = nullptr;
+ bool HasUnsupportedPhi = false;
for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
- // TODO: Also support fixed-order recurrence phis.
- HasUnsupportedPhi |=
- !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
- VPReductionPHIRecipe>(&R);
+ if (isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe>(&R))
+ continue;
auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
- if (!Cur)
+ if (!Cur) {
+ // TODO: Also support fixed-order recurrence phis.
+ HasUnsupportedPhi = true;
continue;
+ }
// For now, only a single reduction is supported.
// TODO: Support multiple MaxNum/MinNum reductions and other reductions.
if (RedPhiR)
return false;
if (Cur->getRecurrenceKind() != RecurKind::FMaxNum &&
- Cur->getRecurrenceKind() != RecurKind::FMinNum)
+ Cur->getRecurrenceKind() != RecurKind::FMinNum) {
+ HasUnsupportedPhi = true;
continue;
+ }
RedPhiR = Cur;
}
if (!RedPhiR)
return true;
- RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
- assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
- "unsupported reduction");
-
- VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
- if (!MinMaxOp)
- return false;
-
// We won't be able to resume execution in the scalar tail, if there are
// unsupported header phis or there is no scalar tail at all, due to
// tail-folding.
if (HasUnsupportedPhi || !Plan.hasScalarTail())
return false;
+ VPValue *MinMaxOp = GetMinMaxCompareValue(RedPhiR);
+ if (!MinMaxOp)
+ return false;
+
+ RecurKind RedPhiRK = RedPhiR->getRecurrenceKind();
+ assert((RedPhiRK == RecurKind::FMaxNum || RedPhiRK == RecurKind::FMinNum) &&
+ "unsupported reduction");
+
/// Check if the vector loop of \p Plan can early exit and restart
/// execution of last vector iteration in the scalar loop. This requires all
/// recipes up to early exit point be side-effect free as they are
@@ -774,7 +776,7 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
RdxResult->setOperand(1, NewSel);
auto *ScalarPH = Plan.getScalarPreheader();
- // Update the resume phis for inductions in the scalar preheader. If AnyNaN is
+ // Update resume phis for inductions in the scalar preheader. If AnyNaN is
// true, the resume from the start of the last vector iteration via the
// canonical IV, otherwise from the original value.
for (auto &R : ScalarPH->phis()) {
@@ -782,8 +784,13 @@ bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
VPValue *VecV = ResumeR->getOperand(0);
if (VecV == RdxResult)
continue;
- if (VecV != &Plan.getVectorTripCount())
+ // Bail out and abandon the current, partially modified, VPlan if we
+ // encounter resume phi that cannot be updated yet.
+ if (VecV != &Plan.getVectorTripCount()) {
+ LLVM_DEBUG(dbgs() << "Found resume phi we cannot update for VPlan with "
+ "FMaxNum/FMinNum reduction.\n");
return false;
+ }
auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
ResumeR->setOperand(0, NewSel);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index b1692d3e6b337..04cb7a7a5c19b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -103,10 +103,10 @@ struct VPlanTransforms {
/// not valid.
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
- /// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
- /// If they do, try to update the vector loop to exit early if any input is
- /// NaN and resume executing in the scalar loop to handle the NaNs there.
- /// Return false if this attempt was unsuccessful.
+ /// Check if \p Plan contains any FMaxNum or FMinNum reductions. If they do,
+ /// try to update the vector loop to exit early if any input is NaN and resume
+ /// executing in the scalar loop to handle the NaNs there. Return false if
+ /// this attempt was unsuccessful.
static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
/// Clear NSW/NUW flags from reduction instructions if necessary.
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 6204d540df40c..49c25b02ce964 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -324,6 +324,122 @@ exit:
ret float %max.next
}
+define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_10(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 10, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 10, %entry ], [ %iv.next, %loop ]
+ %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+ %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+ %l = load float, ptr %gep.src, align 4
+ %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret float %max.next
+}
+
+define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_induction_starts_at_value(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]
+ %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+ %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+ %l = load float, ptr %gep.src, align 4
+ %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret float %max.next
+}
+
+define float @fmaxnum_with_additional_add(ptr noalias %src, ptr noalias %src.2, i64 %n) {
+; CHECK-LABEL: define float @fmaxnum_with_additional_add(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[SRC_2:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX:%.*]] = phi float [ -1.000000e+07, %[[ENTRY]] ], [ [[MAX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds nuw i32, ptr [[SRC_2]], i64 [[IV]]
+; CHECK-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
+; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[L_SRC_2]]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds nuw float, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: [[MAX_NEXT]] = call float @llvm.maxnum.f32(float [[L]], float [[MAX]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[MAX_NEXT_LCSSA:%.*]] = phi float [ [[MAX_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: store i32 [[SUM_NEXT_LCSSA]], ptr [[SRC_2]], align 4
+; CHECK-NEXT: ret float [[MAX_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %max = phi float [ -1.000000e+07, %entry ], [ %max.next, %loop ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %loop ]
+ %gep.src.2 = getelementptr inbounds nuw i32, ptr %src.2, i64 %iv
+ %l.src.2 = load i32, ptr %gep.src.2, align 4
+ %sum.next = add i32 %sum, %l.src.2
+ %gep.src = getelementptr inbounds nuw float, ptr %src, i64 %iv
+ %l = load float, ptr %gep.src, align 4
+ %max.next = call float @llvm.maxnum.f32(float %l, float %max)
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store i32 %sum.next, ptr %src.2
+ ret float %max.next
+}
+
define float @fmax_select_with_blend(ptr %A, ptr %B) {
; CHECK-LABEL: define float @fmax_select_with_blend(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
More information about the lldb-commits
mailing list