[clang] [llvm] [Clang][CodeGen] Avoid failing downcast when __builtin_prefech's immarg is poison (PR #201623)
via cfe-commits
cfe-commits at lists.llvm.org
Thu Jun 4 09:28:14 PDT 2026
https://github.com/jjppp created https://github.com/llvm/llvm-project/pull/201623
Fixes #201448
This PR fixes downcast failures in lowering `__builtin_prefetch` for immediate arguments.
The code currently assumes that the immediate operands are `ConstantInt`s. This is not always true as poison values may come from UB expressions (e.g., `__builtin_prefetch(0, 2 >> 32)`). This would cause the downcast `cast<ConstantInt>` to fail.
Since the source program already contains UB, this PR avoids downcast failure by replacing poisonous values with a legal default value `0` in CodeGen.
A regression test is also added to cover the poison immediate argument case.
>From 5e77e5fdad4ead4d908a3640117d0fdbaa73cd77 Mon Sep 17 00:00:00 2001
From: jpwang <jpwang at smail.nju.edu.cn>
Date: Fri, 5 Jun 2026 00:16:33 +0800
Subject: [PATCH] [Clang][CodeGen] Avoid failing downcast when
__builtin_prefech's immarg is poison
---
clang/test/CodeGen/prefetch-poison-rw.c | 6 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 965 ++++++++++--------
2 files changed, 517 insertions(+), 454 deletions(-)
create mode 100644 clang/test/CodeGen/prefetch-poison-rw.c
diff --git a/clang/test/CodeGen/prefetch-poison-rw.c b/clang/test/CodeGen/prefetch-poison-rw.c
new file mode 100644
index 0000000000000..c13d4bb246d54
--- /dev/null
+++ b/clang/test/CodeGen/prefetch-poison-rw.c
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm %s -o - | FileCheck %s
+
+void test_poison_rw(void) {
+ __builtin_prefetch(0, 2 >> 32);
+ // CHECK: call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 1)
+}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index eca5bb1598ae0..ff55e3862610a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -190,11 +190,12 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
// Assemble the power of 2 part.
unsigned RoundParts = llvm::bit_floor(NumParts);
unsigned RoundBits = PartBits * RoundParts;
- EVT RoundVT = RoundBits == ValueBits ?
- ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
+ EVT RoundVT = RoundBits == ValueBits
+ ? ValueVT
+ : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
SDValue Lo, Hi;
- EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
+ EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits / 2);
if (RoundParts > 2) {
Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
@@ -262,7 +263,7 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
ValueVT.bitsLT(PartEVT)) {
// For an FP value in an integer part, we need to truncate to the right
// width first.
- PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
+ PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
}
@@ -277,8 +278,8 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
// indicate whether the truncated bits will always be
// zero or sign-extension.
if (AssertOp)
- Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
- DAG.getValueType(ValueVT));
+ Val =
+ DAG.getNode(*AssertOp, DL, PartEVT, Val, DAG.getValueType(ValueVT));
return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
}
return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
@@ -368,7 +369,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
NumParts = NumRegs; // Silence a compiler warning.
assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
assert(RegisterVT.getSizeInBits() ==
- Parts[0].getSimpleValueType().getSizeInBits() &&
+ Parts[0].getSimpleValueType().getSizeInBits() &&
"Part type sizes don't match!");
// Assemble the parts into intermediate operands.
@@ -451,21 +452,21 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
if (ValueVT.getVectorNumElements() != 1) {
- // Certain ABIs require that vectors are passed as integers. For vectors
- // are the same size, this is an obvious bitcast.
- if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
- return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
- } else if (ValueVT.bitsLT(PartEVT)) {
- const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
- EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
- // Drop the extra bits.
- Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
- return DAG.getBitcast(ValueVT, Val);
- }
-
- diagnosePossiblyInvalidConstraint(
- *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
- return DAG.getUNDEF(ValueVT);
+ // Certain ABIs require that vectors are passed as integers. For vectors
+ // are the same size, this is an obvious bitcast.
+ if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
+ return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
+ } else if (ValueVT.bitsLT(PartEVT)) {
+ const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
+ EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
+ // Drop the extra bits.
+ Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
+ return DAG.getBitcast(ValueVT, Val);
+ }
+
+ diagnosePossiblyInvalidConstraint(
+ *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
+ return DAG.getUNDEF(ValueVT);
}
// Handle cases such as i8 -> <1 x i1>
@@ -542,12 +543,11 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
if (ValueVT.isFloatingPoint()) {
// FP values need to be bitcast, then extended if they are being put
// into a larger container.
- ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
+ ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
}
assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
- ValueVT.isInteger() &&
- "Unknown mismatch!");
+ ValueVT.isInteger() && "Unknown mismatch!");
ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
if (PartVT == MVT::x86mmx)
@@ -560,8 +560,7 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
// If the parts cover less bits than value has, truncate the value.
assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
- ValueVT.isInteger() &&
- "Unknown mismatch!");
+ ValueVT.isInteger() && "Unknown mismatch!");
ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
if (PartVT == MVT::x86mmx)
@@ -592,8 +591,9 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
unsigned RoundParts = llvm::bit_floor(NumParts);
unsigned RoundBits = RoundParts * PartBits;
unsigned OddParts = NumParts - RoundParts;
- SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
- DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
+ SDValue OddVal =
+ DAG.getNode(ISD::SRL, DL, ValueVT, Val,
+ DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
CallConv);
@@ -609,22 +609,21 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
// The number of parts is a power of 2. Repeatedly bisect the value using
// EXTRACT_ELEMENT.
- Parts[0] = DAG.getNode(ISD::BITCAST, DL,
- EVT::getIntegerVT(*DAG.getContext(),
- ValueVT.getSizeInBits()),
- Val);
+ Parts[0] = DAG.getNode(
+ ISD::BITCAST, DL,
+ EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()), Val);
for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
for (unsigned i = 0; i < NumParts; i += StepSize) {
unsigned ThisBits = StepSize * PartBits / 2;
EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
SDValue &Part0 = Parts[i];
- SDValue &Part1 = Parts[i+StepSize/2];
+ SDValue &Part1 = Parts[i + StepSize / 2];
- Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
- ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
- Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
- ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
+ Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, ThisVT, Part0,
+ DAG.getIntPtrConstant(1, DL));
+ Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, ThisVT, Part0,
+ DAG.getIntPtrConstant(0, DL));
if (ThisBits == PartBits && ThisVT != PartVT) {
Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
@@ -915,9 +914,9 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
for (unsigned i = 0; i != NumRegs; ++i) {
SDValue P;
if (!Glue) {
- P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
+ P = DAG.getCopyFromReg(Chain, dl, Regs[Part + i], RegisterVT);
} else {
- P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
+ P = DAG.getCopyFromReg(Chain, dl, Regs[Part + i], RegisterVT, *Glue);
*Glue = P.getValue(2);
}
@@ -930,7 +929,7 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
continue;
const FunctionLoweringInfo::LiveOutInfo *LOI =
- FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
+ FuncInfo.GetLiveOutRegInfo(Regs[Part + i]);
if (!LOI)
continue;
@@ -1027,7 +1026,7 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
// c3 = TokenFactor c1, c2
// ...
// = op c3, ..., f2
- Chain = Chains[NumRegs-1];
+ Chain = Chains[NumRegs - 1];
else
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
}
@@ -1075,8 +1074,8 @@ void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching,
for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
MVT RegisterVT = RegVTs[Value];
- unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
- RegisterVT);
+ unsigned NumRegs =
+ TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], RegisterVT);
for (unsigned i = 0; i != NumRegs; ++i) {
assert(Reg < Regs.size() && "Mismatch in # registers expected");
Register TheReg = Regs[Reg++];
@@ -1145,7 +1144,7 @@ SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
for (; i != e; ++i) {
assert(Pending[i].getNode()->getNumOperands() > 1);
if (Pending[i].getNode()->getOperand(0) == Root)
- break; // Don't add the root if we already indirectly depend on it.
+ break; // Don't add the root if we already indirectly depend on it.
}
if (i == e)
@@ -1207,11 +1206,9 @@ SDValue SelectionDAGBuilder::getRoot() {
// Chain up all pending constrained intrinsics together with all
// pending loads, by simply appending them to PendingLoads and
// then calling getMemoryRoot().
- PendingLoads.reserve(PendingLoads.size() +
- PendingConstrainedFP.size() +
+ PendingLoads.reserve(PendingLoads.size() + PendingConstrainedFP.size() +
PendingConstrainedFPStrict.size());
- PendingLoads.append(PendingConstrainedFP.begin(),
- PendingConstrainedFP.end());
+ PendingLoads.append(PendingConstrainedFP.begin(), PendingConstrainedFP.end());
PendingLoads.append(PendingConstrainedFPStrict.begin(),
PendingConstrainedFPStrict.end());
PendingConstrainedFP.clear();
@@ -1425,10 +1422,13 @@ void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
// Note: this doesn't use InstVisitor, because it has to work with
// ConstantExpr's in addition to instructions.
switch (Opcode) {
- default: llvm_unreachable("Unknown instruction type encountered!");
+ default:
+ llvm_unreachable("Unknown instruction type encountered!");
// Build the switch statement using the Instruction.def file.
-#define HANDLE_INST(NUM, OPCODE, CLASS) \
- case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
+#define HANDLE_INST(NUM, OPCODE, CLASS) \
+ case Instruction::OPCODE: \
+ visit##OPCODE((const CLASS &)I); \
+ break;
#include "llvm/IR/Instruction.def"
}
}
@@ -1786,8 +1786,8 @@ SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
DAG.getDataLayout(), InReg, Ty,
std::nullopt); // This is not an ABI copy.
SDValue Chain = DAG.getEntryNode();
- Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
- V);
+ Result =
+ RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
resolveDanglingDebugInfo(V, Result);
}
@@ -1800,7 +1800,8 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
// to do this first, so that we don't create a CopyFromReg if we already
// have a regular SDValue.
SDValue &N = NodeMap[V];
- if (N.getNode()) return N;
+ if (N.getNode())
+ return N;
// If there's a virtual register allocated and initialized for this
// value, use it.
@@ -1903,7 +1904,8 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
for (const Use &U : C->operands()) {
SDNode *Val = getValue(U).getNode();
// If the operand is an empty aggregate, there are no values.
- if (!Val) continue;
+ if (!Val)
+ continue;
// Add each leaf value from the operand to the Constants list
// to form a flattened list of all the values.
for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
@@ -1914,7 +1916,7 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
}
if (const ConstantDataSequential *CDS =
- dyn_cast<ConstantDataSequential>(C)) {
+ dyn_cast<ConstantDataSequential>(C)) {
SmallVector<SDValue, 4> Ops;
for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
@@ -2118,7 +2120,7 @@ static void findUnwindDestinations(
SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
&UnwindDests) {
EHPersonality Personality =
- classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
+ classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
@@ -2245,8 +2247,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
commonAlignment(BaseAlign, Offsets[i]));
}
- Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
- MVT::Other, Chains);
+ Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, Chains);
} else if (I.getNumOperands() != 0) {
SmallVector<Type *, 4> Types;
ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);
@@ -2341,7 +2342,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
CallingConv::ID CallConv =
- DAG.getMachineFunction().getFunction().getCallingConv();
+ DAG.getMachineFunction().getFunction().getCallingConv();
Chain = DAG.getTargetLoweringInfo().LowerReturn(
Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
@@ -2374,17 +2375,19 @@ void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
/// CopyTo/FromReg.
void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
// No need to export constants.
- if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
+ if (!isa<Instruction>(V) && !isa<Argument>(V))
+ return;
// Already exported?
- if (FuncInfo.isExportedInst(V)) return;
+ if (FuncInfo.isExportedInst(V))
+ return;
Register Reg = FuncInfo.InitializeRegForValue(V);
CopyValueToVirtualRegister(V, Reg);
}
-bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
- const BasicBlock *FromBB) {
+bool SelectionDAGBuilder::isExportableFromCurrentBlock(
+ const Value *V, const BasicBlock *FromBB) {
// The operands of the setcc have to be in this block. We don't know
// how to export them from some other block.
if (const Instruction *VI = dyn_cast<Instruction>(V)) {
@@ -2447,15 +2450,10 @@ static bool InBlock(const Value *V, const BasicBlock *BB) {
/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
/// This function emits a branch and is used at the leaves of an OR or an
/// AND operator tree.
-void
-SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
- MachineBasicBlock *TBB,
- MachineBasicBlock *FBB,
- MachineBasicBlock *CurBB,
- MachineBasicBlock *SwitchBB,
- BranchProbability TProb,
- BranchProbability FProb,
- bool InvertCond) {
+void SelectionDAGBuilder::EmitBranchForMergedCondition(
+ const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+ MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
+ BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
const BasicBlock *BB = CurBB->getBasicBlock();
// If the leaf of the tree is a comparison, merge the condition into
@@ -2494,8 +2492,8 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
// Create a CaseBlock record representing this branch.
ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
- CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
- nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
+ CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), nullptr, TBB,
+ FBB, CurBB, getCurSDLoc(), TProb, FProb);
SL->SwitchCases.push_back(CB);
}
@@ -2635,15 +2633,11 @@ bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether(
return true;
}
-void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
- MachineBasicBlock *TBB,
- MachineBasicBlock *FBB,
- MachineBasicBlock *CurBB,
- MachineBasicBlock *SwitchBB,
- Instruction::BinaryOps Opc,
- BranchProbability TProb,
- BranchProbability FProb,
- bool InvertCond) {
+void SelectionDAGBuilder::FindMergedConditions(
+ const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
+ MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
+ Instruction::BinaryOps Opc, BranchProbability TProb,
+ BranchProbability FProb, bool InvertCond) {
// Skip over not part of the tree and remember to invert op and operands at
// next level.
Value *NotCond;
@@ -2682,8 +2676,8 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
!InBlock(BOpOp0, CurBB->getBasicBlock()) ||
!InBlock(BOpOp1, CurBB->getBasicBlock())) {
- EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
- TProb, FProb, InvertCond);
+ EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
+ InvertCond);
return;
}
@@ -2765,9 +2759,10 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
/// If the set of cases should be emitted as a series of branches, return true.
/// If we should emit this as a bunch of and/or'd together conditions, return
/// false.
-bool
-SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
- if (Cases.size() != 2) return true;
+bool SelectionDAGBuilder::ShouldEmitAsBranches(
+ const std::vector<CaseBlock> &Cases) {
+ if (Cases.size() != 2)
+ return true;
// If this is two comparisons of the same values or'd or and'd together, they
// will get folded into a single comparison, so don't emit two blocks.
@@ -2780,8 +2775,7 @@ SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
// Handle: (X != null) | (Y != null) --> (X|Y) != 0
// Handle: (X == null) & (Y == null) --> (X|Y) == 0
- if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
- Cases[0].CC == Cases[1].CC &&
+ if (Cases[0].CmpRHS == Cases[1].CmpRHS && Cases[0].CC == Cases[1].CC &&
isa<Constant>(Cases[0].CmpRHS) &&
cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
@@ -2948,8 +2942,8 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
} else {
assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
- const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
- const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
+ const APInt &Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
+ const APInt &High = cast<ConstantInt>(CB.CmpRHS)->getValue();
SDValue CmpOp = getValue(CB.CmpMHS);
EVT VT = CmpOp.getValueType();
@@ -2958,10 +2952,10 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
ISD::SETLE);
} else {
- SDValue SUB = DAG.getNode(ISD::SUB, dl,
- VT, CmpOp, DAG.getConstant(Low, dl, VT));
- Cond = DAG.getSetCC(dl, MVT::i1, SUB,
- DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
+ SDValue SUB =
+ DAG.getNode(ISD::SUB, dl, VT, CmpOp, DAG.getConstant(Low, dl, VT));
+ Cond = DAG.getSetCC(dl, MVT::i1, SUB, DAG.getConstant(High - Low, dl, VT),
+ ISD::SETULE);
}
}
@@ -3044,12 +3038,12 @@ void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
// for the switch statement if the value being switched on exceeds the
// largest case in the switch.
SDValue CMP = DAG.getSetCC(
- dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
- Sub.getValueType()),
+ dl,
+ TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+ Sub.getValueType()),
Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
- SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
- MVT::Other, CopyTo, CMP,
+ SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, CopyTo, CMP,
DAG.getBasicBlock(JT.Default));
// Avoid emitting unnecessary branches to the next block.
@@ -3186,8 +3180,7 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
// Otherwise branch to success MBB.
- SDValue Br = DAG.getNode(ISD::BR, dl,
- MVT::Other, BrCond,
+ SDValue Br = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
DAG.getBasicBlock(SPD.getSuccessMBB()));
DAG.setRoot(Br);
@@ -3307,7 +3300,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
B.Reg = FuncInfo.CreateReg(B.RegVT);
SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
- MachineBasicBlock* MBB = B.Cases[0].ThisBB;
+ MachineBasicBlock *MBB = B.Cases[0].ThisBB;
if (!B.FallthroughUnreachable)
addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
@@ -3317,7 +3310,8 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
SDValue Root = CopyTo;
if (!B.FallthroughUnreachable) {
// Conditional branch to the default block.
- SDValue RangeCmp = DAG.getSetCC(dl,
+ SDValue RangeCmp = DAG.getSetCC(
+ dl,
TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
RangeSub.getValueType()),
RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
@@ -3360,12 +3354,12 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
} else {
// Make desired shift
- SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
- DAG.getConstant(1, dl, VT), ShiftOp);
+ SDValue SwitchVal =
+ DAG.getNode(ISD::SHL, dl, VT, DAG.getConstant(1, dl, VT), ShiftOp);
// Emit bit tests and jumps
- SDValue AndOp = DAG.getNode(ISD::AND, dl,
- VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
+ SDValue AndOp = DAG.getNode(ISD::AND, dl, VT, SwitchVal,
+ DAG.getConstant(B.Mask, dl, VT));
Cmp = DAG.getSetCC(
dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
@@ -3380,14 +3374,13 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
// and hence we need to normalize them to let the sum of them become one.
SwitchBB->normalizeSuccProbs();
- SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
- MVT::Other, getControlRoot(),
+ SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
Cmp, DAG.getBasicBlock(B.TargetBB));
// Avoid emitting unnecessary branches to the next block.
if (NextMBB != NextBlock(SwitchBB))
- BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
- DAG.getBasicBlock(NextMBB));
+ BrAnd =
+ DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, DAG.getBasicBlock(NextMBB));
DAG.setRoot(BrAnd);
}
@@ -3425,9 +3418,9 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
case Intrinsic::seh_try_end:
case Intrinsic::seh_scope_end:
if (EHPadMBB)
- // a block referenced by EH table
- // so dtor-funclet not removed by opts
- EHPadMBB->setMachineBlockAddressTaken();
+ // a block referenced by EH table
+ // so dtor-funclet not removed by opts
+ EHPadMBB->setMachineBlockAddressTaken();
break;
case Intrinsic::experimental_patchpoint_void:
case Intrinsic::experimental_patchpoint:
@@ -3577,8 +3570,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
CallBrMBB->normalizeSuccProbs();
// Drop into default successor.
- DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
- MVT::Other, getControlRoot(),
+ DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
DAG.getBasicBlock(Return)));
}
@@ -3587,8 +3579,7 @@ void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
}
void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
- assert(FuncInfo.MBB->isEHPad() &&
- "Call to landingpad not in landing pad!");
+ assert(FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!");
// If there aren't registers to copy the values into (e.g., during SjLj
// exceptions), then don't bother to create these DAG nodes.
@@ -3629,8 +3620,8 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
dl, ValueVTs[1]);
// Merge into one.
- SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
- DAG.getVTList(ValueVTs), Ops);
+ SDValue Res =
+ DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Ops);
setValue(&LP, Res);
}
@@ -3656,16 +3647,15 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
BasicBlock *BB = I.getSuccessor(i);
bool Inserted = Done.insert(BB).second;
if (!Inserted)
- continue;
+ continue;
MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
addSuccessorWithProb(IndirectBrMBB, Succ);
}
IndirectBrMBB->normalizeSuccProbs();
- DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
- MVT::Other, getControlRoot(),
- getValue(I.getAddress())));
+ DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), MVT::Other,
+ getControlRoot(), getValue(I.getAddress())));
}
void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
@@ -3682,8 +3672,8 @@ void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
Flags.copyFMF(*FPOp);
SDValue Op = getValue(I.getOperand(0));
- SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
- Op, Flags);
+ SDValue UnNodeValue =
+ DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), Op, Flags);
setValue(&I, UnNodeValue);
}
@@ -3702,8 +3692,8 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));
- SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
- Op1, Op2, Flags);
+ SDValue BinNodeValue =
+ DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, Flags);
setValue(&I, BinNodeValue);
}
@@ -3741,8 +3731,8 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
Flags.setExact(exact);
Flags.setNoSignedWrap(nsw);
Flags.setNoUnsignedWrap(nuw);
- SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
- Flags);
+ SDValue Res =
+ DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, Flags);
setValue(&I, Res);
}
@@ -3807,9 +3797,8 @@ void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
// Check if the condition of the select has one use or two users that are both
// selects with the same condition.
static bool hasOnlySelectUsers(const Value *Cond) {
- return llvm::all_of(Cond->users(), [](const Value *V) {
- return isa<SelectInst>(V);
- });
+ return llvm::all_of(Cond->users(),
+ [](const Value *V) { return isa<SelectInst>(V); });
}
void SelectionDAGBuilder::visitSelect(const User &I) {
@@ -3817,12 +3806,13 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
ValueVTs);
unsigned NumValues = ValueVTs.size();
- if (NumValues == 0) return;
+ if (NumValues == 0)
+ return;
SmallVector<SDValue, 4> Values(NumValues);
- SDValue Cond = getValue(I.getOperand(0));
- SDValue LHSVal = getValue(I.getOperand(1));
- SDValue RHSVal = getValue(I.getOperand(2));
+ SDValue Cond = getValue(I.getOperand(0));
+ SDValue LHSVal = getValue(I.getOperand(1));
+ SDValue RHSVal = getValue(I.getOperand(2));
SmallVector<SDValue, 1> BaseOps(1, Cond);
ISD::NodeType OpCode =
Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
@@ -3851,8 +3841,8 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
// If the vselect is legal, assume we want to leave this as a vector setcc +
// vselect. Otherwise, if this is going to be scalarized, we want to see if
// min/max is legal on the scalar type.
- bool UseScalarMinMax = VT.isVector() &&
- !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
+ bool UseScalarMinMax =
+ VT.isVector() && !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
// ValueTracking's select pattern matching does not account for -0.0,
// so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
@@ -3861,17 +3851,27 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
auto SPR = matchSelectPattern(&I, LHS, RHS);
ISD::NodeType Opc = ISD::DELETED_NODE;
switch (SPR.Flavor) {
- case SPF_UMAX: Opc = ISD::UMAX; break;
- case SPF_UMIN: Opc = ISD::UMIN; break;
- case SPF_SMAX: Opc = ISD::SMAX; break;
- case SPF_SMIN: Opc = ISD::SMIN; break;
+ case SPF_UMAX:
+ Opc = ISD::UMAX;
+ break;
+ case SPF_UMIN:
+ Opc = ISD::UMIN;
+ break;
+ case SPF_SMAX:
+ Opc = ISD::SMAX;
+ break;
+ case SPF_SMIN:
+ Opc = ISD::SMIN;
+ break;
case SPF_FMINNUM:
if (!TLI.isProfitableToCombineMinNumMaxNum(VT))
break;
switch (SPR.NaNBehavior) {
- case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
- case SPNB_RETURNS_NAN: break;
+ case SPNB_NA:
+ llvm_unreachable("No NaN behavior for FP op?");
+ case SPNB_RETURNS_NAN:
+ break;
case SPNB_RETURNS_OTHER:
Opc = ISD::FMINIMUMNUM;
Flags.setNoSignedZeros(true);
@@ -3889,8 +3889,10 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
break;
switch (SPR.NaNBehavior) {
- case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
- case SPNB_RETURNS_NAN: break;
+ case SPNB_NA:
+ llvm_unreachable("No NaN behavior for FP op?");
+ case SPNB_RETURNS_NAN:
+ break;
case SPNB_RETURNS_OTHER:
Opc = ISD::FMAXIMUMNUM;
Flags.setNoSignedZeros(true);
@@ -3910,7 +3912,8 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
IsUnaryAbs = true;
Opc = ISD::ABS;
break;
- default: break;
+ default:
+ break;
}
if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
@@ -4112,17 +4115,16 @@ void SelectionDAGBuilder::visitBitCast(const User &I) {
// BitCast assures us that source and destination are the same size so this is
// either a BITCAST or a no-op.
if (DestVT != N.getValueType())
- setValue(&I, DAG.getNode(ISD::BITCAST, dl,
- DestVT, N)); // convert types.
+ setValue(&I, DAG.getNode(ISD::BITCAST, dl, DestVT, N)); // convert types.
// Check if the original LLVM IR Operand was a ConstantInt, because getValue()
// might fold any kind of constant expression to an integer constant and that
// is not what we are looking for. Only recognize a bitcast of a genuine
// constant integer as an opaque constant.
- else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
+ else if (ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
- /*isOpaque*/true));
+ /*isOpaque*/ true));
else
- setValue(&I, N); // noop cast.
+ setValue(&I, N); // noop cast.
}
void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
@@ -4339,7 +4341,7 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
// replacing the shuffle with extract and build vector.
// to insert and build vector.
EVT EltVT = VT.getVectorElementType();
- SmallVector<SDValue,8> Ops;
+ SmallVector<SDValue, 8> Ops;
for (int Idx : Mask) {
SDValue Res;
@@ -4347,7 +4349,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) {
Res = DAG.getUNDEF(EltVT);
} else {
SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
- if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
+ if (Idx >= (int)SrcNumElts)
+ Idx -= SrcNumElts;
Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
DAG.getVectorIdxConstant(Idx, DL));
@@ -4390,19 +4393,20 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
unsigned i = 0;
// Copy the beginning value(s) from the original aggregate.
for (; i != LinearIndex; ++i)
- Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
- SDValue(Agg.getNode(), Agg.getResNo() + i);
+ Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i])
+ : SDValue(Agg.getNode(), Agg.getResNo() + i);
// Copy values from the inserted value(s).
if (NumValValues) {
SDValue Val = getValue(Op1);
for (; i != LinearIndex + NumValValues; ++i)
- Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
- SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
+ Values[i] =
+ FromUndef ? DAG.getUNDEF(AggValueVTs[i])
+ : SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
}
// Copy remaining value(s) from the original aggregate.
for (; i != NumAggValues; ++i)
- Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
- SDValue(Agg.getNode(), Agg.getResNo() + i);
+ Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i])
+ : SDValue(Agg.getNode(), Agg.getResNo() + i);
setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
DAG.getVTList(AggValueVTs), Values));
@@ -4435,9 +4439,9 @@ void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
// Copy out the selected value(s).
for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
Values[i - LinearIndex] =
- OutOfUndef ?
- DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
- SDValue(Agg.getNode(), Agg.getResNo() + i);
+ OutOfUndef
+ ? DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i))
+ : SDValue(Agg.getNode(), Agg.getResNo() + i);
setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
DAG.getVTList(ValValueVTs), Values));
@@ -4618,7 +4622,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
// If this is a fixed sized alloca in the entry block of the function,
// allocate it statically on the stack.
if (FuncInfo.StaticAllocaMap.count(&I))
- return; // getValue will auto-populate this.
+ return; // getValue will auto-populate this.
SDLoc dl = getCurSDLoc();
Type *Ty = I.getAllocatedType();
@@ -4806,8 +4810,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
PendingLoads.push_back(Chain);
}
- setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
- DAG.getVTList(ValueVTs), Values));
+ setValue(&I,
+ DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values));
}
void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
@@ -4837,8 +4841,7 @@ void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
"call visitLoadFromSwiftError when backend supports swifterror");
- assert(!I.isVolatile() &&
- !I.hasMetadata(LLVMContext::MD_nontemporal) &&
+ assert(!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) &&
!I.hasMetadata(LLVMContext::MD_invariant_load) &&
"Support volatile, non temporal, invariant for load_from_swift_error");
@@ -4998,7 +5001,7 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
SDValue &Scale, SelectionDAGBuilder *SDB,
const BasicBlock *CurBB, uint64_t ElemSize) {
- SelectionDAG& DAG = SDB->DAG;
+ SelectionDAG &DAG = SDB->DAG;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const DataLayout &DL = DAG.getDataLayout();
@@ -5085,7 +5088,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
}
- SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
+ SDValue Ops[] = {getMemoryRoot(), Src0, Mask, Base, Index, Scale};
SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
Ops, MMO, ISD::SIGNED_SCALED, false);
DAG.setRoot(Scatter);
@@ -5184,7 +5187,7 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
}
- SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
+ SDValue Ops[] = {Root, Src0, Mask, Base, Index, Scale};
SDValue Gather =
DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,
ISD::SIGNED_SCALED, ISD::NON_EXTLOAD);
@@ -5213,11 +5216,10 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
MemVT.getStoreSize(), I.getAlign(), AAMDNodes(),
nullptr, SSID, SuccessOrdering, FailureOrdering);
- SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
- dl, MemVT, VTs, InChain,
- getValue(I.getPointerOperand()),
- getValue(I.getCompareOperand()),
- getValue(I.getNewValOperand()), MMO);
+ SDValue L = DAG.getAtomicCmpSwap(
+ ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
+ getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
+ getValue(I.getNewValOperand()), MMO);
SDValue OutChain = L.getValue(2);
@@ -5229,22 +5231,53 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
SDLoc dl = getCurSDLoc();
ISD::NodeType NT;
switch (I.getOperation()) {
- default: llvm_unreachable("Unknown atomicrmw operation");
- case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
- case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
- case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
- case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
- case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
- case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
- case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
- case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
- case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
- case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
- case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
- case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
- case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
- case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
- case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
+ default:
+ llvm_unreachable("Unknown atomicrmw operation");
+ case AtomicRMWInst::Xchg:
+ NT = ISD::ATOMIC_SWAP;
+ break;
+ case AtomicRMWInst::Add:
+ NT = ISD::ATOMIC_LOAD_ADD;
+ break;
+ case AtomicRMWInst::Sub:
+ NT = ISD::ATOMIC_LOAD_SUB;
+ break;
+ case AtomicRMWInst::And:
+ NT = ISD::ATOMIC_LOAD_AND;
+ break;
+ case AtomicRMWInst::Nand:
+ NT = ISD::ATOMIC_LOAD_NAND;
+ break;
+ case AtomicRMWInst::Or:
+ NT = ISD::ATOMIC_LOAD_OR;
+ break;
+ case AtomicRMWInst::Xor:
+ NT = ISD::ATOMIC_LOAD_XOR;
+ break;
+ case AtomicRMWInst::Max:
+ NT = ISD::ATOMIC_LOAD_MAX;
+ break;
+ case AtomicRMWInst::Min:
+ NT = ISD::ATOMIC_LOAD_MIN;
+ break;
+ case AtomicRMWInst::UMax:
+ NT = ISD::ATOMIC_LOAD_UMAX;
+ break;
+ case AtomicRMWInst::UMin:
+ NT = ISD::ATOMIC_LOAD_UMIN;
+ break;
+ case AtomicRMWInst::FAdd:
+ NT = ISD::ATOMIC_LOAD_FADD;
+ break;
+ case AtomicRMWInst::FSub:
+ NT = ISD::ATOMIC_LOAD_FSUB;
+ break;
+ case AtomicRMWInst::FMax:
+ NT = ISD::ATOMIC_LOAD_FMAX;
+ break;
+ case AtomicRMWInst::FMin:
+ NT = ISD::ATOMIC_LOAD_FMIN;
+ break;
case AtomicRMWInst::FMaximum:
NT = ISD::ATOMIC_LOAD_FMAXIMUM;
break;
@@ -5285,9 +5318,8 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);
SDValue L =
- DAG.getAtomic(NT, dl, MemVT, InChain,
- getValue(I.getPointerOperand()), getValue(I.getValOperand()),
- MMO);
+ DAG.getAtomic(NT, dl, MemVT, InChain, getValue(I.getPointerOperand()),
+ getValue(I.getValOperand()), MMO);
SDValue OutChain = L.getValue(1);
@@ -5402,7 +5434,7 @@ SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands(
// Build the operand list.
SmallVector<SDValue, 8> Ops;
- if (HasChain) { // If this intrinsic has side-effects, chainify it.
+ if (HasChain) { // If this intrinsic has side-effects, chainify it.
if (OnlyLoad) {
// We don't need to serialize loads against other loads.
Ops.push_back(DAG.getRoot());
@@ -5701,8 +5733,8 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
/// limited-precision mode.
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
const TargetLowering &TLI, SDNodeFlags Flags) {
- if (Op.getValueType() == MVT::f32 &&
- LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
+ if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 &&
+ LimitFloatPrecision <= 18) {
// Put the exponent in the right bit position for later addition to the
// final result:
@@ -5725,8 +5757,8 @@ static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
const TargetLowering &TLI, SDNodeFlags Flags) {
// TODO: What fast-math-flags should be set on the floating-point nodes?
- if (Op.getValueType() == MVT::f32 &&
- LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
+ if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 &&
+ LimitFloatPrecision <= 18) {
SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
// Scale the exponent by log(2).
@@ -5824,8 +5856,8 @@ static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
const TargetLowering &TLI, SDNodeFlags Flags) {
// TODO: What fast-math-flags should be set on the floating-point nodes?
- if (Op.getValueType() == MVT::f32 &&
- LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
+ if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 &&
+ LimitFloatPrecision <= 18) {
SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
// Get the exponent.
@@ -5921,8 +5953,8 @@ static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
const TargetLowering &TLI, SDNodeFlags Flags) {
// TODO: What fast-math-flags should be set on the floating-point nodes?
- if (Op.getValueType() == MVT::f32 &&
- LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
+ if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 &&
+ LimitFloatPrecision <= 18) {
SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
// Scale the exponent by log10(2) [0.30102999f].
@@ -6009,8 +6041,8 @@ static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
/// limited-precision mode.
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
const TargetLowering &TLI, SDNodeFlags Flags) {
- if (Op.getValueType() == MVT::f32 &&
- LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
+ if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 &&
+ LimitFloatPrecision <= 18)
return getLimitedPrecisionExp2(Op, dl, DAG);
// No special expansion.
@@ -6099,9 +6131,9 @@ static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
}
-static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
- SDValue LHS, SDValue RHS, SDValue Scale,
- SelectionDAG &DAG, const TargetLowering &TLI) {
+static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS,
+ SDValue RHS, SDValue Scale, SelectionDAG &DAG,
+ const TargetLowering &TLI) {
EVT VT = LHS.getValueType();
bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
@@ -6128,8 +6160,8 @@ static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
if ((ScaleInt > 0 || (Saturating && Signed)) &&
(TLI.isTypeLegal(VT) ||
(VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
- TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
- Opcode, VT, ScaleInt);
+ TargetLowering::LegalizeAction Action =
+ TLI.getFixedPointOperationAction(Opcode, VT, ScaleInt);
if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
EVT PromVT;
if (VT.isScalarInteger())
@@ -6253,8 +6285,8 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
// we should only emit as ArgDbgValue if the Variable is an argument to the
// current function, and the dbg.value intrinsic is found in the entry
// block.
- bool VariableIsFunctionInputArg = Variable->isParameter() &&
- !DL->getInlinedAt();
+ bool VariableIsFunctionInputArg =
+ Variable->isParameter() && !DL->getInlinedAt();
bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
if (!IsInPrologue && !VariableIsFunctionInputArg)
return false;
@@ -6331,7 +6363,7 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
SDValue LCandidate = peekThroughBitcasts(N);
if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
if (FrameIndexSDNode *FINode =
- dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
+ dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
Op = MachineOperand::CreateFI(FINode->getIndex());
}
@@ -6382,8 +6414,8 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
};
// Check if ValueMap has reg number.
- DenseMap<const Value *, Register>::const_iterator
- VMI = FuncInfo.ValueMap.find(V);
+ DenseMap<const Value *, Register>::const_iterator VMI =
+ FuncInfo.ValueMap.find(V);
if (VMI != FuncInfo.ValueMap.end()) {
const auto &TLI = DAG.getTargetLoweringInfo();
RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
@@ -6646,9 +6678,15 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
return;
}
- case Intrinsic::vastart: visitVAStart(I); return;
- case Intrinsic::vaend: visitVAEnd(I); return;
- case Intrinsic::vacopy: visitVACopy(I); return;
+ case Intrinsic::vastart:
+ visitVAStart(I);
+ return;
+ case Intrinsic::vaend:
+ visitVAEnd(I);
+ return;
+ case Intrinsic::vacopy:
+ visitVACopy(I);
+ return;
case Intrinsic::returnaddress:
setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
TLI.getValueType(DAG.getDataLayout(), I.getType()),
@@ -6676,8 +6714,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue RegName =
DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
- Res = DAG.getNode(ISD::READ_REGISTER, sdl,
- DAG.getVTList(VT, MVT::Other), Chain, RegName);
+ Res = DAG.getNode(ISD::READ_REGISTER, sdl, DAG.getVTList(VT, MVT::Other),
+ Chain, RegName);
setValue(&I, Res);
DAG.setRoot(Res.getValue(1));
return;
@@ -6737,11 +6775,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// FIXME: Support passing different dest/src alignments to the memcpy DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol,
- MCI.isForceInlined(), &I, std::nullopt,
- MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata(), BatchAA);
+ SDValue MC = DAG.getMemcpy(
+ Root, sdl, Dst, Src, Size, Alignment, isVol, MCI.isForceInlined(), &I,
+ std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), BatchAA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6758,8 +6795,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
bool isVol = MSII.isVolatile();
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MC = DAG.getMemset(
- Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(),
- &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
+ Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(), &I,
+ MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6868,9 +6905,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::eh_return_i32:
case Intrinsic::eh_return_i64:
DAG.getMachineFunction().setCallsEHReturn(true);
- DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
- MVT::Other,
- getControlRoot(),
+ DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, MVT::Other, getControlRoot(),
getValue(I.getArgOperand(0)),
getValue(I.getArgOperand(1))));
return;
@@ -6893,7 +6928,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// Get and store the index of the function context.
MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
AllocaInst *FnCtx =
- cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
+ cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
int FI = FuncInfo.StaticAllocaMap[FnCtx];
MFI.setFunctionContextIndex(FI);
return;
@@ -6909,12 +6944,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
return;
}
case Intrinsic::eh_sjlj_longjmp:
- DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
- getRoot(), getValue(I.getArgOperand(0))));
+ DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, getRoot(),
+ getValue(I.getArgOperand(0))));
return;
case Intrinsic::eh_sjlj_setup_dispatch:
- DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
- getRoot()));
+ DAG.setRoot(
+ DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, getRoot()));
return;
case Intrinsic::masked_gather:
visitMaskedGather(I);
@@ -7034,8 +7069,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// clang-format on
EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
- setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
- getValue(I.getArgOperand(0))));
+ setValue(&I, DAG.getNode(Opcode, sdl, RetVT, getValue(I.getArgOperand(0))));
return;
}
case Intrinsic::minnum:
@@ -7150,9 +7184,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
SDValue Result;
- Result = DAG.getNode(
- ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
- DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
+ Result =
+ DAG.getNode(ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
+ DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
setValue(&I, Result);
return;
@@ -7474,8 +7508,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Op1 = getValue(I.getArgOperand(0));
SDValue Op2 = getValue(I.getArgOperand(1));
SDValue Op3 = getValue(I.getArgOperand(2));
- setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
- Op1, Op2, Op3, DAG, TLI));
+ setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl, Op1,
+ Op2, Op3, DAG, TLI));
return;
}
case Intrinsic::smax: {
@@ -7536,7 +7570,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
}
case Intrinsic::stackrestore:
Res = getValue(I.getArgOperand(0));
- DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
+ DAG.setRoot(
+ DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
return;
case Intrinsic::get_dynamic_area_offset: {
SDValue Op = getRoot();
@@ -7585,7 +7620,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
if (TLI.useLoadStackGuardNode(M))
Src = getLoadStackGuard(DAG, sdl, Chain);
else
- Src = getValue(I.getArgOperand(0)); // The guard's value.
+ Src = getValue(I.getArgOperand(0)); // The guard's value.
AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
@@ -7679,7 +7714,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::gcwrite:
llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
case Intrinsic::get_rounding:
- Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
+ Res =
+ DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
setValue(&I, Res);
DAG.setRoot(Res.getValue(1));
return;
@@ -7711,7 +7747,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
MVT::i32)));
break;
- default: llvm_unreachable("unknown trap intrinsic");
+ default:
+ llvm_unreachable("unknown trap intrinsic");
}
DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(),
I.hasFnAttr(Attribute::NoMerge));
@@ -7748,13 +7785,26 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
case Intrinsic::smul_with_overflow: {
ISD::NodeType Op;
switch (Intrinsic) {
- default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
- case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
- case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
- case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
- case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
- case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
- case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
+ default:
+ llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ case Intrinsic::uadd_with_overflow:
+ Op = ISD::UADDO;
+ break;
+ case Intrinsic::sadd_with_overflow:
+ Op = ISD::SADDO;
+ break;
+ case Intrinsic::usub_with_overflow:
+ Op = ISD::USUBO;
+ break;
+ case Intrinsic::ssub_with_overflow:
+ Op = ISD::SSUBO;
+ break;
+ case Intrinsic::umul_with_overflow:
+ Op = ISD::UMULO;
+ break;
+ case Intrinsic::smul_with_overflow:
+ Op = ISD::SMULO;
+ break;
}
SDValue Op1 = getValue(I.getArgOperand(0));
SDValue Op2 = getValue(I.getArgOperand(1));
@@ -7767,17 +7817,28 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
return;
}
case Intrinsic::prefetch: {
+ auto *DefaultConstantInt =
+ llvm::ConstantInt::get(llvm::Type::getInt32Ty(*DAG.getContext()), 0);
+ auto GetConstantIntOrDefault = [&](Value *Val) {
+ if (auto *Constant = dyn_cast<ConstantInt>(Val)) {
+ return Constant;
+ }
+ // Cases where Val is non-constant should be rejected by Sema.
+ return DefaultConstantInt;
+ };
+
SDValue Ops[5];
- unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
- auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
+ unsigned rw = GetConstantIntOrDefault(I.getArgOperand(1))->getZExtValue();
+ auto Flags =
+ rw == 0 ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore;
Ops[0] = DAG.getRoot();
Ops[1] = getValue(I.getArgOperand(0));
- Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
- MVT::i32);
- Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
- MVT::i32);
- Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
- MVT::i32);
+ Ops[2] = DAG.getTargetConstant(*GetConstantIntOrDefault(I.getArgOperand(1)),
+ sdl, MVT::i32);
+ Ops[3] = DAG.getTargetConstant(*GetConstantIntOrDefault(I.getArgOperand(2)),
+ sdl, MVT::i32);
+ Ops[4] = DAG.getTargetConstant(*GetConstantIntOrDefault(I.getArgOperand(3)),
+ sdl, MVT::i32);
SDValue Result = DAG.getMemIntrinsicNode(
ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
@@ -7917,8 +7978,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// Create a MCSymbol for the label to avoid any target lowering
// that would make this PC relative.
SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
- SDValue OffsetVal =
- DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
+ SDValue OffsetVal = DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
// Add the offset to the FP.
SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
@@ -8298,10 +8358,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
- SDValue VectorInduction = DAG.getNode(
- ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
- SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
- VectorTripCount, ISD::CondCode::SETULT);
+ SDValue VectorInduction =
+ DAG.getNode(ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
+ SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction, VectorTripCount,
+ ISD::CondCode::SETULT);
setValue(&I, SetCC);
return;
}
@@ -8329,8 +8389,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
CountVT = VT;
}
- SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
- ElementCount::get(VF, IsScalable));
+ SDValue MaxEVL =
+ DAG.getElementCount(sdl, CountVT, ElementCount::get(VF, IsScalable));
SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
// Clip to the result type if needed.
@@ -8598,7 +8658,8 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
unsigned Opcode;
switch (FPI.getIntrinsicID()) {
- default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ default:
+ llvm_unreachable("Impossible intrinsic"); // Can't reach here.
#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
case Intrinsic::INTRINSIC: \
Opcode = ISD::STRICT_##DAGN; \
@@ -8625,7 +8686,8 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
// A few strict DAG nodes carry additional operands that are not
// set up by the default code above.
switch (Opcode) {
- default: break;
+ default:
+ break;
case ISD::STRICT_FP_ROUND:
Opers.push_back(
DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
@@ -8757,7 +8819,7 @@ void SelectionDAGBuilder::visitVPGather(
if (!Alignment)
Alignment = DAG.getEVTAlign(VT.getScalarType());
unsigned AS =
- PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
+ PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand::Flags MMOFlags =
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
@@ -9461,10 +9523,9 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
const Value *Length = I.getArgOperand(2);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
- std::pair<SDValue, SDValue> Res =
- TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
- getValue(Src), getValue(Char), getValue(Length),
- MachinePointerInfo(Src));
+ std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemchr(
+ DAG, getCurSDLoc(), DAG.getRoot(), getValue(Src), getValue(Char),
+ getValue(Length), MachinePointerInfo(Src));
if (Res.first.getNode()) {
setValue(&I, Res.first);
PendingLoads.push_back(Res.second);
@@ -9603,10 +9664,9 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
- std::pair<SDValue, SDValue> Res =
- TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
- getValue(Arg0), getValue(Arg1),
- MachinePointerInfo(Arg0));
+ std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrnlen(
+ DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1),
+ MachinePointerInfo(Arg0));
if (Res.first.getNode()) {
processIntegerCallValue(I, Res.first, false);
PendingLoads.push_back(Res.second);
@@ -9706,7 +9766,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
LibInfo->getLibFunc(*F, Func) && LibInfo->hasOptimizedCodeGen(Func)) {
switch (Func) {
- default: break;
+ default:
+ break;
case LibFunc_bcmp:
if (visitMemCmpBCmpCall(I))
return;
@@ -9936,8 +9997,7 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
RegsForValue AssignedRegs;
explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
- : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
- }
+ : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {}
/// Whether or not this operand accesses memory
bool hasMemory(const TargetLowering &TLI) const {
@@ -9953,7 +10013,6 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
}
};
-
} // end anonymous namespace
/// Make sure that the output operand \p OpInfo and its corresponding input
@@ -10497,15 +10556,14 @@ static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info,
}
// Treat indirect 'X' constraint as memory.
- if (OpInfo.ConstraintType == TargetLowering::C_Other &&
- OpInfo.isIndirect)
+ if (OpInfo.ConstraintType == TargetLowering::C_Other && OpInfo.isIndirect)
OpInfo.ConstraintType = TargetLowering::C_Memory;
if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
OpInfo.ConstraintType == TargetLowering::C_Other) {
std::vector<SDValue> Ops;
TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
- Ops, DAG);
+ Ops, DAG);
if (Ops.empty()) {
if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
if (isa<ConstantSDNode>(InOperandVal)) {
@@ -10839,8 +10897,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
}
void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
- DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
- MVT::Other, getRoot(),
+ DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
DAG.getSrcValue(I.getArgOperand(0))));
}
@@ -10861,15 +10918,13 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
}
void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
- DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
- MVT::Other, getRoot(),
+ DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
DAG.getSrcValue(I.getArgOperand(0))));
}
void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
- DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
- MVT::Other, getRoot(),
+ DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), MVT::Other, getRoot(),
getValue(I.getArgOperand(0)),
getValue(I.getArgOperand(1)),
DAG.getSrcValue(I.getArgOperand(0)),
@@ -10946,8 +11001,7 @@ void SelectionDAGBuilder::populateCallLoweringInfo(
// Populate the argument list.
// Attributes for args start at offset 1, after the return attribute.
- for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
- ArgI != ArgE; ++ArgI) {
+ for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
const Value *V = Call->getOperand(ArgI);
assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
@@ -11084,13 +11138,13 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));
// Handle immediate and symbolic callees.
- if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
+ if (auto *ConstCallee = dyn_cast<ConstantSDNode>(Callee))
Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
/*isTarget=*/true);
- else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
- Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
- SDLoc(SymbolicCallee),
- SymbolicCallee->getValueType(0));
+ else if (auto *SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
+ Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
+ SDLoc(SymbolicCallee),
+ SymbolicCallee->getValueType(0));
// Get the real number of arguments participating in the call <numArgs>
SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));
@@ -11167,7 +11221,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
Ops.push_back(getValue(CB.getArgOperand(i)));
// Push the arguments from the call instruction.
- SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
+ SDNode::op_iterator e = HasGlue ? Call->op_end() - 2 : Call->op_end() - 1;
Ops.append(Call->op_begin() + 2, e);
// Push live variables for the stack map.
@@ -11373,7 +11427,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());
- DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
+ DemoteStackSlot =
+ CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
Entry.IsSRet = true;
Entry.Alignment = Alignment;
@@ -11455,8 +11510,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
}
EVT VT = getValueType(DL, ArgTy);
- SDValue Op = SDValue(Args[i].Node.getNode(),
- Args[i].Node.getResNo() + Value);
+ SDValue Op =
+ SDValue(Args[i].Node.getNode(), Args[i].Node.getResNo() + Value);
ISD::ArgFlagsTy Flags;
// Certain targets (such as MIPS), may have a different ABI alignment
@@ -11554,8 +11609,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
else if (Args[i].IsZExt)
ExtendKind = ISD::ZERO_EXTEND;
- // Conservatively only handle 'returned' on non-vectors that can be lowered,
- // for now.
+ // Conservatively only handle 'returned' on non-vectors that can be
+ // lowered, for now.
if (Args[i].IsReturned && !Op.getValueType().isVector() &&
CanLowerReturn) {
assert((CLI.RetTy == Args[i].Ty ||
@@ -11719,7 +11774,7 @@ void TargetLowering::LowerOperationWrapper(SDNode *N,
// If the original node has multiple results, then the return node should
// have the same number of results.
assert((N->getNumValues() == Res->getNumValues()) &&
- "Lowering returned the wrong number of results!");
+ "Lowering returned the wrong number of results!");
// Places new result values base on N result number.
for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
@@ -11770,7 +11825,7 @@ static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
const BasicBlock &Entry = A->getParent()->front();
for (const User *U : A->users())
if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
- return false; // Use not in entry block.
+ return false; // Use not in entry block.
return true;
}
@@ -11782,10 +11837,9 @@ using ArgCopyElisionMapTy =
/// Scan the entry block of the function in FuncInfo for arguments that look
/// like copies into a local alloca. Record any copied arguments in
/// ArgCopyElisionCandidates.
-static void
-findArgumentCopyElisionCandidates(const DataLayout &DL,
- FunctionLoweringInfo *FuncInfo,
- ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
+static void findArgumentCopyElisionCandidates(
+ const DataLayout &DL, FunctionLoweringInfo *FuncInfo,
+ ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
// Record the state of every static alloca used in the entry block. Argument
// allocas are all used in the entry block, so we need approximately as many
// entries as we have arguments.
@@ -12098,9 +12152,9 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
// For scalable vectors, use the minimum size; individual targets
// are responsible for handling scalable vector arguments and
// return values.
- ISD::InputArg MyFlags(
- Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
- i * RegisterVT.getStoreSize().getKnownMinValue());
+ ISD::InputArg MyFlags(Flags, RegisterVT, VT, ArgTy, isArgValueUsed,
+ ArgNo,
+ i * RegisterVT.getStoreSize().getKnownMinValue());
if (NumRegs > 1 && i == 0)
MyFlags.Flags.setSplit();
// if it isn't first piece, alignment must be 1
@@ -12144,8 +12198,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
F.getCallingConv(), AssertOp);
- MachineFunction& MF = SDB->DAG.getMachineFunction();
- MachineRegisterInfo& RegInfo = MF.getRegInfo();
+ MachineFunction &MF = SDB->DAG.getMachineFunction();
+ MachineRegisterInfo &RegInfo = MF.getRegInfo();
Register SRetReg =
RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
FuncInfo->DemoteRegister = SRetReg;
@@ -12182,17 +12236,16 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
ArrayRef(&InVals[i], NumParts), ArgHasUses);
}
- // If this argument is unused then remember its value. It is used to generate
- // debugging information.
+ // If this argument is unused then remember its value. It is used to
+ // generate debugging information.
bool isSwiftErrorArg =
- TLI->supportSwiftError() &&
- Arg.hasAttribute(Attribute::SwiftError);
+ TLI->supportSwiftError() && Arg.hasAttribute(Attribute::SwiftError);
if (!ArgHasUses && !isSwiftErrorArg) {
SDB->setUnusedArgValue(&Arg, InVals[i]);
// Also remember any frame index for use in FastISel.
if (FrameIndexSDNode *FI =
- dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
+ dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
}
@@ -12236,7 +12289,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
// Note down frame index.
if (FrameIndexSDNode *FI =
- dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
+ dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
@@ -12253,9 +12306,9 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
// significant bits will be in the second operand.
unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
if (LoadSDNode *LNode =
- dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
+ dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
if (FrameIndexSDNode *FI =
- dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
+ dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
}
@@ -12319,8 +12372,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
/// directly add them, because expansion might result in multiple MBB's for one
/// BB. As such, the start of the BB might correspond to a different MBB than
/// the end.
-void
-SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
+void SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(
+ const BasicBlock *LLVMBB) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
@@ -12328,7 +12381,8 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
// Check PHI nodes in successors that expect a value to be available from this
// block.
for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
- if (!isa<PHINode>(SuccBB->begin())) continue;
+ if (!isa<PHINode>(SuccBB->begin()))
+ continue;
MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
// If this terminator has multiple identical successors (common for
@@ -12384,7 +12438,8 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
SmallVector<EVT, 4> ValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
for (EVT VT : ValueVTs) {
- const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
+ const unsigned NumRegisters =
+ TLI.getNumRegisters(*DAG.getContext(), VT);
for (unsigned i = 0; i != NumRegisters; ++i)
FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i);
Reg += NumRegisters;
@@ -12490,14 +12545,14 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
// as a tie-breaker as clusters are guaranteed to never overlap.
llvm::sort(W.FirstCluster, W.LastCluster + 1,
[](const CaseCluster &a, const CaseCluster &b) {
- return a.Prob != b.Prob ?
- a.Prob > b.Prob :
- a.Low->getValue().slt(b.Low->getValue());
- });
+ return a.Prob != b.Prob
+ ? a.Prob > b.Prob
+ : a.Low->getValue().slt(b.Low->getValue());
+ });
// Rearrange the case blocks so that the last one falls through if possible
// without changing the order of probabilities.
- for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
+ for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
--I;
if (I->Prob > W.LastCluster->Prob)
break;
@@ -12526,136 +12581,137 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
} else {
Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
CurMF->insert(BBI, Fallthrough);
- // Put Cond in a virtual register to make it available from the new blocks.
+ // Put Cond in a virtual register to make it available from the new
+ // blocks.
ExportFromCurrentBlock(Cond);
}
UnhandledProbs -= I->Prob;
switch (I->Kind) {
- case CC_JumpTable: {
- // FIXME: Optimize away range check based on pivot comparisons.
- JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
- SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
-
- // The jump block hasn't been inserted yet; insert it here.
- MachineBasicBlock *JumpMBB = JT->MBB;
- CurMF->insert(BBI, JumpMBB);
-
- auto JumpProb = I->Prob;
- auto FallthroughProb = UnhandledProbs;
-
- // If the default statement is a target of the jump table, we evenly
- // distribute the default probability to successors of CurMBB. Also
- // update the probability on the edge from JumpMBB to Fallthrough.
- for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
- SE = JumpMBB->succ_end();
- SI != SE; ++SI) {
- if (*SI == DefaultMBB) {
- JumpProb += DefaultProb / 2;
- FallthroughProb -= DefaultProb / 2;
- JumpMBB->setSuccProbability(SI, DefaultProb / 2);
- JumpMBB->normalizeSuccProbs();
- break;
- }
+ case CC_JumpTable: {
+ // FIXME: Optimize away range check based on pivot comparisons.
+ JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
+ SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
+
+ // The jump block hasn't been inserted yet; insert it here.
+ MachineBasicBlock *JumpMBB = JT->MBB;
+ CurMF->insert(BBI, JumpMBB);
+
+ auto JumpProb = I->Prob;
+ auto FallthroughProb = UnhandledProbs;
+
+ // If the default statement is a target of the jump table, we evenly
+ // distribute the default probability to successors of CurMBB. Also
+ // update the probability on the edge from JumpMBB to Fallthrough.
+ for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
+ SE = JumpMBB->succ_end();
+ SI != SE; ++SI) {
+ if (*SI == DefaultMBB) {
+ JumpProb += DefaultProb / 2;
+ FallthroughProb -= DefaultProb / 2;
+ JumpMBB->setSuccProbability(SI, DefaultProb / 2);
+ JumpMBB->normalizeSuccProbs();
+ break;
}
+ }
- // If the default clause is unreachable, propagate that knowledge into
- // JTH->FallthroughUnreachable which will use it to suppress the range
- // check.
- //
- // However, don't do this if we're doing branch target enforcement,
- // because a table branch _without_ a range check can be a tempting JOP
- // gadget - out-of-bounds inputs that are impossible in correct
- // execution become possible again if an attacker can influence the
- // control flow. So if an attacker doesn't already have a BTI bypass
- // available, we don't want them to be able to get one out of this
- // table branch.
- if (FallthroughUnreachable) {
- Function &CurFunc = CurMF->getFunction();
- if (!CurFunc.hasFnAttribute("branch-target-enforcement"))
- JTH->FallthroughUnreachable = true;
- }
+ // If the default clause is unreachable, propagate that knowledge into
+ // JTH->FallthroughUnreachable which will use it to suppress the range
+ // check.
+ //
+ // However, don't do this if we're doing branch target enforcement,
+ // because a table branch _without_ a range check can be a tempting JOP
+ // gadget - out-of-bounds inputs that are impossible in correct
+ // execution become possible again if an attacker can influence the
+ // control flow. So if an attacker doesn't already have a BTI bypass
+ // available, we don't want them to be able to get one out of this
+ // table branch.
+ if (FallthroughUnreachable) {
+ Function &CurFunc = CurMF->getFunction();
+ if (!CurFunc.hasFnAttribute("branch-target-enforcement"))
+ JTH->FallthroughUnreachable = true;
+ }
- if (!JTH->FallthroughUnreachable)
- addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
- addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
- CurMBB->normalizeSuccProbs();
+ if (!JTH->FallthroughUnreachable)
+ addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
+ addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
+ CurMBB->normalizeSuccProbs();
- // The jump table header will be inserted in our current block, do the
- // range check, and fall through to our fallthrough block.
- JTH->HeaderBB = CurMBB;
- JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
+ // The jump table header will be inserted in our current block, do the
+ // range check, and fall through to our fallthrough block.
+ JTH->HeaderBB = CurMBB;
+ JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
- // If we're in the right place, emit the jump table header right now.
- if (CurMBB == SwitchMBB) {
- visitJumpTableHeader(*JT, *JTH, SwitchMBB);
- JTH->Emitted = true;
- }
- break;
+ // If we're in the right place, emit the jump table header right now.
+ if (CurMBB == SwitchMBB) {
+ visitJumpTableHeader(*JT, *JTH, SwitchMBB);
+ JTH->Emitted = true;
+ }
+ break;
+ }
+ case CC_BitTests: {
+ // FIXME: Optimize away range check based on pivot comparisons.
+ BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
+
+ // The bit test blocks haven't been inserted yet; insert them here.
+ for (BitTestCase &BTC : BTB->Cases)
+ CurMF->insert(BBI, BTC.ThisBB);
+
+ // Fill in fields of the BitTestBlock.
+ BTB->Parent = CurMBB;
+ BTB->Default = Fallthrough;
+
+ BTB->DefaultProb = UnhandledProbs;
+ // If the cases in bit test don't form a contiguous range, we evenly
+ // distribute the probability on the edge to Fallthrough to two
+ // successors of CurMBB.
+ if (!BTB->ContiguousRange) {
+ BTB->Prob += DefaultProb / 2;
+ BTB->DefaultProb -= DefaultProb / 2;
}
- case CC_BitTests: {
- // FIXME: Optimize away range check based on pivot comparisons.
- BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
-
- // The bit test blocks haven't been inserted yet; insert them here.
- for (BitTestCase &BTC : BTB->Cases)
- CurMF->insert(BBI, BTC.ThisBB);
-
- // Fill in fields of the BitTestBlock.
- BTB->Parent = CurMBB;
- BTB->Default = Fallthrough;
-
- BTB->DefaultProb = UnhandledProbs;
- // If the cases in bit test don't form a contiguous range, we evenly
- // distribute the probability on the edge to Fallthrough to two
- // successors of CurMBB.
- if (!BTB->ContiguousRange) {
- BTB->Prob += DefaultProb / 2;
- BTB->DefaultProb -= DefaultProb / 2;
- }
- if (FallthroughUnreachable)
- BTB->FallthroughUnreachable = true;
+ if (FallthroughUnreachable)
+ BTB->FallthroughUnreachable = true;
- // If we're in the right place, emit the bit test header right now.
- if (CurMBB == SwitchMBB) {
- visitBitTestHeader(*BTB, SwitchMBB);
- BTB->Emitted = true;
- }
- break;
+ // If we're in the right place, emit the bit test header right now.
+ if (CurMBB == SwitchMBB) {
+ visitBitTestHeader(*BTB, SwitchMBB);
+ BTB->Emitted = true;
+ }
+ break;
+ }
+ case CC_Range: {
+ const Value *RHS, *LHS, *MHS;
+ ISD::CondCode CC;
+ if (I->Low == I->High) {
+ // Check Cond == I->Low.
+ CC = ISD::SETEQ;
+ LHS = Cond;
+ RHS = I->Low;
+ MHS = nullptr;
+ } else {
+ // Check I->Low <= Cond <= I->High.
+ CC = ISD::SETLE;
+ LHS = I->Low;
+ MHS = Cond;
+ RHS = I->High;
}
- case CC_Range: {
- const Value *RHS, *LHS, *MHS;
- ISD::CondCode CC;
- if (I->Low == I->High) {
- // Check Cond == I->Low.
- CC = ISD::SETEQ;
- LHS = Cond;
- RHS=I->Low;
- MHS = nullptr;
- } else {
- // Check I->Low <= Cond <= I->High.
- CC = ISD::SETLE;
- LHS = I->Low;
- MHS = Cond;
- RHS = I->High;
- }
- // If Fallthrough is unreachable, fold away the comparison.
- if (FallthroughUnreachable)
- CC = ISD::SETTRUE;
+ // If Fallthrough is unreachable, fold away the comparison.
+ if (FallthroughUnreachable)
+ CC = ISD::SETTRUE;
- // The false probability is the sum of all unhandled cases.
- CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
- getCurSDLoc(), I->Prob, UnhandledProbs);
+ // The false probability is the sum of all unhandled cases.
+ CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
+ getCurSDLoc(), I->Prob, UnhandledProbs);
- if (CurMBB == SwitchMBB)
- visitSwitchCase(CB, SwitchMBB);
- else
- SL->SwitchCases.push_back(CB);
+ if (CurMBB == SwitchMBB)
+ visitSwitchCase(CB, SwitchMBB);
+ else
+ SL->SwitchCases.push_back(CB);
- break;
- }
+ break;
+ }
}
CurMBB = Fallthrough;
}
@@ -12708,8 +12764,8 @@ void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
// single cluster, RHS.Low == Pivot, and we can branch to its destination
// directly if RHS.High equals the current upper bound.
MachineBasicBlock *RightMBB;
- if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
- W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
+ if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
+ (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
RightMBB = FirstRight->MBB;
} else {
RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
@@ -12996,7 +13052,8 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
ValueVTs);
unsigned NumValues = ValueVTs.size();
- if (NumValues == 0) return;
+ if (NumValues == 0)
+ return;
SmallVector<SDValue, 4> Values(NumValues);
SDValue Op = getValue(I.getOperand(0));
More information about the cfe-commits
mailing list