[llvm] f270a4d - [AArch64] Don't tail call memset if it would convert to a bzero. (#98969)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 17 01:31:56 PDT 2024
Author: Amara Emerson
Date: 2024-07-17T01:31:52-07:00
New Revision: f270a4dd6667759d7305797a077ae09648318ac7
URL: https://github.com/llvm/llvm-project/commit/f270a4dd6667759d7305797a077ae09648318ac7
DIFF: https://github.com/llvm/llvm-project/commit/f270a4dd6667759d7305797a077ae09648318ac7.diff
LOG: [AArch64] Don't tail call memset if it would convert to a bzero. (#98969)
Well, not quite that simple. We can tc memset since it returns the first
argument but bzero doesn't do that and therefore we can end up
miscompiling.
This patch also refactors the logic out of isInTailCallPosition() into the callers.
As a result memcpy and memmove are also modified to do the same thing
for consistency.
rdar://131419786
Added:
llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
Modified:
llvm/include/llvm/CodeGen/Analysis.h
llvm/include/llvm/CodeGen/SelectionDAG.h
llvm/lib/CodeGen/Analysis.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
llvm/lib/Target/Lanai/LanaiISelLowering.cpp
llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
llvm/lib/Target/Mips/MipsISelLowering.cpp
llvm/lib/Target/PowerPC/PPCISelLowering.cpp
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/Sparc/SparcISelLowering.cpp
llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
llvm/lib/Target/X86/X86ISelLowering.cpp
llvm/lib/Target/X86/X86ISelLoweringCall.cpp
llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
llvm/lib/Target/XCore/XCoreISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 6f7ed22b8ac71..362cc30bbd06a 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -126,7 +126,8 @@ ICmpInst::Predicate getICmpCondCode(ISD::CondCode Pred);
/// between it and the return.
///
/// This function only tests target-independent requirements.
-bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM);
+bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM,
+ bool ReturnsFirstArg = false);
/// Test if given that the input instruction is in the tail call position, if
/// there is an attribute mismatch between the caller and the callee that will
@@ -144,7 +145,12 @@ bool attributesPermitTailCall(const Function *F, const Instruction *I,
/// optimization.
bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
const ReturnInst *Ret,
- const TargetLoweringBase &TLI);
+ const TargetLoweringBase &TLI,
+ bool ReturnsFirstArg = false);
+
+/// Returns true if the parent of \p CI returns CI's first argument after
+/// calling \p CI.
+bool funcReturnsFirstArgOfCall(const CallInst &CI);
DenseMap<const MachineBasicBlock *, int>
getEHScopeMembership(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 8e189e9e8bf86..16ec65f2e7daa 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1182,16 +1182,22 @@ class SelectionDAG {
/// stack arguments from being clobbered.
SDValue getStackArgumentTokenFactor(SDValue Chain);
- SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, bool isTailCall,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo = AAMDNodes(),
- AAResults *AA = nullptr);
-
+ /* \p CI if not null is the memset call being lowered.
+ * \p OverrideTailCall is an optional parameter that can be used to override
+ * the tail call optimization decision. */
+ SDValue
+ getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
+ SDValue Size, Align Alignment, bool isVol, bool AlwaysInline,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
+ MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo = AAMDNodes(), AAResults *AA = nullptr);
+
+ /* \p CI if not null is the memset call being lowered.
+ * \p OverrideTailCall is an optional parameter that can be used to override
+ * the tail call optimization decision. */
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol, bool isTailCall,
+ SDValue Size, Align Alignment, bool isVol,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1199,7 +1205,7 @@ class SelectionDAG {
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, bool isTailCall,
+ bool AlwaysInline, const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes());
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 7fc18639e5852..128060ec912c7 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -532,7 +532,8 @@ static bool nextRealType(SmallVectorImpl<Type *> &SubTypes,
/// between it and the return.
///
/// This function only tests target-independent requirements.
-bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM) {
+bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM,
+ bool ReturnsFirstArg) {
const BasicBlock *ExitBB = Call.getParent();
const Instruction *Term = ExitBB->getTerminator();
const ReturnInst *Ret = dyn_cast<ReturnInst>(Term);
@@ -575,7 +576,8 @@ bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM) {
const Function *F = ExitBB->getParent();
return returnTypeIsEligibleForTailCall(
- F, &Call, Ret, *TM.getSubtargetImpl(*F)->getTargetLowering());
+ F, &Call, Ret, *TM.getSubtargetImpl(*F)->getTargetLowering(),
+ ReturnsFirstArg);
}
bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
@@ -638,26 +640,11 @@ bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
return CallerAttrs == CalleeAttrs;
}
-/// Check whether B is a bitcast of a pointer type to another pointer type,
-/// which is equal to A.
-static bool isPointerBitcastEqualTo(const Value *A, const Value *B) {
- assert(A && B && "Expected non-null inputs!");
-
- auto *BitCastIn = dyn_cast<BitCastInst>(B);
-
- if (!BitCastIn)
- return false;
-
- if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
- return false;
-
- return A == BitCastIn->getOperand(0);
-}
-
bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
const Instruction *I,
const ReturnInst *Ret,
- const TargetLoweringBase &TLI) {
+ const TargetLoweringBase &TLI,
+ bool ReturnsFirstArg) {
// If the block ends with a void return or unreachable, it doesn't matter
// what the call's return type is.
if (!Ret || Ret->getNumOperands() == 0) return true;
@@ -671,26 +658,11 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
if (!attributesPermitTailCall(F, I, Ret, TLI, &AllowDifferingSizes))
return false;
- const Value *RetVal = Ret->getOperand(0), *CallVal = I;
- // Intrinsic like llvm.memcpy has no return value, but the expanded
- // libcall may or may not have return value. On most platforms, it
- // will be expanded as memcpy in libc, which returns the first
- // argument. On other platforms like arm-none-eabi, memcpy may be
- // expanded as library call without return value, like __aeabi_memcpy.
- const CallInst *Call = cast<CallInst>(I);
- if (Function *F = Call->getCalledFunction()) {
- Intrinsic::ID IID = F->getIntrinsicID();
- if (((IID == Intrinsic::memcpy &&
- TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) ||
- (IID == Intrinsic::memmove &&
- TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove")) ||
- (IID == Intrinsic::memset &&
- TLI.getLibcallName(RTLIB::MEMSET) == StringRef("memset"))) &&
- (RetVal == Call->getArgOperand(0) ||
- isPointerBitcastEqualTo(RetVal, Call->getArgOperand(0))))
- return true;
- }
+ // If the return value is the first argument of the call.
+ if (ReturnsFirstArg)
+ return true;
+ const Value *RetVal = Ret->getOperand(0), *CallVal = I;
SmallVector<unsigned, 4> RetPath, CallPath;
SmallVector<Type *, 4> RetSubTypes, CallSubTypes;
@@ -739,6 +711,15 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
return true;
}
+bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
+ const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
+ Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
+ bool ReturnsFirstArg = false;
+ if (RetVal && ((RetVal == CI.getArgOperand(0))))
+ ReturnsFirstArg = true;
+ return ReturnsFirstArg;
+}
+
static void collectEHScopeMembers(
DenseMap<const MachineBasicBlock *, int> &EHScopeMembership, int EHScope,
const MachineBasicBlock *MBB) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9bd0d1c51fbc2..94349ec97693f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -75,6 +75,7 @@
#include <cstdint>
#include <cstdlib>
#include <limits>
+#include <optional>
#include <set>
#include <string>
#include <utility>
@@ -8236,12 +8237,11 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
}
}
-SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
- SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool AlwaysInline, bool isTailCall,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo, AAResults *AA) {
+SDValue SelectionDAG::getMemcpy(
+ SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
+ Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
+ std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
+ MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -8296,6 +8296,18 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
+ bool IsTailCall = false;
+ if (OverrideTailCall.has_value()) {
+ IsTailCall = *OverrideTailCall;
+ } else {
+ bool LowersToMemcpy =
+ TLI->getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy");
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+ IsTailCall = CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(),
+ ReturnsFirstArg && LowersToMemcpy);
+ }
+
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
@@ -8304,7 +8316,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
TLI->getPointerTy(getDataLayout())),
std::move(Args))
.setDiscardResult()
- .setTailCall(isTailCall);
+ .setTailCall(IsTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
@@ -8352,7 +8364,8 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool isTailCall,
+ bool isVol, const CallInst *CI,
+ std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo, AAResults *AA) {
@@ -8398,6 +8411,19 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
+
+ bool IsTailCall = false;
+ if (OverrideTailCall.has_value()) {
+ IsTailCall = *OverrideTailCall;
+ } else {
+ bool LowersToMemmove =
+ TLI->getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove");
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+ IsTailCall = CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(),
+ ReturnsFirstArg && LowersToMemmove);
+ }
+
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
@@ -8406,7 +8432,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
TLI->getPointerTy(getDataLayout())),
std::move(Args))
.setDiscardResult()
- .setTailCall(isTailCall);
+ .setTailCall(IsTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
@@ -8454,7 +8480,8 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool AlwaysInline, bool isTailCall,
+ bool isVol, bool AlwaysInline,
+ const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
@@ -8514,8 +8541,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
return Entry;
};
+ bool UseBZero = isNullConstant(Src) && BzeroName;
// If zeroing out and bzero is present, use it.
- if (isNullConstant(Src) && BzeroName) {
+ if (UseBZero) {
TargetLowering::ArgListTy Args;
Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
@@ -8533,8 +8561,16 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
TLI->getPointerTy(DL)),
std::move(Args));
}
-
- CLI.setDiscardResult().setTailCall(isTailCall);
+ bool LowersToMemset =
+ TLI->getLibcallName(RTLIB::MEMSET) == StringRef("memset");
+ // If we're going to use bzero, make sure not to tail call unless the
+ // subsequent return doesn't need a value, as bzero doesn't return the first
+ // arg unlike memset.
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI) && !UseBZero;
+ bool IsTailCall =
+ CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(), ReturnsFirstArg && LowersToMemset);
+ CLI.setDiscardResult().setTailCall(IsTailCall);
std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 51cbdd9b3ad31..d5cbb733a408d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6460,14 +6460,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Align SrcAlign = MCI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MCI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing
diff erent dest/src alignments to the memcpy DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MC = DAG.getMemcpy(
- Root, sdl, Op1, Op2, Op3, Alignment, isVol,
- /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
+ SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
+ /* AlwaysInline */ false, &I, std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)),
+ I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6482,13 +6482,13 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Align SrcAlign = MCI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MCI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing
diff erent dest/src alignments to the memcpy DAG
// node.
- SDValue MC = DAG.getMemcpy(
- getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
- /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
+ SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
+ /* AlwaysInline */ true, &I, std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)),
+ I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6500,11 +6500,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// @llvm.memset defines 0 and 1 to both mean no alignment.
Align Alignment = MSI.getDestAlign().valueOrOne();
bool isVol = MSI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MS = DAG.getMemset(
Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
- isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
+ &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
updateDAGForMaybeTailCall(MS);
return;
}
@@ -6517,10 +6516,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// @llvm.memset defines 0 and 1 to both mean no alignment.
Align DstAlign = MSII.getDestAlign().valueOrOne();
bool isVol = MSII.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
- /* AlwaysInline */ true, isTC,
+ /* AlwaysInline */ true, &I,
MachinePointerInfo(I.getArgOperand(0)),
I.getAAMetadata());
updateDAGForMaybeTailCall(MC);
@@ -6536,12 +6534,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Align SrcAlign = MMI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MMI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing
diff erent dest/src alignments to the memmove DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
- isTC, MachinePointerInfo(I.getArgOperand(0)),
+ SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,
+ /* OverrideTailCall */ std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)),
I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MM);
@@ -9039,11 +9037,10 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
// because the return pointer needs to be adjusted by the size of
// the copied memory.
SDValue Root = getMemoryRoot();
- SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
- /*isTailCall=*/false,
- MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata());
+ SDValue MC = DAG.getMemcpy(
+ Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,
+ std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
assert(MC.getNode() != nullptr &&
"** memcpy should not be lowered as TailCall in mempcpy context **");
DAG.setRoot(MC);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index df9b0ae1a632f..eef83a845e2c3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8578,7 +8578,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain, DL, DstAddr, Arg, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ false,
- /*isTailCall = */ false, DstInfo, MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, DstInfo, MachinePointerInfo());
MemOpChains.push_back(Cpy);
} else {
@@ -10878,8 +10878,9 @@ SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
DAG.getConstant(VaListSize, DL, MVT::i32),
- Align(PtrSize), false, false, false,
- MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
+ Align(PtrSize), false, false, /*CI=*/nullptr,
+ std::nullopt, MachinePointerInfo(DestSV),
+ MachinePointerInfo(SrcSV));
}
SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index bb8e21772e566..a79d8f7bd1b5e 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3822,7 +3822,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ true,
- /*isTailCall = */ false, DstInfo,
+ /*CI=*/nullptr, std::nullopt, DstInfo,
MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
MemOpChains.push_back(Cpy);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 79cffc0da7a4f..7aeaebc584c64 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -178,7 +178,7 @@ static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
return DAG.getMemcpy(
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
/*isVolatile=*/false, /*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
bool
@@ -1038,10 +1038,10 @@ HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
SDLoc DL(Op);
// Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
// we need to memcopy 12 bytes from va_list to another similar list.
- return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
- DAG.getIntPtrConstant(12, DL), Align(4),
- /*isVolatile*/ false, false, false,
- MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
+ return DAG.getMemcpy(
+ Chain, DL, DestPtr, SrcPtr, DAG.getIntPtrConstant(12, DL), Align(4),
+ /*isVolatile*/ false, false, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
}
SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 06fd7ac807d8a..f6763a35cc0d5 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -648,7 +648,7 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
/*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(),
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 79da36c03e304..ba6be85c7f2e8 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -4225,7 +4225,7 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
- /*AlwaysInline=*/false, /*isTailCall=*/IsTailCall,
+ /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt,
MachinePointerInfo(), MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index fc066f001316d..ba7b6c85bd81a 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -864,11 +864,12 @@ SDValue MSP430TargetLowering::LowerCCCCallTo(
if (Flags.isByVal()) {
SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i16);
- MemOp = DAG.getMemcpy(
- Chain, dl, PtrOff, Arg, SizeNode, Flags.getNonZeroByValAlign(),
- /*isVolatile*/ false,
- /*AlwaysInline=*/true,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ MemOp = DAG.getMemcpy(Chain, dl, PtrOff, Arg, SizeNode,
+ Flags.getNonZeroByValAlign(),
+ /*isVolatile*/ false,
+ /*AlwaysInline=*/true,
+ /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo());
} else {
MemOp = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
}
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index ef70ef2772681..0f2047fcac640 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -4506,7 +4506,7 @@ void MipsTargetLowering::passByValArg(
Chain = DAG.getMemcpy(
Chain, DL, Dst, Src, DAG.getConstant(MemCpySize, DL, PtrTy),
Align(Alignment), /*isVolatile=*/false, /*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
MemOpChains.push_back(Chain);
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 411114599543c..a11ab93b8db3c 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3904,8 +3904,8 @@ SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
// 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2),
DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8),
- false, true, false, MachinePointerInfo(),
- MachinePointerInfo());
+ false, true, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo());
}
SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
@@ -5275,9 +5275,9 @@ static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
SDValue Chain, ISD::ArgFlagsTy Flags,
SelectionDAG &DAG, const SDLoc &dl) {
SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
- return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode,
- Flags.getNonZeroByValAlign(), false, false, false,
- MachinePointerInfo(), MachinePointerInfo());
+ return DAG.getMemcpy(
+ Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), false, false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 953196a586b6e..fef1441eca9c6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19932,7 +19932,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
- /*AlwaysInline=*/false, IsTailCall,
+ /*AlwaysInline=*/false, /*CI*/ nullptr, IsTailCall,
MachinePointerInfo(), MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 0dba6c47be030..50aa19446f880 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -868,8 +868,8 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
Chain = DAG.getMemcpy(Chain, dl, FIPtr, Arg, SizeNode, Alignment,
false, // isVolatile,
(Size <= 32), // AlwaysInline if size <= 32,
- false, // isTailCall
- MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
else {
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 0a7229a2bc0fb..b2b88143354a5 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -3951,7 +3951,7 @@ SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(0) : 32;
return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL),
Align(8), /*isVolatile*/ false, /*AlwaysInline*/ false,
- /*isTailCall*/ false, MachinePointerInfo(DstSV),
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
MachinePointerInfo(SrcSV));
}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index a793e59c3d1a7..f77076d7244ca 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1123,10 +1123,11 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
SDValue SizeNode =
DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
- Chain = DAG.getMemcpy(
- Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
- /*isVolatile*/ false, /*AlwaysInline=*/false,
- /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
+ Chain = DAG.getMemcpy(Chain, DL, FINode, OutVal, SizeNode,
+ Out.Flags.getNonZeroByValAlign(),
+ /*isVolatile*/ false, /*AlwaysInline=*/false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
OutVal = FINode;
}
// Count the number of fixed args *after* legalization.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9d651d4db6731..bcdd31c22d314 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -25147,7 +25147,8 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
Chain, DL, DstPtr, SrcPtr,
DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false,
- false, MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
+ MachinePointerInfo(SrcSV));
}
// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index ed3fb13b2b232..3c49b6ea8ec7b 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -1240,7 +1240,7 @@ static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
return DAG.getMemcpy(
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
/*isVolatile*/ false, /*AlwaysInline=*/true,
- /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
/// Return true if the calling convention is one that we can guarantee TCO for.
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index e5f07f230fe6c..055466ac660cc 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -147,7 +147,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
DAG.getConstant(Offset, dl, AddrVT)),
Val, DAG.getConstant(BytesLeft, dl, SizeVT), Alignment,
isVolatile, AlwaysInline,
- /* isTailCall */ false, DstPtrInfo.getWithOffset(Offset)));
+ /* CI */ nullptr, DstPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
@@ -255,7 +255,7 @@ static SDValue emitConstantSizeRepmov(
DAG.getNode(ISD::ADD, dl, DstVT, Dst, DAG.getConstant(Offset, dl, DstVT)),
DAG.getNode(ISD::ADD, dl, SrcVT, Src, DAG.getConstant(Offset, dl, SrcVT)),
DAG.getConstant(BytesLeft, dl, SizeVT), Alignment, isVolatile,
- /*AlwaysInline*/ true, /*isTailCall*/ false,
+ /*AlwaysInline*/ true, /*CI=*/nullptr, std::nullopt,
DstPtrInfo.getWithOffset(Offset), SrcPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index c81da0365af3d..f4d32833bce99 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1301,8 +1301,8 @@ SDValue XCoreTargetLowering::LowerCCCArguments(
InVals.push_back(FIN);
MemOps.push_back(DAG.getMemcpy(
Chain, dl, FIN, ArgDI.SDV, DAG.getConstant(Size, dl, MVT::i32),
- Alignment, false, false, false, MachinePointerInfo(),
- MachinePointerInfo()));
+ Alignment, false, false, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo()));
} else {
InVals.push_back(ArgDI.SDV);
}
@@ -1704,7 +1704,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
bool isTail = isInTailCallPosition(DAG, ST, Chain);
return DAG.getMemmove(Chain, dl, ST->getBasePtr(), LD->getBasePtr(),
DAG.getConstant(StoreBits / 8, dl, MVT::i32),
- Alignment, false, isTail,
+ Alignment, false, nullptr, isTail,
ST->getPointerInfo(), LD->getPointerInfo());
}
}
diff --git a/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
new file mode 100644
index 0000000000000..34c6c63cc1798
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
@@ -0,0 +1,20 @@
+; RUN: llc -o - %s | FileCheck %s
+; RUN: llc -global-isel -o - %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx15.0.0"
+
+define ptr @test() {
+; CHECK-LABEL: test:
+; CHECK: bl _bzero
+ %1 = tail call ptr @fn(i32 noundef 1) #3
+ tail call void @llvm.memset.p0.i64(ptr noundef nonnull align 1 dereferenceable(1000) %1, i8 noundef 0, i64 noundef 1000, i1 noundef false) #3
+ ret ptr %1
+}
+
+declare ptr @fn(i32 noundef)
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write)
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
+
+attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: write) }
+attributes #3 = { nounwind optsize }
More information about the llvm-commits
mailing list