[llvm] [AArch64] Don't tail call memset if it would convert to a bzero. (PR #98969)
Amara Emerson via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 16 17:16:24 PDT 2024
https://github.com/aemerson updated https://github.com/llvm/llvm-project/pull/98969
>From f9b342324834480eb3403892af205f202300f1b1 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Mon, 15 Jul 2024 14:48:08 -0700
Subject: [PATCH 1/6] [AArch64] Don't tail call memset if it would convert to a
bzero.
Well, not quite that simple. We can tc memset since it returns the first
argument but bzero doesn't do that and therefore we can end up miscompiling.
rdar://131419786
---
llvm/lib/CodeGen/Analysis.cpp | 7 ++++++-
.../AArch64/no-tail-call-bzero-from-memset.ll | 20 +++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 7fc18639e5852..2a3015866da5f 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -677,6 +677,8 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
// will be expanded as memcpy in libc, which returns the first
// argument. On other platforms like arm-none-eabi, memcpy may be
// expanded as library call without return value, like __aeabi_memcpy.
+ // Similarly, llvm.memset can be expanded to bzero, which doesn't have a
+ // return value either.
const CallInst *Call = cast<CallInst>(I);
if (Function *F = Call->getCalledFunction()) {
Intrinsic::ID IID = F->getIntrinsicID();
@@ -685,7 +687,10 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
(IID == Intrinsic::memmove &&
TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove")) ||
(IID == Intrinsic::memset &&
- TLI.getLibcallName(RTLIB::MEMSET) == StringRef("memset"))) &&
+ TLI.getLibcallName(RTLIB::MEMSET) == StringRef("memset") &&
+ (!isa<ConstantInt>(Call->getOperand(1)) ||
+ !cast<ConstantInt>(Call->getOperand(1))->isZero() ||
+ !TLI.getLibcallName(RTLIB::BZERO)))) &&
(RetVal == Call->getArgOperand(0) ||
isPointerBitcastEqualTo(RetVal, Call->getArgOperand(0))))
return true;
diff --git a/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
new file mode 100644
index 0000000000000..90e641cd4fe3d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
@@ -0,0 +1,20 @@
+; RUN: llc -o - %s | FileCheck %s
+; RUN: llc -global-isel -o - %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx15.0.0"
+
+define ptr @test() {
+; CHECK-LABEL: test:
+; CHECK-NOT: b _bzero
+ %1 = tail call ptr @fn(i32 noundef 1) #3
+ tail call void @llvm.memset.p0.i64(ptr noundef nonnull align 1 dereferenceable(1000) %1, i8 noundef 0, i64 noundef 1000, i1 noundef false) #3
+ ret ptr %1
+}
+
+declare ptr @fn(i32 noundef)
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write)
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
+
+attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: write) }
+attributes #3 = { nounwind optsize }
>From a316dcf44c9738f39c49c556195df1e231296b0f Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Mon, 15 Jul 2024 15:29:29 -0700
Subject: [PATCH 2/6] Change to positive check in test.
---
llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
index 90e641cd4fe3d..34c6c63cc1798 100644
--- a/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
+++ b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
@@ -5,7 +5,7 @@ target triple = "arm64-apple-macosx15.0.0"
define ptr @test() {
; CHECK-LABEL: test:
-; CHECK-NOT: b _bzero
+; CHECK: bl _bzero
%1 = tail call ptr @fn(i32 noundef 1) #3
tail call void @llvm.memset.p0.i64(ptr noundef nonnull align 1 dereferenceable(1000) %1, i8 noundef 0, i64 noundef 1000, i1 noundef false) #3
ret ptr %1
>From 5aad864dec2a452a1367f118bf8c722317c7cceb Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Tue, 16 Jul 2024 10:28:32 -0700
Subject: [PATCH 3/6] Refactor.
---
llvm/include/llvm/CodeGen/Analysis.h | 14 ++++++--
llvm/include/llvm/CodeGen/SelectionDAG.h | 2 +-
llvm/lib/CodeGen/Analysis.cpp | 36 +++++++++++--------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 18 +++++++---
.../SelectionDAG/SelectionDAGBuilder.cpp | 8 ++---
llvm/lib/Target/X86/X86SelectionDAGInfo.cpp | 2 +-
6 files changed, 53 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 6f7ed22b8ac71..9825e1ddd0bdf 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -126,7 +126,8 @@ ICmpInst::Predicate getICmpCondCode(ISD::CondCode Pred);
/// between it and the return.
///
/// This function only tests target-independent requirements.
-bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM);
+bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM,
+ bool ReturnsFirstArg = false);
/// Test if given that the input instruction is in the tail call position, if
/// there is an attribute mismatch between the caller and the callee that will
@@ -144,7 +145,16 @@ bool attributesPermitTailCall(const Function *F, const Instruction *I,
/// optimization.
bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
const ReturnInst *Ret,
- const TargetLoweringBase &TLI);
+ const TargetLoweringBase &TLI,
+ bool ReturnsFirstArg = false);
+
+/// Check whether B is a bitcast of a pointer type to another pointer type,
+/// which is equal to A.
+bool isPointerBitcastEqualTo(const Value *A, const Value *B);
+
+/// Returns true if the parent of \p CI returns CI's first argument after
+/// calling \p CI.
+bool funcReturnsFirstArgOfCall(const CallInst &CI);
DenseMap<const MachineBasicBlock *, int>
getEHScopeMembership(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 8e189e9e8bf86..5484921973642 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1199,7 +1199,7 @@ class SelectionDAG {
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, bool isTailCall,
+ bool AlwaysInline, const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes());
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 2a3015866da5f..7514e33d47626 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -532,7 +532,8 @@ static bool nextRealType(SmallVectorImpl<Type *> &SubTypes,
/// between it and the return.
///
/// This function only tests target-independent requirements.
-bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM) {
+bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM,
+ bool ReturnsFirstArg) {
const BasicBlock *ExitBB = Call.getParent();
const Instruction *Term = ExitBB->getTerminator();
const ReturnInst *Ret = dyn_cast<ReturnInst>(Term);
@@ -575,7 +576,8 @@ bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM) {
const Function *F = ExitBB->getParent();
return returnTypeIsEligibleForTailCall(
- F, &Call, Ret, *TM.getSubtargetImpl(*F)->getTargetLowering());
+ F, &Call, Ret, *TM.getSubtargetImpl(*F)->getTargetLowering(),
+ ReturnsFirstArg);
}
bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
@@ -638,9 +640,7 @@ bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
return CallerAttrs == CalleeAttrs;
}
-/// Check whether B is a bitcast of a pointer type to another pointer type,
-/// which is equal to A.
-static bool isPointerBitcastEqualTo(const Value *A, const Value *B) {
+bool llvm::isPointerBitcastEqualTo(const Value *A, const Value *B) {
assert(A && B && "Expected non-null inputs!");
auto *BitCastIn = dyn_cast<BitCastInst>(B);
@@ -657,7 +657,8 @@ static bool isPointerBitcastEqualTo(const Value *A, const Value *B) {
bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
const Instruction *I,
const ReturnInst *Ret,
- const TargetLoweringBase &TLI) {
+ const TargetLoweringBase &TLI,
+ bool ReturnsFirstArg) {
// If the block ends with a void return or unreachable, it doesn't matter
// what the call's return type is.
if (!Ret || Ret->getNumOperands() == 0) return true;
@@ -671,26 +672,23 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
if (!attributesPermitTailCall(F, I, Ret, TLI, &AllowDifferingSizes))
return false;
+ // If the return value is the first argument of the call.
+ if (ReturnsFirstArg)
+ return true;
+
const Value *RetVal = Ret->getOperand(0), *CallVal = I;
// Intrinsic like llvm.memcpy has no return value, but the expanded
// libcall may or may not have return value. On most platforms, it
// will be expanded as memcpy in libc, which returns the first
// argument. On other platforms like arm-none-eabi, memcpy may be
// expanded as library call without return value, like __aeabi_memcpy.
- // Similarly, llvm.memset can be expanded to bzero, which doesn't have a
- // return value either.
const CallInst *Call = cast<CallInst>(I);
if (Function *F = Call->getCalledFunction()) {
Intrinsic::ID IID = F->getIntrinsicID();
if (((IID == Intrinsic::memcpy &&
TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) ||
(IID == Intrinsic::memmove &&
- TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove")) ||
- (IID == Intrinsic::memset &&
- TLI.getLibcallName(RTLIB::MEMSET) == StringRef("memset") &&
- (!isa<ConstantInt>(Call->getOperand(1)) ||
- !cast<ConstantInt>(Call->getOperand(1))->isZero() ||
- !TLI.getLibcallName(RTLIB::BZERO)))) &&
+ TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove"))) &&
(RetVal == Call->getArgOperand(0) ||
isPointerBitcastEqualTo(RetVal, Call->getArgOperand(0))))
return true;
@@ -744,6 +742,16 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
return true;
}
+bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
+ const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
+ Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
+ bool ReturnsFirstArg = false;
+ if (RetVal && ((RetVal == CI.getArgOperand(0) ||
+ isPointerBitcastEqualTo(RetVal, CI.getArgOperand(0)))))
+ ReturnsFirstArg = true;
+ return ReturnsFirstArg;
+}
+
static void collectEHScopeMembers(
DenseMap<const MachineBasicBlock *, int> &EHScopeMembership, int EHScope,
const MachineBasicBlock *MBB) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 897bdc71818f8..fcceadf7f9ce8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8454,7 +8454,8 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool AlwaysInline, bool isTailCall,
+ bool isVol, bool AlwaysInline,
+ const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
@@ -8514,8 +8515,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
return Entry;
};
+ bool UseBZero = isNullConstant(Src) && BzeroName;
// If zeroing out and bzero is present, use it.
- if (isNullConstant(Src) && BzeroName) {
+ if (UseBZero) {
TargetLowering::ArgListTy Args;
Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
@@ -8533,8 +8535,16 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
TLI->getPointerTy(DL)),
std::move(Args));
}
-
- CLI.setDiscardResult().setTailCall(isTailCall);
+ bool LowersToMemset =
+ TLI->getLibcallName(RTLIB::MEMSET) == StringRef("memset");
+ // If we're going to use bzero, make sure not to tail call unless the
+ // subsequent return doesn't need a value, as bzero doesn't return the first
+ // arg unlike memset.
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI) && !UseBZero;
+ bool IsTailCall =
+ CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(), ReturnsFirstArg && LowersToMemset);
+ CLI.setDiscardResult().setTailCall(IsTailCall);
std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b0746014daf5a..b17a5815c19b3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6500,11 +6500,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// @llvm.memset defines 0 and 1 to both mean no alignment.
Align Alignment = MSI.getDestAlign().valueOrOne();
bool isVol = MSI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MS = DAG.getMemset(
- Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
- isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
+ Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false, &I,
+ MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
updateDAGForMaybeTailCall(MS);
return;
}
@@ -6517,10 +6516,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// @llvm.memset defines 0 and 1 to both mean no alignment.
Align DstAlign = MSII.getDestAlign().valueOrOne();
bool isVol = MSII.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
- /* AlwaysInline */ true, isTC,
+ /* AlwaysInline */ true, &I,
MachinePointerInfo(I.getArgOperand(0)),
I.getAAMetadata());
updateDAGForMaybeTailCall(MC);
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index e5f07f230fe6c..8f1acd1f1cd60 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -147,7 +147,7 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
DAG.getConstant(Offset, dl, AddrVT)),
Val, DAG.getConstant(BytesLeft, dl, SizeVT), Alignment,
isVolatile, AlwaysInline,
- /* isTailCall */ false, DstPtrInfo.getWithOffset(Offset)));
+ /* CI */ nullptr, DstPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
>From 4e261c8a9632adc451f6823c3b8ecc90e5039916 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Tue, 16 Jul 2024 11:16:53 -0700
Subject: [PATCH 4/6] clang-format
---
llvm/lib/CodeGen/Analysis.cpp | 14 +++++++-------
.../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 4 ++--
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 7514e33d47626..a34fc6c28c430 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -743,13 +743,13 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
}
bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
- const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
- Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
- bool ReturnsFirstArg = false;
- if (RetVal && ((RetVal == CI.getArgOperand(0) ||
- isPointerBitcastEqualTo(RetVal, CI.getArgOperand(0)))))
- ReturnsFirstArg = true;
- return ReturnsFirstArg;
+ const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
+ Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
+ bool ReturnsFirstArg = false;
+ if (RetVal && ((RetVal == CI.getArgOperand(0) ||
+ isPointerBitcastEqualTo(RetVal, CI.getArgOperand(0)))))
+ ReturnsFirstArg = true;
+ return ReturnsFirstArg;
}
static void collectEHScopeMembers(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b17a5815c19b3..155d33ce78fa6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6502,8 +6502,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
bool isVol = MSI.isVolatile();
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MS = DAG.getMemset(
- Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false, &I,
- MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
+ Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
+ &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
updateDAGForMaybeTailCall(MS);
return;
}
>From 9f755d1f5c63cee92bca161caa798cfb748aecd6 Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Tue, 16 Jul 2024 15:11:04 -0700
Subject: [PATCH 5/6] Convert getMemmove()
---
llvm/include/llvm/CodeGen/Analysis.h | 4 ---
llvm/include/llvm/CodeGen/SelectionDAG.h | 6 ++++-
llvm/lib/CodeGen/Analysis.cpp | 26 +++----------------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 18 +++++++++++--
.../SelectionDAG/SelectionDAGBuilder.cpp | 6 ++---
llvm/lib/Target/XCore/XCoreISelLowering.cpp | 2 +-
6 files changed, 29 insertions(+), 33 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 9825e1ddd0bdf..362cc30bbd06a 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -148,10 +148,6 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
const TargetLoweringBase &TLI,
bool ReturnsFirstArg = false);
-/// Check whether B is a bitcast of a pointer type to another pointer type,
-/// which is equal to A.
-bool isPointerBitcastEqualTo(const Value *A, const Value *B);
-
/// Returns true if the parent of \p CI returns CI's first argument after
/// calling \p CI.
bool funcReturnsFirstArgOfCall(const CallInst &CI);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 5484921973642..fac16f97212ec 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1190,8 +1190,12 @@ class SelectionDAG {
const AAMDNodes &AAInfo = AAMDNodes(),
AAResults *AA = nullptr);
+ /* \p CI if not null is the memset call being lowered.
+ * \p OverrideTailCall is an optional parameter that can be used to override
+ * the tail call optimization decision. */
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol, bool isTailCall,
+ SDValue Size, Align Alignment, bool isVol,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index a34fc6c28c430..2d45a171b2b5d 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -640,20 +640,6 @@ bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
return CallerAttrs == CalleeAttrs;
}
-bool llvm::isPointerBitcastEqualTo(const Value *A, const Value *B) {
- assert(A && B && "Expected non-null inputs!");
-
- auto *BitCastIn = dyn_cast<BitCastInst>(B);
-
- if (!BitCastIn)
- return false;
-
- if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
- return false;
-
- return A == BitCastIn->getOperand(0);
-}
-
bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
const Instruction *I,
const ReturnInst *Ret,
@@ -685,12 +671,9 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
const CallInst *Call = cast<CallInst>(I);
if (Function *F = Call->getCalledFunction()) {
Intrinsic::ID IID = F->getIntrinsicID();
- if (((IID == Intrinsic::memcpy &&
- TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) ||
- (IID == Intrinsic::memmove &&
- TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove"))) &&
- (RetVal == Call->getArgOperand(0) ||
- isPointerBitcastEqualTo(RetVal, Call->getArgOperand(0))))
+ if ((IID == Intrinsic::memcpy &&
+ TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) &&
+ (RetVal == Call->getArgOperand(0)))
return true;
}
@@ -746,8 +729,7 @@ bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
bool ReturnsFirstArg = false;
- if (RetVal && ((RetVal == CI.getArgOperand(0) ||
- isPointerBitcastEqualTo(RetVal, CI.getArgOperand(0)))))
+ if (RetVal && ((RetVal == CI.getArgOperand(0))))
ReturnsFirstArg = true;
return ReturnsFirstArg;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index fcceadf7f9ce8..f88806f2d34d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8352,7 +8352,8 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool isTailCall,
+ bool isVol, const CallInst *CI,
+ std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo, AAResults *AA) {
@@ -8398,6 +8399,19 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
+
+ bool IsTailCall = false;
+ if (OverrideTailCall.has_value()) {
+ IsTailCall = *OverrideTailCall;
+ } else {
+ bool LowersToMemmove =
+ TLI->getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove");
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+ IsTailCall = CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(),
+ ReturnsFirstArg && LowersToMemmove);
+ }
+
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
@@ -8406,7 +8420,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
TLI->getPointerTy(getDataLayout())),
std::move(Args))
.setDiscardResult()
- .setTailCall(isTailCall);
+ .setTailCall(IsTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 155d33ce78fa6..a9708f30cf5da 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6534,12 +6534,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Align SrcAlign = MMI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MMI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing different dest/src alignments to the memmove DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
- isTC, MachinePointerInfo(I.getArgOperand(0)),
+ SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,
+ /* OverrideTailCall */ std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)),
I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MM);
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index c81da0365af3d..d3deb2b71e09f 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1704,7 +1704,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
bool isTail = isInTailCallPosition(DAG, ST, Chain);
return DAG.getMemmove(Chain, dl, ST->getBasePtr(), LD->getBasePtr(),
DAG.getConstant(StoreBits / 8, dl, MVT::i32),
- Alignment, false, isTail,
+ Alignment, false, nullptr, isTail,
ST->getPointerInfo(), LD->getPointerInfo());
}
}
>From 4efefbd44083dfc2bc77ffb8c2e1b08a82b8294c Mon Sep 17 00:00:00 2001
From: Amara Emerson <amara at apple.com>
Date: Tue, 16 Jul 2024 17:14:37 -0700
Subject: [PATCH 6/6] Convert memcpy.
---
llvm/include/llvm/CodeGen/SelectionDAG.h | 16 +++++-----
llvm/lib/CodeGen/Analysis.cpp | 14 ---------
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 26 ++++++++++++-----
.../SelectionDAG/SelectionDAGBuilder.cpp | 29 +++++++++----------
.../Target/AArch64/AArch64ISelLowering.cpp | 7 +++--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +-
.../Target/Hexagon/HexagonISelLowering.cpp | 10 +++----
llvm/lib/Target/Lanai/LanaiISelLowering.cpp | 2 +-
.../LoongArch/LoongArchISelLowering.cpp | 2 +-
llvm/lib/Target/MSP430/MSP430ISelLowering.cpp | 11 +++----
llvm/lib/Target/Mips/MipsISelLowering.cpp | 2 +-
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 10 +++----
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +-
llvm/lib/Target/Sparc/SparcISelLowering.cpp | 4 +--
.../Target/SystemZ/SystemZISelLowering.cpp | 2 +-
.../WebAssembly/WebAssemblyISelLowering.cpp | 9 +++---
llvm/lib/Target/X86/X86ISelLowering.cpp | 3 +-
llvm/lib/Target/X86/X86ISelLoweringCall.cpp | 2 +-
llvm/lib/Target/X86/X86SelectionDAGInfo.cpp | 2 +-
llvm/lib/Target/XCore/XCoreISelLowering.cpp | 4 +--
20 files changed, 81 insertions(+), 78 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index fac16f97212ec..16ec65f2e7daa 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1182,13 +1182,15 @@ class SelectionDAG {
/// stack arguments from being clobbered.
SDValue getStackArgumentTokenFactor(SDValue Chain);
- SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, bool isTailCall,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo = AAMDNodes(),
- AAResults *AA = nullptr);
+ /* \p CI if not null is the memset call being lowered.
+ * \p OverrideTailCall is an optional parameter that can be used to override
+ * the tail call optimization decision. */
+ SDValue
+ getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
+ SDValue Size, Align Alignment, bool isVol, bool AlwaysInline,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
+ MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo = AAMDNodes(), AAResults *AA = nullptr);
/* \p CI if not null is the memset call being lowered.
* \p OverrideTailCall is an optional parameter that can be used to override
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 2d45a171b2b5d..128060ec912c7 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -663,20 +663,6 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
return true;
const Value *RetVal = Ret->getOperand(0), *CallVal = I;
- // Intrinsic like llvm.memcpy has no return value, but the expanded
- // libcall may or may not have return value. On most platforms, it
- // will be expanded as memcpy in libc, which returns the first
- // argument. On other platforms like arm-none-eabi, memcpy may be
- // expanded as library call without return value, like __aeabi_memcpy.
- const CallInst *Call = cast<CallInst>(I);
- if (Function *F = Call->getCalledFunction()) {
- Intrinsic::ID IID = F->getIntrinsicID();
- if ((IID == Intrinsic::memcpy &&
- TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) &&
- (RetVal == Call->getArgOperand(0)))
- return true;
- }
-
SmallVector<unsigned, 4> RetPath, CallPath;
SmallVector<Type *, 4> RetSubTypes, CallSubTypes;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f88806f2d34d2..bdb9c8e9ca8f0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -75,6 +75,7 @@
#include <cstdint>
#include <cstdlib>
#include <limits>
+#include <optional>
#include <set>
#include <string>
#include <utility>
@@ -8236,12 +8237,11 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
}
}
-SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
- SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool AlwaysInline, bool isTailCall,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo, AAResults *AA) {
+SDValue SelectionDAG::getMemcpy(
+ SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
+ Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
+ std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
+ MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -8296,6 +8296,18 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
+ bool IsTailCall = false;
+ if (OverrideTailCall.has_value()) {
+ IsTailCall = *OverrideTailCall;
+ } else {
+ bool LowersToMemcpy =
+ TLI->getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy");
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+ IsTailCall = CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(),
+ ReturnsFirstArg && LowersToMemcpy);
+ }
+
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
@@ -8304,7 +8316,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
TLI->getPointerTy(getDataLayout())),
std::move(Args))
.setDiscardResult()
- .setTailCall(isTailCall);
+ .setTailCall(IsTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a9708f30cf5da..8924fdd12a8ca 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6460,14 +6460,14 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Align SrcAlign = MCI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MCI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing different dest/src alignments to the memcpy DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MC = DAG.getMemcpy(
- Root, sdl, Op1, Op2, Op3, Alignment, isVol,
- /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
+ SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
+ /* AlwaysInline */ false, &I, std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)),
+ I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6482,13 +6482,13 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
Align SrcAlign = MCI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MCI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing different dest/src alignments to the memcpy DAG
// node.
- SDValue MC = DAG.getMemcpy(
- getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
- /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
+ SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
+ /* AlwaysInline */ true, &I, std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)),
+ I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -9037,11 +9037,10 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
// because the return pointer needs to be adjusted by the size of
// the copied memory.
SDValue Root = getMemoryRoot();
- SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
- /*isTailCall=*/false,
- MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata());
+ SDValue MC = DAG.getMemcpy(
+ Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,
+ std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
assert(MC.getNode() != nullptr &&
"** memcpy should not be lowered as TailCall in mempcpy context **");
DAG.setRoot(MC);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7294da474c4bc..cfe0ff1c13d25 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8573,7 +8573,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain, DL, DstAddr, Arg, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ false,
- /*isTailCall = */ false, DstInfo, MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, DstInfo, MachinePointerInfo());
MemOpChains.push_back(Cpy);
} else {
@@ -10873,8 +10873,9 @@ SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
DAG.getConstant(VaListSize, DL, MVT::i32),
- Align(PtrSize), false, false, false,
- MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
+ Align(PtrSize), false, false, /*CI=*/nullptr,
+ std::nullopt, MachinePointerInfo(DestSV),
+ MachinePointerInfo(SrcSV));
}
SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a733295d2a511..6e23ddac00430 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3822,7 +3822,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ true,
- /*isTailCall = */ false, DstInfo,
+ /*CI=*/nullptr, std::nullopt, DstInfo,
MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
MemOpChains.push_back(Cpy);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 09fe5264e0955..918fc127762fc 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -178,7 +178,7 @@ static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
return DAG.getMemcpy(
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
/*isVolatile=*/false, /*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
bool
@@ -1038,10 +1038,10 @@ HexagonTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
SDLoc DL(Op);
// Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
// we need to memcopy 12 bytes from va_list to another similar list.
- return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
- DAG.getIntPtrConstant(12, DL), Align(4),
- /*isVolatile*/ false, false, false,
- MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
+ return DAG.getMemcpy(
+ Chain, DL, DestPtr, SrcPtr, DAG.getIntPtrConstant(12, DL), Align(4),
+ /*isVolatile*/ false, false, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
}
SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 8aef45e401d43..c7848356682d0 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -648,7 +648,7 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
/*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(),
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 5c4fddd5116ea..1141232b38dd0 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -4225,7 +4225,7 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
- /*AlwaysInline=*/false, /*isTailCall=*/IsTailCall,
+ /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt,
MachinePointerInfo(), MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index fc066f001316d..ba7b6c85bd81a 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -864,11 +864,12 @@ SDValue MSP430TargetLowering::LowerCCCCallTo(
if (Flags.isByVal()) {
SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i16);
- MemOp = DAG.getMemcpy(
- Chain, dl, PtrOff, Arg, SizeNode, Flags.getNonZeroByValAlign(),
- /*isVolatile*/ false,
- /*AlwaysInline=*/true,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ MemOp = DAG.getMemcpy(Chain, dl, PtrOff, Arg, SizeNode,
+ Flags.getNonZeroByValAlign(),
+ /*isVolatile*/ false,
+ /*AlwaysInline=*/true,
+ /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo());
} else {
MemOp = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
}
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 7d8adb9a00311..8cb64a0e6f4bf 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -4506,7 +4506,7 @@ void MipsTargetLowering::passByValArg(
Chain = DAG.getMemcpy(
Chain, DL, Dst, Src, DAG.getConstant(MemCpySize, DL, PtrTy),
Align(Alignment), /*isVolatile=*/false, /*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
MemOpChains.push_back(Chain);
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 6aabf5cd8c594..bd275c02eadd8 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3904,8 +3904,8 @@ SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
// 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2),
DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8),
- false, true, false, MachinePointerInfo(),
- MachinePointerInfo());
+ false, true, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo());
}
SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
@@ -5275,9 +5275,9 @@ static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
SDValue Chain, ISD::ArgFlagsTy Flags,
SelectionDAG &DAG, const SDLoc &dl) {
SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
- return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode,
- Flags.getNonZeroByValAlign(), false, false, false,
- MachinePointerInfo(), MachinePointerInfo());
+ return DAG.getMemcpy(
+ Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), false, false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index caa4ebacc41da..b1ff74613df14 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19887,7 +19887,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
- /*AlwaysInline=*/false, IsTailCall,
+ /*AlwaysInline=*/false, /*CI*/ nullptr, IsTailCall,
MachinePointerInfo(), MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 0dba6c47be030..50aa19446f880 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -868,8 +868,8 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
Chain = DAG.getMemcpy(Chain, dl, FIPtr, Arg, SizeNode, Alignment,
false, // isVolatile,
(Size <= 32), // AlwaysInline if size <= 32,
- false, // isTailCall
- MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
else {
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 0a7229a2bc0fb..b2b88143354a5 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -3951,7 +3951,7 @@ SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(0) : 32;
return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL),
Align(8), /*isVolatile*/ false, /*AlwaysInline*/ false,
- /*isTailCall*/ false, MachinePointerInfo(DstSV),
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
MachinePointerInfo(SrcSV));
}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index a793e59c3d1a7..f77076d7244ca 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1123,10 +1123,11 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
SDValue SizeNode =
DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
- Chain = DAG.getMemcpy(
- Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
- /*isVolatile*/ false, /*AlwaysInline=*/false,
- /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
+ Chain = DAG.getMemcpy(Chain, DL, FINode, OutVal, SizeNode,
+ Out.Flags.getNonZeroByValAlign(),
+ /*isVolatile*/ false, /*AlwaysInline=*/false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
OutVal = FINode;
}
// Count the number of fixed args *after* legalization.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1d947ac2346d0..e075bb8895eea 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -25166,7 +25166,8 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
Chain, DL, DstPtr, SrcPtr,
DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false,
- false, MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
+ MachinePointerInfo(SrcSV));
}
// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index ed3fb13b2b232..3c49b6ea8ec7b 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -1240,7 +1240,7 @@ static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
return DAG.getMemcpy(
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
/*isVolatile*/ false, /*AlwaysInline=*/true,
- /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
/// Return true if the calling convention is one that we can guarantee TCO for.
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index 8f1acd1f1cd60..055466ac660cc 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -255,7 +255,7 @@ static SDValue emitConstantSizeRepmov(
DAG.getNode(ISD::ADD, dl, DstVT, Dst, DAG.getConstant(Offset, dl, DstVT)),
DAG.getNode(ISD::ADD, dl, SrcVT, Src, DAG.getConstant(Offset, dl, SrcVT)),
DAG.getConstant(BytesLeft, dl, SizeVT), Alignment, isVolatile,
- /*AlwaysInline*/ true, /*isTailCall*/ false,
+ /*AlwaysInline*/ true, /*CI=*/nullptr, std::nullopt,
DstPtrInfo.getWithOffset(Offset), SrcPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index d3deb2b71e09f..f4d32833bce99 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1301,8 +1301,8 @@ SDValue XCoreTargetLowering::LowerCCCArguments(
InVals.push_back(FIN);
MemOps.push_back(DAG.getMemcpy(
Chain, dl, FIN, ArgDI.SDV, DAG.getConstant(Size, dl, MVT::i32),
- Alignment, false, false, false, MachinePointerInfo(),
- MachinePointerInfo()));
+ Alignment, false, false, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo()));
} else {
InVals.push_back(ArgDI.SDV);
}
More information about the llvm-commits
mailing list